sunnypilot v2026.003.000 release

date: 2026-08-19T09:43:43 master commit: ba29a38507
2026-08-21 04:53:47 +08:00 · 2026-08-19 09:43:44 +00:00
commit e01ac7f80f
4046 changed files with 997234 additions and 0 deletions
@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+import time
+from multiprocessing import Process
+
+from openpilot.common.params import Params
+from openpilot.system.manager.process import launcher
+from openpilot.common.swaglog import cloudlog
+from openpilot.common.hardware import HARDWARE
+from openpilot.common.version import get_build_metadata
+
+ATHENA_MGR_PID_PARAM = "AthenadPid"
+
+
+def main():
+  manage_athenad("DongleId", ATHENA_MGR_PID_PARAM, 'athenad', 'openpilot.system.athena.athenad')
+
+
+def manage_athenad(dongle_id_param, pid_param, process_name, target):
+  params = Params()
+  dongle_id = params.get(dongle_id_param)
+  build_metadata = get_build_metadata()
+
+  cloudlog.bind_global(dongle_id=dongle_id,
+                       version=build_metadata.openpilot.version,
+                       origin=build_metadata.openpilot.git_normalized_origin,
+                       branch=build_metadata.channel,
+                       commit=build_metadata.openpilot.git_commit,
+                       dirty=build_metadata.openpilot.is_dirty,
+                       device=HARDWARE.get_device_type())
+
+  try:
+    while 1:
+      cloudlog.info(f"starting {process_name} daemon")
+      proc = Process(name=process_name, target=launcher, args=(target, process_name))
+      proc.start()
+      proc.join()
+      cloudlog.event(f"{process_name} exited", exitcode=proc.exitcode)
+      time.sleep(5)
+  except Exception:
+    cloudlog.exception(f"manage_{process_name}.exception")
+  finally:
+    params.remove(pid_param)
+
+if __name__ == '__main__':
+  main()
@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+import time
+import json
+import jwt
+from typing import cast
+from pathlib import Path
+
+from datetime import datetime, timedelta, UTC
+from openpilot.common.api import api_get, get_key_pair
+from openpilot.common.params import Params
+from openpilot.common.spinner import Spinner
+from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+from openpilot.common.hardware import HARDWARE, PC
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.swaglog import cloudlog
+
+
+UNREGISTERED_DONGLE_ID = "UnregisteredDevice"
+
+def is_registered_device() -> bool:
+  dongle = Params().get("DongleId")
+  return dongle not in (None, UNREGISTERED_DONGLE_ID)
+
+
+def register(show_spinner=False) -> str | None:
+  """
+  All devices built since March 2024 come with all
+  info stored in /persist/. This is kept around
+  only for devices built before then.
+
+  With a backend update to take serial number instead
+  of dongle ID to some endpoints, this can be removed
+  entirely.
+  """
+  params = Params()
+
+  dongle_id: str | None = params.get("DongleId")
+  if dongle_id is None and Path(Paths.persist_root()+"/comma/dongle_id").is_file():
+    # not all devices will have this; added early in comma 3X production (2/28/24)
+    with open(Paths.persist_root()+"/comma/dongle_id") as f:
+      dongle_id = f.read().strip()
+
+  # Create registration token, in the future, this key will make JWTs directly
+  jwt_algo, private_key, public_key = get_key_pair()
+
+  if not public_key:
+    dongle_id = UNREGISTERED_DONGLE_ID
+    cloudlog.warning("missing public key")
+  elif dongle_id is None:
+    if show_spinner:
+      spinner = Spinner()
+      spinner.update("registering device")
+
+    # Block until we get the imei
+    serial = HARDWARE.get_serial()
+    start_time = time.monotonic()
+    imei: str | None = None
+    while imei is None:
+      try:
+        imei = HARDWARE.get_imei()
+      except Exception:
+        cloudlog.exception("Error getting imei, trying again...")
+        time.sleep(1)
+
+      if time.monotonic() - start_time > 60 and show_spinner:
+        spinner.update(f"registering device - serial: {serial}, IMEI: {imei}")
+
+    backoff = 0
+    start_time = time.monotonic()
+    while True:
+      try:
+        register_token = jwt.encode({'register': True, 'exp': datetime.now(UTC).replace(tzinfo=None) + timedelta(hours=1)},
+                                    cast(str, private_key), algorithm=jwt_algo)
+        cloudlog.info("getting pilotauth")
+        cloudlog.info("getting pilotauth")
+        resp = api_get("v2/pilotauth/", method='POST', timeout=15,
+                       imei=imei, imei2="", serial=serial, public_key=public_key, register_token=register_token)
+
+        if resp.status_code in (402, 403):
+          cloudlog.info(f"Unable to register device, got {resp.status_code}")
+          dongle_id = UNREGISTERED_DONGLE_ID
+        else:
+          dongleauth = json.loads(resp.text)
+          dongle_id = dongleauth["dongle_id"]
+        break
+      except NotImplementedError:
+        # dependency issues with PyJWT will hang the registration test in backoff loop otherwise
+        raise
+      except Exception:
+        cloudlog.exception("failed to authenticate")
+        backoff = min(backoff + 1, 15)
+        time.sleep(backoff)
+
+      if time.monotonic() - start_time > 60 and show_spinner:
+        spinner.update(f"registering device - serial: {serial}, IMEI: {imei}")
+        return UNREGISTERED_DONGLE_ID  # hotfix to prevent an infinite wait for registration
+
+    if show_spinner:
+      spinner.close()
+
+  if dongle_id:
+    params.put("DongleId", dongle_id, block=True)
+    set_offroad_alert("Offroad_UnregisteredHardware", (dongle_id == UNREGISTERED_DONGLE_ID) and not PC)
+  return dongle_id
+
+
+if __name__ == "__main__":
+  print(register())
@@ -0,0 +1,116 @@
+import json
+from collections.abc import Callable, Mapping
+from typing import Any
+
+# a minimal implementation of json-rpc 2.0 https://www.jsonrpc.org/specification
+
+JSONRPC_VERSION = "2.0"
+
+# JSON-RPC 2.0 reserved / application error codes
+PARSE_ERROR = -32700
+INVALID_REQUEST = -32600
+METHOD_NOT_FOUND = -32601
+INVALID_PARAMS = -32602
+SERVER_ERROR = -32000
+
+JsonDict = dict[str, Any]
+MethodMap = Mapping[str, Callable[..., Any]]
+
+
+class Dispatcher(dict[str, Callable[..., Any]]):
+  def add_method(self, f: Callable[..., Any] | None = None, *, name: str | None = None):
+    def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+      self[name or fn.__name__] = fn
+      return fn
+    return decorator(f) if f is not None else decorator
+
+
+dispatcher = Dispatcher()
+
+
+def dumps_call(method: str, params: Any = None, request_id: Any = None) -> str:
+  msg: JsonDict = {"jsonrpc": JSONRPC_VERSION, "method": method, "id": request_id}
+  if params is not None:
+    msg["params"] = params
+  return json.dumps(msg)
+
+
+def dumps_result(request_id: Any, result: Any) -> str:
+  return json.dumps({"jsonrpc": JSONRPC_VERSION, "id": request_id, "result": result})
+
+
+def dumps_error(request_id: Any, message: str, code: int = SERVER_ERROR) -> str:
+  return json.dumps({
+    "jsonrpc": JSONRPC_VERSION,
+    "id": request_id,
+    "error": {"code": code, "message": message},
+  })
+
+
+def loads(raw: str | bytes) -> JsonDict:
+  if isinstance(raw, bytes):
+    raw = raw.decode()
+  data = json.loads(raw)
+  if not isinstance(data, dict):
+    raise ValueError("message must be a JSON object")
+  return data
+
+
+def is_call(msg: JsonDict) -> bool:
+  return "method" in msg
+
+
+def is_response(msg: JsonDict) -> bool:
+  return "id" in msg and ("result" in msg or "error" in msg)
+
+
+def error_message(err: Any) -> str:
+  """Normalize JSON-RPC object errors and plain-string errors."""
+  if isinstance(err, str):
+    return err
+  if isinstance(err, dict):
+    data = err.get("data")
+    if isinstance(data, dict) and data.get("message"):
+      return str(data["message"])
+    if err.get("message") is not None:
+      return str(err["message"])
+  return str(err)
+
+
+def _invoke(fn: Callable[..., Any], params: Any) -> Any:
+  if params is None:
+    return fn()
+  if isinstance(params, dict):
+    return fn(**params)
+  if isinstance(params, (list, tuple)):
+    return fn(*params)
+  raise TypeError("params must be a list, object, or omitted")
+
+
+def handle(raw: str | bytes | JsonDict, methods: MethodMap | None = None) -> str:
+  methods = dispatcher if methods is None else methods
+
+  try:
+    msg = raw if isinstance(raw, dict) else loads(raw)
+  except (TypeError, ValueError, UnicodeDecodeError):
+    return dumps_error(None, "parse error", PARSE_ERROR)
+
+  if not is_call(msg):
+    raise ValueError("not a call")
+
+  req_id = msg.get("id")
+  name = msg.get("method")
+  if not isinstance(name, str):
+    return dumps_error(req_id, "invalid request", INVALID_REQUEST)
+
+  try:
+    fn = methods[name]
+  except KeyError:
+    return dumps_error(req_id, f"method not found: {name}", METHOD_NOT_FOUND)
+
+  try:
+    return dumps_result(req_id, _invoke(fn, msg.get("params")))
+  except TypeError as e:
+    return dumps_error(req_id, str(e), INVALID_PARAMS)
+  except Exception as e:
+    return dumps_error(req_id, str(e), SERVER_ERROR)
@@ -0,0 +1,69 @@
+import http.server
+import socket
+
+
+class MockResponse:
+  def __init__(self, json, status_code):
+    self.json = json
+    self.text = json
+    self.status_code = status_code
+
+
+class EchoSocket:
+  def __init__(self, port):
+    self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    self.socket.bind(('127.0.0.1', port))
+    self.socket.listen(1)
+
+  def run(self):
+    conn, _ = self.socket.accept()
+    conn.settimeout(5.0)
+
+    try:
+      while True:
+        data = conn.recv(4096)
+        if data:
+          print(f'EchoSocket got {data}')
+          conn.sendall(data)
+        else:
+          break
+    finally:
+      conn.shutdown(0)
+      conn.close()
+      self.socket.shutdown(0)
+      self.socket.close()
+
+
+class MockApi:
+  def __init__(self, dongle_id):
+    pass
+
+  def get_token(self):
+    return "fake-token"
+
+
+class MockWebsocket:
+  def __init__(self, recv_queue, send_queue):
+    self.recv_queue = recv_queue
+    self.send_queue = send_queue
+    self.sock = socket.socket()
+
+  def recv(self):
+    data = self.recv_queue.get()
+    if isinstance(data, Exception):
+      raise data
+    return data
+
+  def send(self, data, opcode):
+    self.send_queue.put_nowait((data, opcode))
+
+  def close(self):
+    self.sock.close()
+
+
+class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+  def do_PUT(self):
+    length = int(self.headers['Content-Length'])
+    self.rfile.read(length)
+    self.send_response(201, "Created")
+    self.end_headers()
@@ -0,0 +1,507 @@
+from functools import wraps
+import json
+import multiprocessing
+import os
+import requests
+import shutil
+import time
+import threading
+import queue
+from dataclasses import asdict, replace
+from datetime import datetime, timedelta
+
+from websocket import ABNF
+from websocket._exceptions import WebSocketConnectionClosedException
+
+from openpilot.common.parameterized import parameterized
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.cereal import messaging
+
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.system.athena import athenad
+from openpilot.system.athena.athenad import MAX_RETRY_COUNT, UPLOAD_SESS, dispatcher
+from openpilot.system.athena.tests.helpers import HTTPRequestHandler, MockWebsocket, MockApi, EchoSocket
+from openpilot.selfdrive.test.helpers import http_server_context
+from openpilot.common.hardware.hw import Paths
+
+
+def seed_athena_server(host, port):
+  with Timeout(2, 'HTTP Server seeding failed'):
+    while True:
+      try:
+        UPLOAD_SESS.put(f'http://{host}:{port}/qlog.zst', data='', timeout=10)
+        break
+      except requests.exceptions.ConnectionError:
+        time.sleep(0.1)
+
+def with_upload_handler(func):
+  @wraps(func)
+  def wrapper(*args, **kwargs):
+    end_event = threading.Event()
+    thread = threading.Thread(target=athenad.upload_handler, args=(end_event,))
+    thread.start()
+    try:
+      return func(*args, **kwargs)
+    finally:
+      end_event.set()
+      thread.join()
+  return wrapper
+
+def mock_create_connection(mocker):
+  return mocker.patch('openpilot.system.athena.athenad.create_connection')
+
+def host():
+  with http_server_context(handler=HTTPRequestHandler, setup=seed_athena_server) as (host, port):
+    yield f"http://{host}:{port}"
+
+class TestAthenadMethods(OpenpilotTestCase):
+  @classmethod
+  def setup_class(cls):
+    cls.SOCKET_PORT = 45454
+    athenad.Api = MockApi  # ty: ignore[invalid-assignment]  # test double
+    athenad.LOCAL_PORT_WHITELIST = {cls.SOCKET_PORT}
+
+  def setup_method(self):
+    self.default_params = {
+      "DongleId": "0000000000000000",
+      "GithubSshKeys": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC307aE+nuHzTAgaJhzSf5v7ZZQW9gaperjhCmyPyl4PzY7T1mDGenTlVTN7yoVFZ9UfO9oMQqo0n1OwDIiqbIFxqnhrHU0cYfj88rI85m5BEKlNu5RdaVTj1tcbaPpQc5kZEolaI1nDDjzV0lwS7jo5VYDHseiJHlik3HH1SgtdtsuamGR2T80q1SyW+5rHoMOJG73IH2553NnWuikKiuikGHUYBd00K1ilVAK2xSiMWJp55tQfZ0ecr9QjEsJ+J/efL4HqGNXhffxvypCXvbUYAFSddOwXUPo5BTKevpxMtH+2YrkpSjocWA04VnTYFiPG6U4ItKmbLOTFZtPzoez private", # noqa: E501
+      "GithubUsername": "commaci",
+      "AthenadUploadQueue": [],
+    }
+
+    self.params = Params()
+    for k, v in self.default_params.items():
+      self.params.put(k, v, block=True)
+    self.params.put_bool("GsmMetered", True, block=True)
+
+    athenad.upload_queue = queue.PriorityQueue()
+    athenad.cur_upload_items.clear()
+    athenad.cancelled_uploads.clear()
+
+    for i in os.listdir(Paths.log_root()):
+      p = os.path.join(Paths.log_root(), i)
+      if os.path.isdir(p):
+        shutil.rmtree(p)
+      else:
+        os.unlink(p)
+
+  # *** test helpers ***
+
+  @staticmethod
+  def _wait_for_upload():
+    now = time.monotonic()
+    while time.monotonic() - now < 5:
+      if athenad.upload_queue.qsize() == 0:
+        break
+
+  @staticmethod
+  def _create_file(file: str, parent: str | None = None, data: bytes = b'') -> str:
+    fn = os.path.join(Paths.log_root() if parent is None else parent, file)
+    os.makedirs(os.path.dirname(fn), exist_ok=True)
+    with open(fn, 'wb') as f:
+      f.write(data)
+    return fn
+
+  @staticmethod
+  def _video_clips(clip):
+    clips = object.__new__(athenad.VideoClips)
+    clips.lock = threading.Condition()
+    clips.clips = {clip.filename: clip}
+    clips.transcode_proc = None
+    return clips
+
+
+  # *** test cases ***
+
+  def test_echo(self):
+    assert dispatcher["echo"]("bob") == "bob"
+
+  def test_get_message(self):
+    with self.assertRaises(TimeoutError) as _:
+      dispatcher["getMessage"]("controlsState")
+
+    end_event = multiprocessing.Event()
+
+    pub_sock = messaging.pub_sock("deviceState")
+
+    def send_deviceState():
+      while not end_event.is_set():
+        msg = messaging.new_message('deviceState')
+        pub_sock.send(msg.to_bytes())
+        time.sleep(0.01)
+
+    p = multiprocessing.Process(target=send_deviceState)
+    p.start()
+    time.sleep(0.1)
+    try:
+      deviceState = dispatcher["getMessage"]("deviceState")
+      assert deviceState['deviceState']
+    finally:
+      end_event.set()
+      p.join()
+
+  def test_list_data_directory(self):
+    route = '2021-03-29--13-32-47'
+    segments = [0, 1, 2, 3, 11]
+
+    filenames = ['qlog.zst', 'qcamera.ts', 'rlog.zst', 'fcamera.hevc', 'ecamera.hevc', 'dcamera.hevc']
+    files = [f'{route}--{s}/{f}' for s in segments for f in filenames]
+    for file in files:
+      self._create_file(file)
+
+    resp = dispatcher["listDataDirectory"]()
+    assert resp, 'list empty!'
+    assert len(resp) == len(files)
+
+    resp = dispatcher["listDataDirectory"](f'{route}--123')
+    assert len(resp) == 0
+
+    prefix = f'{route}'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+    prefix = f'{route}--1'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+    prefix = f'{route}--1/'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+    prefix = f'{route}--1/q'
+    expected = list(filter(lambda f: f.startswith(prefix), files))
+    resp = dispatcher["listDataDirectory"](prefix)
+    assert resp, 'list empty!'
+    assert len(resp) == len(expected)
+
+  def test_video_clip_hardware_encoder(self, mocker):
+    clip = athenad.VideoClips.Clip("route", "fcamera.hevc", 10, 130, 2, 4, "clip.mp4", 123)
+    clips = self._video_clips(clip)
+    process = mocker.Mock(stdin=None, returncode=0)
+    process.poll.return_value = 0
+    popen = mocker.patch("openpilot.system.athena.athenad.subprocess.Popen", return_value=process)
+    mocker.patch.object(athenad, "PC", False)
+
+    clips._encode(clip, ["segment0", "segment1"], "output.mp4", 10, 120)
+
+    metadata = json.dumps(asdict(clip), separators=(',', ':'))
+    assert popen.call_args.args[0] == [
+      os.path.join(athenad.BASEDIR, "openpilot/system/loggerd/encoderd"), "--clip", "output.mp4", "10", "120",
+      "--bitrate", "2000000", "--speedup", "4", "--metadata", metadata, "--", "segment0", "segment1",
+    ]
+    assert popen.call_args.kwargs["stdin"] == athenad.subprocess.DEVNULL
+    assert clips.transcode_proc is None
+
+  def test_video_clip_hardware_encoder_failure(self, mocker):
+    clip = athenad.VideoClips.Clip("route", "fcamera.hevc", 0, 60, 1, 1, "clip.mp4", 123)
+    clips = self._video_clips(clip)
+    process = mocker.Mock(stdin=None, returncode=1)
+    process.poll.return_value = 1
+    mocker.patch("openpilot.system.athena.athenad.subprocess.Popen", return_value=process)
+    mocker.patch.object(athenad, "PC", False)
+
+    with self.assertRaisesRegex(RuntimeError, "clip encoder exited with code 1"):
+      clips._encode(clip, ["segment"], "output.mp4", 0, 60)
+    assert clips.transcode_proc is None
+
+  def test_video_clip_software_fallback(self, mocker):
+    clip = athenad.VideoClips.Clip("route", "fcamera.hevc", 10, 30, 3, 2, "clip.mp4", 123)
+    clips = self._video_clips(clip)
+    stdin = mocker.Mock()
+    process = mocker.Mock(stdin=stdin, returncode=0)
+    process.poll.return_value = 0
+    popen = mocker.patch("openpilot.system.athena.athenad.subprocess.Popen", return_value=process)
+    mocker.patch.object(athenad, "PC", True)
+
+    clips._encode(clip, ["segment'0", "segment1"], "output.mp4", 10, 20)
+
+    command = popen.call_args.args[0]
+    assert ["-r", "40"] == command[command.index("-r"):command.index("-r") + 2]
+    assert ["-ss", "5.0"] == command[command.index("-ss"):command.index("-ss") + 2]
+    assert ["-t", "10.0"] == command[command.index("-t"):command.index("-t") + 2]
+    assert ["-b:v", "3M"] == command[command.index("-b:v"):command.index("-b:v") + 2]
+    writes = [call.args[0] for call in stdin.write.call_args_list]
+    assert "file 'file:segment'\\''0'\n" in writes[1]
+    assert writes[-1].startswith("file 'file:segment1'")
+
+  def test_strip_extension(self):
+    # any requested log file with an invalid extension won't return as existing
+    fn = self._create_file('qlog.bz2')
+    if fn.endswith('.bz2'):
+      assert athenad.strip_zst_extension(fn) == fn
+
+    fn = self._create_file('qlog.zst')
+    if fn.endswith('.zst'):
+      assert athenad.strip_zst_extension(fn) == fn[:-4]
+
+  @parameterized.expand([True, False], names=("compress",))
+  def test_do_upload(self, host, compress):
+    # random bytes to ensure rather large object post-compression
+    fn = self._create_file('qlog', data=os.urandom(10000 * 1024))
+
+    upload_fn = fn + ('.zst' if compress else '')
+    item = athenad.UploadItem(path=upload_fn, url="http://localhost:1238", headers={}, created_at=int(time.time()*1000), id='')  # noqa: TID251
+    with self.assertRaises(requests.exceptions.ConnectionError):
+      athenad._do_upload(item)
+
+    item = athenad.UploadItem(path=upload_fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='')  # noqa: TID251
+    resp = athenad._do_upload(item)
+    assert resp.status_code == 201
+
+  def test_upload_file_to_url(self, host):
+    fn = self._create_file('qlog.zst')
+
+    resp = dispatcher["uploadFileToUrl"]("qlog.zst", f"{host}/qlog.zst", {})
+    assert resp['enqueued'] == 1
+    assert 'failed' not in resp
+    assert {"path": fn, "url": f"{host}/qlog.zst", "headers": {}}.items() <= resp['items'][0].items()
+    assert resp['items'][0].get('id') is not None
+    assert athenad.upload_queue.qsize() == 1
+
+  def test_upload_file_to_url_duplicate(self, host):
+    self._create_file('qlog.zst')
+
+    url1 = f"{host}/qlog.zst?sig=sig1"
+    dispatcher["uploadFileToUrl"]("qlog.zst", url1, {})
+
+    # Upload same file again, but with different signature
+    url2 = f"{host}/qlog.zst?sig=sig2"
+    resp = dispatcher["uploadFileToUrl"]("qlog.zst", url2, {})
+    assert resp == {'enqueued': 0, 'items': []}
+
+  def test_upload_file_to_url_does_not_exist(self, host):
+    not_exists_resp = dispatcher["uploadFileToUrl"]("does_not_exist.zst", "http://localhost:1238", {})
+    assert not_exists_resp == {'enqueued': 0, 'items': [], 'failed': ['does_not_exist.zst']}
+
+  @with_upload_handler
+  def test_upload_handler(self, host):
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)  # noqa: TID251
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    # TODO: verify that upload actually succeeded
+    # TODO: also check that end_event and metered network raises AbortTransferException
+    assert athenad.upload_queue.qsize() == 0
+
+  @parameterized.expand([(500,True), (412,False)], names=("status", "retry"))
+  @with_upload_handler
+  def test_upload_handler_retry(self, mocker, host, status, retry):
+    mock_put = mocker.patch('openpilot.system.athena.athenad.UPLOAD_SESS.put')
+    mock_put.return_value.__enter__.return_value.status_code = status
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)  # noqa: TID251
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    assert athenad.upload_queue.qsize() == (1 if retry else 0)
+
+    if retry:
+      assert athenad.upload_queue.get().retry_count == 1
+
+  @with_upload_handler
+  def test_upload_handler_timeout(self):
+    """When an upload times out or fails to connect it should be placed back in the queue"""
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)  # noqa: TID251
+    item_no_retry = replace(item, retry_count=MAX_RETRY_COUNT)
+
+    athenad.upload_queue.put_nowait(item_no_retry)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    # Check that upload with retry count exceeded is not put back
+    assert athenad.upload_queue.qsize() == 0
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    # Check that upload item was put back in the queue with incremented retry count
+    assert athenad.upload_queue.qsize() == 1
+    assert athenad.upload_queue.get().retry_count == 1
+
+  @with_upload_handler
+  def test_cancel_upload(self):
+    item = athenad.UploadItem(path="qlog.zst", url="http://localhost:44444/qlog.zst", headers={},
+                              created_at=int(time.time()*1000), id='id', allow_cellular=True)  # noqa: TID251
+    athenad.upload_queue.put_nowait(item)
+    dispatcher["cancelUpload"](item.id)
+
+    assert item.id in athenad.cancelled_uploads
+
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    assert athenad.upload_queue.qsize() == 0
+    assert len(athenad.cancelled_uploads) == 0
+
+  @with_upload_handler
+  def test_cancel_expiry(self):
+    t_future = datetime.now() - timedelta(days=40)
+    ts = int(t_future.strftime("%s")) * 1000
+
+    # Item that would time out if actually uploaded
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url="http://localhost:44444/qlog.zst", headers={}, created_at=ts, id='', allow_cellular=True)
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+    time.sleep(0.1)
+
+    assert athenad.upload_queue.qsize() == 0
+
+  def test_list_upload_queue_empty(self):
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 0
+
+  @with_upload_handler
+  def test_list_upload_queue_current(self, host: str):
+    fn = self._create_file('qlog.zst')
+    item = athenad.UploadItem(path=fn, url=f"{host}/qlog.zst", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)  # noqa: TID251
+
+    athenad.upload_queue.put_nowait(item)
+    self._wait_for_upload()
+
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 1
+    assert items[0]['current']
+
+  def test_list_upload_queue_priority(self):
+    priorities = (25, 50, 99, 75, 0)
+
+    for i in priorities:
+      fn = f'qlog_{i}.zst'
+      fp = self._create_file(fn)
+      item = athenad.UploadItem(
+        path=fp,
+        url=f"http://localhost:44444/{fn}",
+        headers={},
+        created_at=int(time.time()*1000),  # noqa: TID251
+        id='',
+        allow_cellular=True,
+        priority=i
+      )
+      athenad.upload_queue.put_nowait(item)
+
+    for i in sorted(priorities):
+      assert athenad.upload_queue.get_nowait().priority == i
+
+  def test_list_upload_queue(self):
+    item = athenad.UploadItem(path="qlog.zst", url="http://localhost:44444/qlog.zst", headers={},
+                              created_at=int(time.time()*1000), id='id', allow_cellular=True)  # noqa: TID251
+    athenad.upload_queue.put_nowait(item)
+
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 1
+    assert items[0] == asdict(item)
+    assert not items[0]['current']
+
+    assert item.id is not None
+    athenad.cancelled_uploads.add(item.id)
+    items = dispatcher["listUploadQueue"]()
+    assert len(items) == 0
+
+  def test_upload_queue_persistence(self):
+    item1 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id1')  # noqa: TID251
+    item2 = athenad.UploadItem(path="_", url="_", headers={}, created_at=int(time.time()), id='id2')  # noqa: TID251
+
+    athenad.upload_queue.put_nowait(item1)
+    athenad.upload_queue.put_nowait(item2)
+
+    # Ensure canceled items are not persisted
+    assert item2.id is not None
+    athenad.cancelled_uploads.add(item2.id)
+
+    # serialize item
+    athenad.UploadQueueCache.cache(athenad.upload_queue)
+
+    # deserialize item
+    athenad.upload_queue.queue.clear()
+    athenad.UploadQueueCache.initialize(athenad.upload_queue)
+
+    assert athenad.upload_queue.qsize() == 1
+    assert asdict(athenad.upload_queue.queue[-1]) == asdict(item1)
+
+  def test_start_local_proxy(self, mock_create_connection):
+    end_event = threading.Event()
+
+    ws_recv = queue.Queue()
+    ws_send = queue.Queue()
+    mock_ws = MockWebsocket(ws_recv, ws_send)
+    mock_create_connection.return_value = mock_ws
+
+    echo_socket = EchoSocket(self.SOCKET_PORT)
+    socket_thread = threading.Thread(target=echo_socket.run)
+    socket_thread.start()
+
+    athenad.startLocalProxy(end_event, 'ws://localhost:1234', self.SOCKET_PORT)
+
+    ws_recv.put_nowait(b'ping')
+    try:
+      recv = ws_send.get(timeout=5)
+      assert recv == (b'ping', ABNF.OPCODE_BINARY), recv
+    finally:
+      # signal websocket close to athenad.ws_proxy_recv
+      ws_recv.put_nowait(WebSocketConnectionClosedException())
+      socket_thread.join()
+
+  def test_get_ssh_authorized_keys(self):
+    keys = dispatcher["getSshAuthorizedKeys"]()
+    assert keys == self.default_params["GithubSshKeys"]
+
+  def test_get_github_username(self):
+    keys = dispatcher["getGithubUsername"]()
+    assert keys == self.default_params["GithubUsername"]
+
+  def test_get_version(self):
+    resp = dispatcher["getVersion"]()
+    keys = ["version", "remote", "branch", "commit", "commit_date"]
+    assert list(resp.keys()) == keys
+    for k in keys:
+      assert isinstance(resp[k], str), f"{k} is not a string"
+      assert len(resp[k]) > 0, f"{k} has no value"
+
+  def test_jsonrpc_handler(self):
+    end_event = threading.Event()
+    thread = threading.Thread(target=athenad.jsonrpc_handler, args=(end_event,))
+    thread.daemon = True
+    thread.start()
+    try:
+      # with params
+      athenad.recv_queue.put_nowait(json.dumps({"method": "echo", "params": ["hello"], "jsonrpc": "2.0", "id": 0}))
+      _, _, resp = athenad.send_queue.get(timeout=3)
+      assert json.loads(resp) == {'result': 'hello', 'id': 0, 'jsonrpc': '2.0'}
+      # without params
+      athenad.recv_queue.put_nowait(json.dumps({"method": "getNetworkType", "jsonrpc": "2.0", "id": 0}))
+      _, _, resp = athenad.send_queue.get(timeout=3)
+      assert json.loads(resp) == {'result': 1, 'id': 0, 'jsonrpc': '2.0'}
+      # log forwarding
+      athenad.recv_queue.put_nowait(json.dumps({'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}))
+      resp = athenad.log_recv_queue.get(timeout=3)
+      assert json.loads(resp) == {'result': {'success': 1}, 'id': 0, 'jsonrpc': '2.0'}
+    finally:
+      end_event.set()
+      thread.join()
+
+  def test_get_logs_to_send_sorted(self):
+    fl = []
+    for i in range(10):
+      file = f'swaglog.{i:010}'
+      self._create_file(file, Paths.swaglog_root())
+      fl.append(file)
+
+    # ensure the list is all logs except most recent
+    sl = athenad.get_logs_to_send_sorted()
+    assert sl == fl[:-1]
@@ -0,0 +1,102 @@
+import unittest
+import subprocess
+import threading
+import time
+from typing import cast
+
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.system.athena import athenad
+from openpilot.common.hardware import COMMA_HARDWARE
+
+TIMEOUT_TOLERANCE = 20  # seconds
+
+
+def wifi_radio(on: bool) -> None:
+  if not COMMA_HARDWARE:
+    return
+  print(f"wifi {'on' if on else 'off'}")
+  subprocess.run(["nmcli", "radio", "wifi", "on" if on else "off"], check=True)
+
+
+class TestAthenadPing(OpenpilotTestCase):
+  params: Params
+  dongle_id: str
+
+  athenad: threading.Thread
+  exit_event: threading.Event
+
+  def _get_ping_time(self) -> str | None:
+    return cast(str | None, self.params.get("LastAthenaPingTime"))
+
+  def _clear_ping_time(self) -> None:
+    self.params.remove("LastAthenaPingTime")
+
+  def _received_ping(self) -> bool:
+    return self._get_ping_time() is not None
+
+  @classmethod
+  def teardown_class(cls) -> None:
+    wifi_radio(True)
+
+  def setup_method(self) -> None:
+    self.params = Params()
+    self.dongle_id = self.params.get("DongleId")
+
+    wifi_radio(True)
+    self._clear_ping_time()
+
+    self.exit_event = threading.Event()
+    self.athenad = threading.Thread(target=athenad.main, args=(self.exit_event,))
+
+  def teardown_method(self) -> None:
+    if self.athenad.is_alive():
+      self.exit_event.set()
+      self.athenad.join()
+
+  def assertTimeout(self, reconnect_time: float, subtests, mocker) -> None:
+    self.athenad.start()
+
+    mock_create_connection = mocker.patch('openpilot.system.athena.athenad.create_connection',
+                                          new_callable=lambda: mocker.MagicMock(wraps=athenad.create_connection))
+
+    time.sleep(1)
+    mock_create_connection.assert_called_once()
+    mock_create_connection.reset_mock()
+
+    # check normal behavior, server pings on connection
+    with subtests.test("Wi-Fi: receives ping"), Timeout(70, "no ping received"):
+      while not self._received_ping():
+        time.sleep(0.1)
+      print("ping received")
+
+    mock_create_connection.assert_not_called()
+
+    # websocket should attempt reconnect after short time
+    with subtests.test("LTE: attempt reconnect"):
+      wifi_radio(False)
+      print("waiting for reconnect attempt")
+      start_time = time.monotonic()
+      with Timeout(reconnect_time, "no reconnect attempt"):
+        while not mock_create_connection.called:
+          time.sleep(0.1)
+        print(f"reconnect attempt after {time.monotonic() - start_time:.2f}s")
+
+    self._clear_ping_time()
+
+    # check ping received after reconnect
+    with subtests.test("LTE: receives ping"), Timeout(70, "no ping received"):
+      while not self._received_ping():
+        time.sleep(0.1)
+      print("ping received")
+
+  @unittest.skipIf(not COMMA_HARDWARE, "only run on desk")
+  def test_offroad(self, subtests, mocker) -> None:
+    self.params.put_bool("IsOffroad", True, block=True)
+    self.assertTimeout(60 + TIMEOUT_TOLERANCE, subtests, mocker)  # based using TCP keepalive settings
+
+  @unittest.skipIf(not COMMA_HARDWARE, "only run on desk")
+  def test_onroad(self, subtests, mocker) -> None:
+    self.params.put_bool("IsOffroad", False, block=True)
+    self.assertTimeout(21 + TIMEOUT_TOLERANCE, subtests, mocker)
@@ -0,0 +1,77 @@
+import json
+from Crypto.PublicKey import RSA
+from pathlib import Path
+
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.common.params import Params
+from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
+from openpilot.system.athena.tests.helpers import MockResponse
+from openpilot.common.hardware.hw import Paths
+
+
+class TestRegistration(OpenpilotTestCase):
+
+  def setup_method(self):
+    # clear params and setup key paths
+    self.params = Params()
+
+    persist_dir = Path(Paths.persist_root()) / "comma"
+    persist_dir.mkdir(parents=True, exist_ok=True)
+
+    self.priv_key = persist_dir / "id_rsa"
+    self.pub_key = persist_dir / "id_rsa.pub"
+    self.dongle_id = persist_dir / "dongle_id"
+
+  def _generate_keys(self):
+    self.pub_key.touch()
+    k = RSA.generate(2048)
+    with open(self.priv_key, "wb") as f:
+      f.write(k.export_key())
+    with open(self.pub_key, "wb") as f:
+      f.write(k.publickey().export_key())
+
+  def test_valid_cache(self, mocker):
+    # if all params are written, return the cached dongle id.
+    # should work with a dongle ID on either /persist/ or normal params
+    self._generate_keys()
+
+    dongle = "DONGLE_ID_123"
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    for persist, params in [(True, True), (True, False), (False, True)]:
+      self.params.put("DongleId", dongle if params else "", block=True)
+      with open(self.dongle_id, "w") as f:
+        f.write(dongle if persist else "")
+      assert register() == dongle
+      assert not m.called
+
+  def test_no_keys(self, mocker):
+    # missing pubkey
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    dongle = register()
+    assert m.call_count == 0
+    assert dongle == UNREGISTERED_DONGLE_ID
+    assert self.params.get("DongleId") == dongle
+
+  def test_missing_cache(self, mocker):
+    # keys exist but no dongle id
+    self._generate_keys()
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    dongle = "DONGLE_ID_123"
+    m.return_value = MockResponse(json.dumps({'dongle_id': dongle}), 200)
+    assert register() == dongle
+    assert m.call_count == 1
+
+    # call again, shouldn't hit the API this time
+    assert register() == dongle
+    assert m.call_count == 1
+    assert self.params.get("DongleId") == dongle
+
+  def test_unregistered(self, mocker):
+    # keys exist, but unregistered
+    self._generate_keys()
+    m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True)
+    m.return_value = MockResponse(None, 402)
+    dongle = register()
+    assert m.call_count == 1
+    assert dongle == UNREGISTERED_DONGLE_ID
+    assert self.params.get("DongleId") == dongle
@@ -0,0 +1,8 @@
+Import('env', 'arch', 'messaging', 'common', 'visionipc')
+
+libs = [common, messaging, visionipc]
+
+if arch != "Darwin":
+  camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/spectra.cc',
+                           'cameras/cdm.cc', 'sensors/ox03c10.cc', 'sensors/os04c10.cc'])
+  env.Program('camerad', ['main.cc', camera_obj], LIBS=libs)
@@ -0,0 +1,109 @@
+#include "system/camerad/cameras/camera_common.h"
+
+#include <cassert>
+#include <string>
+
+#include "common/swaglog.h"
+#include "system/camerad/cameras/spectra.h"
+
+
+void CameraBuf::init(SpectraCamera *cam, VisionIpcServer * v, int frame_cnt, VisionStreamType type) {
+  vipc_server = v;
+  stream_type = type;
+  frame_buf_count = frame_cnt;
+
+  const SensorInfo *sensor = cam->sensor.get();
+
+  // RAW frames from ISP
+  if (cam->cc.output_type != ISP_IFE_PROCESSED) {
+    camera_bufs_raw = std::make_unique<VisionBuf[]>(frame_buf_count);
+
+    const int raw_frame_size = (sensor->frame_height + sensor->extra_height) * sensor->frame_stride;
+    for (int i = 0; i < frame_buf_count; i++) {
+      camera_bufs_raw[i].allocate(raw_frame_size);
+    }
+    LOGD("allocated %d buffers", frame_buf_count);
+  }
+
+  vipc_server->create_buffers_with_sizes(stream_type, VIPC_BUFFER_COUNT, out_img_width, out_img_height, cam->yuv_size, cam->stride, cam->uv_offset);
+  LOGD("created %d YUV vipc buffers with size %dx%d", VIPC_BUFFER_COUNT, cam->stride, cam->y_height);
+}
+
+CameraBuf::~CameraBuf() {
+  if (camera_bufs_raw != nullptr) {
+    for (int i = 0; i < frame_buf_count; i++) {
+      camera_bufs_raw[i].free();
+    }
+  }
+}
+
+void CameraBuf::sendFrameToVipc() {
+  assert(cur_buf_idx >=0 && cur_buf_idx < frame_buf_count);
+
+  if (camera_bufs_raw) {
+    cur_camera_buf = &camera_bufs_raw[cur_buf_idx];
+  }
+
+  cur_yuv_buf = vipc_server->get_buffer(stream_type, cur_buf_idx);
+
+  VisionIpcBufExtra extra = {
+    cur_frame_data.frame_id,
+    cur_frame_data.timestamp_sof,
+    cur_frame_data.timestamp_eof,
+  };
+  cur_yuv_buf->set_frame_id(cur_frame_data.frame_id);
+  vipc_server->send(cur_yuv_buf, &extra);
+}
+
+// common functions
+
+kj::Array<uint8_t> get_raw_frame_image(const CameraBuf *b) {
+  const uint8_t *dat = (const uint8_t *)b->cur_camera_buf->addr;
+
+  kj::Array<uint8_t> frame_image = kj::heapArray<uint8_t>(b->cur_camera_buf->len);
+  uint8_t *resized_dat = frame_image.begin();
+
+  memcpy(resized_dat, dat, b->cur_camera_buf->len);
+
+  return kj::mv(frame_image);
+}
+
+float calculate_exposure_value(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip) {
+  int lum_med;
+  uint32_t lum_binning[256] = {0};
+  const uint8_t *pix_ptr = b->cur_yuv_buf->y;
+
+  unsigned int lum_total = 0;
+  for (int y = ae_xywh.y; y < ae_xywh.y + ae_xywh.h; y += y_skip) {
+    for (int x = ae_xywh.x; x < ae_xywh.x + ae_xywh.w; x += x_skip) {
+      uint8_t lum = pix_ptr[(y * b->out_img_width) + x];
+      lum_binning[lum]++;
+      lum_total += 1;
+    }
+  }
+
+  // Find mean lumimance value
+  unsigned int lum_cur = 0;
+  for (lum_med = 255; lum_med >= 0; lum_med--) {
+    lum_cur += lum_binning[lum_med];
+
+    if (lum_cur >= lum_total / 2) {
+      break;
+    }
+  }
+
+  return lum_med / 256.0;
+}
+
+int open_v4l_by_name_and_index(const char name[], int index, int flags) {
+  for (int v4l_index = 0; /**/; ++v4l_index) {
+    std::string v4l_name = util::read_file(util::string_format("/sys/class/video4linux/v4l-subdev%d/name", v4l_index));
+    if (v4l_name.empty()) return -1;
+    if (v4l_name.find(name) == 0) {
+      if (index == 0) {
+        return HANDLE_EINTR(open(util::string_format("/dev/v4l-subdev%d", v4l_index).c_str(), flags));
+      }
+      index--;
+    }
+  }
+}
@@ -0,0 +1,46 @@
+#pragma once
+
+#include <memory>
+
+#include "openpilot/cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionipc_server.h"
+#include "common/util.h"
+
+
+const int VIPC_BUFFER_COUNT = 18;
+
+typedef struct FrameMetadata {
+  uint32_t frame_id;
+  uint32_t request_id;
+  uint64_t timestamp_sof;
+  uint64_t timestamp_eof;
+  float processing_time;
+} FrameMetadata;
+
+class SpectraCamera;
+
+class CameraBuf {
+private:
+  int frame_buf_count;
+
+public:
+  VisionIpcServer *vipc_server;
+  VisionStreamType stream_type;
+
+  int cur_buf_idx;
+  FrameMetadata cur_frame_data;
+  VisionBuf *cur_yuv_buf;
+  VisionBuf *cur_camera_buf;
+  std::unique_ptr<VisionBuf[]> camera_bufs_raw;
+  uint32_t out_img_width, out_img_height;
+
+  CameraBuf() = default;
+  ~CameraBuf();
+  void init(SpectraCamera *cam, VisionIpcServer * v, int frame_cnt, VisionStreamType type);
+  void sendFrameToVipc();
+};
+
+void camerad_thread();
+kj::Array<uint8_t> get_raw_frame_image(const CameraBuf *b);
+float calculate_exposure_value(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip);
+int open_v4l_by_name_and_index(const char name[], int index = 0, int flags = O_RDWR | O_NONBLOCK);
@@ -0,0 +1,309 @@
+#include "system/camerad/cameras/camera_common.h"
+#include "system/camerad/cameras/spectra.h"
+
+#include <poll.h>
+#include <sys/ioctl.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cerrno>
+#include <cmath>
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include "common/params.h"
+#include "common/swaglog.h"
+
+
+ExitHandler do_exit;
+
+// for debugging
+const bool env_debug_frames = getenv("DEBUG_FRAMES") != nullptr;
+const bool env_log_raw_frames = getenv("LOG_RAW_FRAMES") != nullptr;
+const bool env_ctrl_exp_from_params = getenv("CTRL_EXP_FROM_PARAMS") != nullptr;
+
+
+class CameraState {
+public:
+  SpectraCamera camera;
+  int exposure_time = 5;
+  bool dc_gain_enabled = false;
+  int dc_gain_weight = 0;
+  int gain_idx = 0;
+  float analog_gain_frac = 0;
+
+  float cur_ev[3] = {};
+  float best_ev_score = 0;
+  int new_exp_g = 0;
+  int new_exp_t = 0;
+
+  Rect ae_xywh = {};
+  float measured_grey_fraction = 0;
+  float target_grey_fraction = 0.125;
+
+  float fl_pix = 0;
+  std::unique_ptr<PubMaster> pm;
+
+  CameraState(SpectraMaster *master, const CameraConfig &config) : camera(master, config) {};
+  ~CameraState();
+  void init(VisionIpcServer *v);
+  void update_exposure_score(float desired_ev, int exp_t, int exp_g_idx, float exp_gain);
+  void set_camera_exposure(float grey_frac);
+  void set_exposure_rect();
+  void sendState();
+
+  float get_gain_factor() const {
+    return (1 + dc_gain_weight * (camera.sensor->dc_gain_factor-1) / camera.sensor->dc_gain_max_weight);
+  }
+};
+
+void CameraState::init(VisionIpcServer *v) {
+  camera.camera_open(v);
+
+  if (!camera.enabled) return;
+
+  fl_pix = camera.cc.focal_len / camera.sensor->pixel_size_mm / camera.sensor->out_scale;
+  set_exposure_rect();
+
+  dc_gain_weight = camera.sensor->dc_gain_min_weight;
+  gain_idx = camera.sensor->analog_gain_rec_idx;
+  cur_ev[0] = cur_ev[1] = cur_ev[2] = get_gain_factor() * camera.sensor->sensor_analog_gains[gain_idx] * exposure_time;
+
+  pm = std::make_unique<PubMaster>(std::vector{camera.cc.publish_name});
+}
+
+CameraState::~CameraState() {}
+
+void CameraState::set_exposure_rect() {
+  // set areas for each camera, shouldn't be changed
+  std::vector<std::pair<Rect, float>> ae_targets = {
+    // (Rect, F)
+    std::make_pair((Rect){96, 400, 1734, 524}, 567.0),  // wide
+    std::make_pair((Rect){96, 160, 1734, 986}, 2648.0), // road
+    std::make_pair((Rect){96, 242, 1736, 906}, 567.0)   // driver
+  };
+  int h_ref = 1208;
+  /*
+    exposure target intrinsics is
+    [
+      [F, 0, 0.5*ae_xywh[2]]
+      [0, F, 0.5*H-ae_xywh[1]]
+      [0, 0, 1]
+    ]
+  */
+  auto ae_target = ae_targets[camera.cc.camera_num];
+  Rect xywh_ref = ae_target.first;
+  float fl_ref = ae_target.second;
+
+  ae_xywh = (Rect){
+    std::max(0, (int)camera.buf.out_img_width / 2 - (int)(fl_pix / fl_ref * xywh_ref.w / 2)),
+    std::max(0, (int)camera.buf.out_img_height / 2 - (int)(fl_pix / fl_ref * (h_ref / 2 - xywh_ref.y))),
+    std::min((int)(fl_pix / fl_ref * xywh_ref.w), (int)camera.buf.out_img_width / 2 + (int)(fl_pix / fl_ref * xywh_ref.w / 2)),
+    std::min((int)(fl_pix / fl_ref * xywh_ref.h), (int)camera.buf.out_img_height / 2 + (int)(fl_pix / fl_ref * (h_ref / 2 - xywh_ref.y)))
+  };
+}
+
+void CameraState::update_exposure_score(float desired_ev, int exp_t, int exp_g_idx, float exp_gain) {
+  float score = camera.sensor->getExposureScore(desired_ev, exp_t, exp_g_idx, exp_gain, gain_idx);
+  if (score < best_ev_score) {
+    new_exp_t = exp_t;
+    new_exp_g = exp_g_idx;
+    best_ev_score = score;
+  }
+}
+
+void CameraState::set_camera_exposure(float grey_frac) {
+  if (!camera.enabled) return;
+  std::vector<double> target_grey_minimums = {0.1, 0.1, 0.125}; // wide, road, driver
+
+  const float dt = 0.05;
+
+  const float ts_grey = 10.0;
+  const float ts_ev = 0.05;
+
+  const float k_grey = (dt / ts_grey) / (1.0 + dt / ts_grey);
+  const float k_ev = (dt / ts_ev) / (1.0 + dt / ts_ev);
+
+  // It takes 3 frames for the commanded exposure settings to take effect. The first frame is already started by the time
+  // we reach this function, the other 2 are due to the register buffering in the sensor.
+  // Therefore we use the target EV from 3 frames ago, the grey fraction that was just measured was the result of that control action.
+  // TODO: Lower latency to 2 frames, by using the histogram outputted by the sensor we can do AE before the debayering is complete
+
+  const auto &sensor = camera.sensor;
+  // Offset idx by one to not get stuck in self loop
+  const float cur_ev_ = cur_ev[(camera.buf.cur_frame_data.frame_id - 1) % 3] * sensor->ev_scale;
+
+  // Scale target grey between min and 0.4 depending on lighting conditions
+  float new_target_grey = std::clamp(0.4 - 0.3 * log2(1.0 + sensor->target_grey_factor*cur_ev_) / log2(6000.0), target_grey_minimums[camera.cc.camera_num], 0.4);
+  float target_grey = (1.0 - k_grey) * target_grey_fraction + k_grey * new_target_grey;
+
+  float desired_ev = std::clamp(cur_ev_ / sensor->ev_scale * target_grey / grey_frac, sensor->min_ev, sensor->max_ev);
+  float k = (1.0 - k_ev) / 3.0;
+  desired_ev = (k * cur_ev[0]) + (k * cur_ev[1]) + (k * cur_ev[2]) + (k_ev * desired_ev);
+
+  best_ev_score = 1e6;
+  new_exp_g = 0;
+  new_exp_t = 0;
+
+  // Hysteresis around high conversion gain
+  // We usually want this on since it results in lower noise, but turn off in very bright day scenes
+  bool enable_dc_gain = dc_gain_enabled;
+  if (!enable_dc_gain && target_grey < sensor->dc_gain_on_grey) {
+    enable_dc_gain = true;
+    dc_gain_weight = sensor->dc_gain_min_weight;
+  } else if (enable_dc_gain && target_grey > sensor->dc_gain_off_grey) {
+    enable_dc_gain = false;
+    dc_gain_weight = sensor->dc_gain_max_weight;
+  }
+
+  if (enable_dc_gain && dc_gain_weight < sensor->dc_gain_max_weight) {dc_gain_weight += 1;}
+  if (!enable_dc_gain && dc_gain_weight > sensor->dc_gain_min_weight) {dc_gain_weight -= 1;}
+
+  std::string gain_bytes, time_bytes;
+  if (env_ctrl_exp_from_params) {
+    static Params params;
+    gain_bytes = params.get("CameraDebugExpGain");
+    time_bytes = params.get("CameraDebugExpTime");
+  }
+
+  if (gain_bytes.size() > 0 && time_bytes.size() > 0) {
+    // Override gain and exposure time
+    gain_idx = std::stoi(gain_bytes);
+    exposure_time = std::stoi(time_bytes);
+
+    new_exp_g = gain_idx;
+    new_exp_t = exposure_time;
+    enable_dc_gain = false;
+  } else {
+    // Simple brute force optimizer to choose sensor parameters to reach desired EV
+    int min_g = std::max(gain_idx - 1, sensor->analog_gain_min_idx);
+    int max_g = std::min(gain_idx + 1, sensor->analog_gain_max_idx);
+    for (int g = min_g; g <= max_g; g++) {
+      float gain = sensor->sensor_analog_gains[g] * get_gain_factor();
+
+      // Compute optimal time for given gain
+      int t = std::clamp(int(std::round(desired_ev / gain)), sensor->exposure_time_min, sensor->exposure_time_max);
+
+      // Only go below recommended gain when absolutely necessary to not overexpose
+      if (g < sensor->analog_gain_rec_idx && t > 20 && g < gain_idx) {
+        continue;
+      }
+
+      update_exposure_score(desired_ev, t, g, gain);
+    }
+  }
+
+  measured_grey_fraction = grey_frac;
+  target_grey_fraction = target_grey;
+
+  analog_gain_frac = sensor->sensor_analog_gains[new_exp_g];
+  gain_idx = new_exp_g;
+  exposure_time = new_exp_t;
+  dc_gain_enabled = enable_dc_gain;
+
+  float gain = analog_gain_frac * get_gain_factor();
+  cur_ev[camera.buf.cur_frame_data.frame_id % 3] = exposure_time * gain;
+
+  // LOGE("ae - camera %d, cur_t %.5f, sof %.5f, dt %.5f", camera.cc.camera_num, 1e-9 * nanos_since_boot(), 1e-9 * camera.buf.cur_frame_data.timestamp_sof, 1e-9 * (nanos_since_boot() - camera.buf.cur_frame_data.timestamp_sof));
+
+  auto exp_reg_array = sensor->getExposureRegisters(exposure_time, new_exp_g, dc_gain_enabled);
+  camera.sensors_i2c(exp_reg_array.data(), exp_reg_array.size(), CAM_SENSOR_PACKET_OPCODE_SENSOR_CONFIG, camera.sensor->data_word);
+}
+
+void CameraState::sendState() {
+  camera.buf.sendFrameToVipc();
+
+  MessageBuilder msg;
+  auto framed = (msg.initEvent().*camera.cc.init_camera_state)();
+  const FrameMetadata &meta = camera.buf.cur_frame_data;
+  framed.setFrameId(meta.frame_id);
+  framed.setRequestId(meta.request_id);
+  framed.setTimestampEof(meta.timestamp_eof);
+  framed.setTimestampSof(meta.timestamp_sof);
+  framed.setIntegLines(exposure_time);
+  framed.setGain(analog_gain_frac * get_gain_factor());
+  framed.setHighConversionGain(dc_gain_enabled);
+  framed.setMeasuredGreyFraction(measured_grey_fraction);
+  framed.setTargetGreyFraction(target_grey_fraction);
+  framed.setProcessingTime(meta.processing_time);
+
+  const float ev = cur_ev[meta.frame_id % 3];
+  const float perc = util::map_val(ev, camera.sensor->min_ev, camera.sensor->max_ev, 0.0f, 100.0f);
+  framed.setExposureValPercent(perc);
+  framed.setSensor(camera.sensor->image_sensor);
+
+  // Log raw frames for road camera
+  if (env_log_raw_frames && camera.cc.stream_type == VISION_STREAM_NARROW_ROAD && meta.frame_id % 100 == 5) {  // no overlap with qlog decimation
+    framed.setImage(get_raw_frame_image(&camera.buf));
+  }
+
+  set_camera_exposure(calculate_exposure_value(&camera.buf, ae_xywh, 2, camera.cc.stream_type != VISION_STREAM_CABIN ? 2 : 4));
+
+  // Send the message
+  pm->send(camera.cc.publish_name, msg);
+}
+
+void camerad_thread() {
+  // TODO: centralize enabled handling
+
+  VisionIpcServer v("camerad");
+
+  // *** initial ISP init ***
+  SpectraMaster m;
+  m.init();
+
+  // *** per-cam init ***
+  std::vector<std::unique_ptr<CameraState>> cams;
+  for (const auto &config : ALL_CAMERA_CONFIGS) {
+    auto cam = std::make_unique<CameraState>(&m, config);
+    cam->init(&v);
+    cams.emplace_back(std::move(cam));
+  }
+
+  v.start_listener();
+
+  // start devices
+  LOG("-- Starting devices");
+  for (auto &cam : cams) cam->camera.sensors_start();
+
+  // poll events
+  LOG("-- Dequeueing Video events");
+  while (!do_exit) {
+    struct pollfd fds[1] = {{.fd = m.video0_fd, .events = POLLPRI}};
+    int ret = poll(fds, std::size(fds), 1000);
+    if (ret < 0) {
+      if (errno == EINTR || errno == EAGAIN) continue;
+      LOGE("poll failed (%d - %d)", ret, errno);
+      break;
+    }
+
+    if (!(fds[0].revents & POLLPRI)) continue;
+
+    struct v4l2_event ev = {0};
+    ret = HANDLE_EINTR(ioctl(fds[0].fd, VIDIOC_DQEVENT, &ev));
+    if (ret == 0) {
+      if (ev.type == V4L_EVENT_CAM_REQ_MGR_EVENT) {
+        struct cam_req_mgr_message *event_data = (struct cam_req_mgr_message *)ev.u.data;
+        if (env_debug_frames) {
+          printf("sess_hdl 0x%6X, link_hdl 0x%6X, frame_id %lu, req_id %lu, timestamp %.2f ms, sof_status %d\n", event_data->session_hdl, event_data->u.frame_msg.link_hdl,
+                 event_data->u.frame_msg.frame_id, event_data->u.frame_msg.request_id, event_data->u.frame_msg.timestamp/1e6, event_data->u.frame_msg.sof_status);
+          do_exit = do_exit || event_data->u.frame_msg.frame_id > (1*20);
+        }
+
+        for (auto &cam : cams) {
+          if (event_data->session_hdl == cam->camera.session_handle) {
+            if (cam->camera.handle_camera_event(event_data)) {
+              cam->sendState();
+            }
+            break;
+          }
+        }
+      } else {
+        LOGE("unhandled event %d\n", ev.type);
+      }
+    } else {
+      LOGE("VIDIOC_DQEVENT failed, errno=%d", errno);
+    }
+  }
+}
@@ -0,0 +1,47 @@
+#include "cdm.h"
+#include "stddef.h"
+
+int write_dmi(uint8_t *dst, uint64_t *addr, uint32_t length, uint32_t dmi_addr, uint8_t sel, uint8_t opcode) {
+  struct cdm_dmi_cmd *cmd = (struct cdm_dmi_cmd*)dst;
+  cmd->cmd = opcode;
+  cmd->length = length - 1;
+  cmd->reserved = 0;
+  cmd->addr = 0; // gets patched in
+  cmd->DMIAddr = dmi_addr;
+  cmd->DMISel = sel;
+
+  *addr = (uint64_t)(dst + offsetof(struct cdm_dmi_cmd, addr));
+  return sizeof(struct cdm_dmi_cmd);
+}
+
+int write_cont(uint8_t *dst, uint32_t reg, const std::vector<uint32_t> &vals) {
+  struct cdm_regcontinuous_cmd *cmd = (struct cdm_regcontinuous_cmd*)dst;
+  cmd->cmd = CAM_CDM_CMD_REG_CONT;
+  cmd->count = vals.size();
+  cmd->offset = reg;
+  cmd->reserved0 = 0;
+  cmd->reserved1 = 0;
+
+  uint32_t *vd = (uint32_t*)(dst + sizeof(struct cdm_regcontinuous_cmd));
+  for (int i = 0; i < vals.size(); i++) {
+    *vd = vals[i];
+    vd++;
+  }
+
+  return sizeof(struct cdm_regcontinuous_cmd) + vals.size()*sizeof(uint32_t);
+}
+
+int write_random(uint8_t *dst, const std::vector<uint32_t> &vals) {
+  struct cdm_regrandom_cmd *cmd = (struct cdm_regrandom_cmd*)dst;
+  cmd->cmd = CAM_CDM_CMD_REG_RANDOM;
+  cmd->count = vals.size() / 2;
+  cmd->reserved = 0;
+
+  uint32_t *vd = (uint32_t*)(dst + sizeof(struct cdm_regrandom_cmd));
+  for (int i = 0; i < vals.size(); i++) {
+    *vd = vals[i];
+    vd++;
+  }
+
+  return sizeof(struct cdm_regrandom_cmd) + vals.size()*sizeof(uint32_t);
+}
@@ -0,0 +1,79 @@
+#pragma once
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <vector>
+#include <memory>
+
+// from drivers/media/platform/msm/camera/cam_cdm/cam_cdm_util.{c,h}
+
+enum cam_cdm_command {
+  CAM_CDM_CMD_UNUSED = 0x0,
+  CAM_CDM_CMD_DMI = 0x1,
+  CAM_CDM_CMD_NOT_DEFINED = 0x2,
+  CAM_CDM_CMD_REG_CONT = 0x3,
+  CAM_CDM_CMD_REG_RANDOM = 0x4,
+  CAM_CDM_CMD_BUFF_INDIRECT = 0x5,
+  CAM_CDM_CMD_GEN_IRQ = 0x6,
+  CAM_CDM_CMD_WAIT_EVENT = 0x7,
+  CAM_CDM_CMD_CHANGE_BASE = 0x8,
+  CAM_CDM_CMD_PERF_CTRL = 0x9,
+  CAM_CDM_CMD_DMI_32 = 0xa,
+  CAM_CDM_CMD_DMI_64 = 0xb,
+  CAM_CDM_CMD_PRIVATE_BASE = 0xc,
+  CAM_CDM_CMD_SWD_DMI_32 = (CAM_CDM_CMD_PRIVATE_BASE + 0x64),
+  CAM_CDM_CMD_SWD_DMI_64 = (CAM_CDM_CMD_PRIVATE_BASE + 0x65),
+  CAM_CDM_CMD_PRIVATE_BASE_MAX = 0x7F
+};
+
+// our helpers
+int write_random(uint8_t *dst, const std::vector<uint32_t> &vals);
+int write_cont(uint8_t *dst, uint32_t reg, const std::vector<uint32_t> &vals);
+int write_dmi(uint8_t *dst, uint64_t *addr, uint32_t length, uint32_t dmi_addr, uint8_t sel, uint8_t opcode = CAM_CDM_CMD_DMI_32);
+
+/**
+ * struct cdm_regrandom_cmd - Definition for CDM random register command.
+ * @count: Number of register writes
+ * @reserved: reserved bits
+ * @cmd: Command ID (CDMCmd)
+ */
+struct cdm_regrandom_cmd {
+  unsigned int count    : 16;
+  unsigned int reserved : 8;
+  unsigned int cmd      : 8;
+} __attribute__((__packed__));
+
+/**
+ * struct cdm_regcontinuous_cmd - Definition for a CDM register range command.
+ * @count: Number of register writes
+ * @reserved0: reserved bits
+ * @cmd: Command ID (CDMCmd)
+ * @offset: Start address of the range of registers
+ * @reserved1: reserved bits
+ */
+struct cdm_regcontinuous_cmd {
+  unsigned int count     : 16;
+  unsigned int reserved0 : 8;
+  unsigned int cmd       : 8;
+  unsigned int offset    : 24;
+  unsigned int reserved1 : 8;
+} __attribute__((__packed__));
+
+/**
+ * struct cdm_dmi_cmd - Definition for a CDM DMI command.
+ * @length: Number of bytes in LUT - 1
+ * @reserved: reserved bits
+ * @cmd: Command ID (CDMCmd)
+ * @addr: Address of the LUT in memory
+ * @DMIAddr: Address of the target DMI config register
+ * @DMISel: DMI identifier
+ */
+struct cdm_dmi_cmd {
+  unsigned int length   : 16;
+  unsigned int reserved : 8;
+  unsigned int cmd      : 8;
+  unsigned int addr;
+  unsigned int DMIAddr  : 24;
+  unsigned int DMISel   : 8;
+} __attribute__((__packed__));
@@ -0,0 +1,73 @@
+#pragma once
+
+#include "common/util.h"
+#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "openpilot/cereal/visionstream.h"
+#include "msgq/visionipc/visionipc_server.h"
+
+#include "media/cam_isp_ife.h"
+
+
+typedef enum {
+  ISP_RAW_OUTPUT,   // raw frame from sensor
+  ISP_IFE_PROCESSED,  // fully processed image through the IFE
+  ISP_BPS_PROCESSED,  // fully processed image through the BPS
+} SpectraOutputType;
+
+// For the comma 3X three camera platform
+
+struct CameraConfig {
+  int camera_num;
+  VisionStreamType stream_type;
+  float focal_len;  // millimeters
+  const char *publish_name;
+  cereal::FrameData::Builder (cereal::Event::Builder::*init_camera_state)();
+  bool enabled;
+  uint32_t phy;
+  bool vignetting_correction;
+  SpectraOutputType output_type;
+  bool staggered_sof;  // SOF is staggered (half-period offset) from other cameras
+};
+
+// NOTE: to be able to disable road and wide road, we still have to configure the sensor over i2c
+// If you don't do this, the strobe GPIO is an output (even in reset it seems!)
+const CameraConfig WIDE_ROAD_CAMERA_CONFIG = {
+  .camera_num = 0,
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .focal_len = 1.71,
+  .publish_name = "wideRoadCameraState",
+  .init_camera_state = &cereal::Event::Builder::initWideRoadCameraState,
+  .enabled = !getenv("DISABLE_WIDE_ROAD"),
+  .phy = CAM_ISP_IFE_IN_RES_PHY_0,
+  .vignetting_correction = false,
+  .output_type = ISP_IFE_PROCESSED,
+  .staggered_sof = false,
+};
+
+const CameraConfig NARROW_ROAD_CAMERA_CONFIG = {
+  .camera_num = 1,
+  .stream_type = VISION_STREAM_NARROW_ROAD,
+  .focal_len = 8.0,
+  .publish_name = "narrowRoadCameraState",
+  .init_camera_state = &cereal::Event::Builder::initNarrowRoadCameraState,
+  .enabled = !getenv("DISABLE_ROAD"),
+  .phy = CAM_ISP_IFE_IN_RES_PHY_1,
+  .vignetting_correction = true,
+  .output_type = ISP_IFE_PROCESSED,
+  .staggered_sof = false,
+};
+
+const CameraConfig CABIN_CAMERA_CONFIG = {
+  .camera_num = 2,
+  .stream_type = VISION_STREAM_CABIN,
+  .focal_len = 1.71,
+  .publish_name = "cabinCameraState",
+  .init_camera_state = &cereal::Event::Builder::initCabinCameraState,
+  .enabled = !getenv("DISABLE_DRIVER"),
+  .phy = CAM_ISP_IFE_IN_RES_PHY_2,
+  .vignetting_correction = false,
+  .output_type = ISP_BPS_PROCESSED,
+  .staggered_sof = true,
+};
+
+const CameraConfig ALL_CAMERA_CONFIGS[] = {WIDE_ROAD_CAMERA_CONFIG, NARROW_ROAD_CAMERA_CONFIG, CABIN_CAMERA_CONFIG};
@@ -0,0 +1,236 @@
+#pragma once
+
+#include "cdm.h"
+
+#include "system/camerad/cameras/hw.h"
+#include "system/camerad/sensors/sensor.h"
+
+int build_common_ife_bps(uint8_t *dst, const CameraConfig cam, const SensorInfo *s, std::vector<uint32_t> &patches, bool ife) {
+  uint8_t *start = dst;
+
+  /*
+    Common between IFE and BPS.
+  */
+
+  // IFE -> BPS addresses
+  /*
+  std::map<uint32_t, uint32_t> addrs = {
+    {0xf30, 0x3468},
+  };
+  */
+
+  // YUV
+  dst += write_cont(dst, ife ? 0xf30 : 0x3468, {
+    0x00680208,
+    0x00000108,
+    0x00400000,
+    0x03ff0000,
+    0x01c01ed8,
+    0x00001f68,
+    0x02000000,
+    0x03ff0000,
+    0x1fb81e88,
+    0x000001c0,
+    0x02000000,
+    0x03ff0000,
+  });
+
+  return dst - start;
+}
+
+int build_update(uint8_t *dst, const CameraConfig cam, const SensorInfo *s, std::vector<uint32_t> &patches) {
+  uint8_t *start = dst;
+
+  // init sequence
+  dst += write_random(dst, {
+    0x2c, 0xffffffff,
+    0x30, 0xffffffff,
+    0x34, 0xffffffff,
+    0x38, 0xffffffff,
+    0x3c, 0xffffffff,
+  });
+
+  // demux cfg
+  dst += write_cont(dst, 0x560, {
+    0x00000001,
+    0x04440444,
+    0x04450445,
+    0x04440444,
+    0x04450445,
+    0x000000ca,
+    0x0000009c,
+  });
+
+  // white balance
+  dst += write_cont(dst, 0x6fc, {
+    0x00800080,
+    0x00000080,
+    0x00000000,
+    0x00000000,
+  });
+
+  // module config/enables (e.g. enable debayer, white balance, etc.)
+  dst += write_cont(dst, 0x40, {
+    0x00000c06 | ((uint32_t)(cam.vignetting_correction) << 8),
+  });
+  dst += write_cont(dst, 0x44, {
+    0x00000000,
+  });
+  dst += write_cont(dst, 0x48, {
+    (1 << 3) | (1 << 1),
+  });
+  dst += write_cont(dst, 0x4c, {
+    0x00000019,
+  });
+  dst += write_cont(dst, 0xf00, {
+    0x00000000,
+  });
+
+  // cropping
+  dst += write_cont(dst, 0xe0c, {
+    0x00000e00,
+  });
+  dst += write_cont(dst, 0xe2c, {
+    0x00000e00,
+  });
+
+  // black level scale + offset
+  dst += write_cont(dst, 0x6b0, {
+    ((uint32_t)(1 << 11) << 0xf) | (s->black_level << (14 - s->bits_per_pixel)),
+    0x0,
+    0x0,
+  });
+
+  return dst - start;
+}
+
+
+int build_initial_config(uint8_t *dst, const CameraConfig cam, const SensorInfo *s, std::vector<uint32_t> &patches, uint32_t out_width, uint32_t out_height) {
+  uint8_t *start = dst;
+
+  // start with the every frame config
+  dst += build_update(dst, cam, s, patches);
+
+  uint64_t addr;
+
+  // setup
+  dst += write_cont(dst, 0x478, {
+    0x00000004,
+    0x004000c0,
+  });
+  dst += write_cont(dst, 0x488, {
+    0x00000000,
+    0x00000000,
+    0x00000f0f,
+  });
+  dst += write_cont(dst, 0x49c, {
+    0x00000001,
+  });
+  dst += write_cont(dst, 0xce4, {
+    0x00000000,
+    0x00000000,
+  });
+
+  // linearization
+  dst += write_cont(dst, 0x4dc, {
+    0x00000000,
+  });
+  dst += write_cont(dst, 0x4e0, s->linearization_pts);
+  dst += write_cont(dst, 0x4f0, s->linearization_pts);
+  dst += write_cont(dst, 0x500, s->linearization_pts);
+  dst += write_cont(dst, 0x510, s->linearization_pts);
+  // TODO: this is DMI64 in the dump, does that matter?
+  dst += write_dmi(dst, &addr, s->linearization_lut.size()*sizeof(uint32_t), 0xc24, 9);
+  patches.push_back(addr - (uint64_t)start);
+
+  // vignetting correction
+  dst += write_cont(dst, 0x6bc, {
+    0x0b3c0000,
+    0x00670067,
+    0xd3b1300c,
+    0x13b1300c,
+  });
+  dst += write_cont(dst, 0x6d8, {
+    0xec4e4000,
+    0x0100c003,
+  });
+  dst += write_dmi(dst, &addr, s->vignetting_lut.size()*sizeof(uint32_t), 0xc24, 14); // GRR
+  patches.push_back(addr - (uint64_t)start);
+  dst += write_dmi(dst, &addr, s->vignetting_lut.size()*sizeof(uint32_t), 0xc24, 15); // GBB
+  patches.push_back(addr - (uint64_t)start);
+
+  // debayer
+  dst += write_cont(dst, 0x6f8, {
+    0x00000100,
+  });
+  dst += write_cont(dst, 0x71c, {
+    0x00008000,
+    0x08000066,
+  });
+
+  // color correction
+  dst += write_cont(dst, 0x760, s->color_correct_matrix);
+
+  // gamma
+  dst += write_cont(dst, 0x798, {
+    0x00000000,
+  });
+  dst += write_dmi(dst, &addr, s->gamma_lut_rgb.size()*sizeof(uint32_t), 0xc24, 26);  // G
+  patches.push_back(addr - (uint64_t)start);
+  dst += write_dmi(dst, &addr, s->gamma_lut_rgb.size()*sizeof(uint32_t), 0xc24, 28);  // B
+  patches.push_back(addr - (uint64_t)start);
+  dst += write_dmi(dst, &addr, s->gamma_lut_rgb.size()*sizeof(uint32_t), 0xc24, 30);  // R
+  patches.push_back(addr - (uint64_t)start);
+
+  // output size/scaling
+  dst += write_cont(dst, 0xa3c, {
+    0x00000003,
+    ((out_width - 1) << 16) | (s->frame_width - 1),
+    0x30036666,
+    0x00000000,
+    0x00000000,
+    s->frame_width - 1,
+    ((out_height - 1) << 16) | (s->frame_height - 1),
+    0x30036666,
+    0x00000000,
+    0x00000000,
+    s->frame_height - 1,
+  });
+  dst += write_cont(dst, 0xa68, {
+    0x00000003,
+    ((out_width / 2 - 1) << 16) | (s->frame_width - 1),
+    0x3006cccc,
+    0x00000000,
+    0x00000000,
+    s->frame_width - 1,
+    ((out_height / 2 - 1) << 16) | (s->frame_height - 1),
+    0x3006cccc,
+    0x00000000,
+    0x00000000,
+    s->frame_height - 1,
+  });
+
+  // cropping
+  dst += write_cont(dst, 0xe10, {
+    out_height - 1,
+    out_width - 1,
+  });
+  dst += write_cont(dst, 0xe30, {
+    out_height / 2 - 1,
+    out_width - 1,
+  });
+  dst += write_cont(dst, 0xe18, {
+    0x0ff00000,
+    0x00000016,
+  });
+  dst += write_cont(dst, 0xe38, {
+    0x0ff00000,
+    0x00000017,
+  });
+
+  dst += build_common_ife_bps(dst, cam, s, patches, true);
+
+  return dst - start;
+}
+
+
@@ -0,0 +1,130 @@
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <tuple>
+
+// NV12 subset copied from media/msm_media_info.h.
+#ifndef MSM_MEDIA_ALIGN
+#define MSM_MEDIA_ALIGN(__sz, __align) (((__align) & ((__align) - 1)) ? \
+  ((((__sz) + (__align) - 1) / (__align)) * (__align)) : \
+  (((__sz) + (__align) - 1) & (~((__align) - 1))))
+#endif
+
+#ifndef MSM_MEDIA_MAX
+#define MSM_MEDIA_MAX(__a, __b) ((__a) > (__b) ? (__a) : (__b))
+#endif
+
+enum color_fmts {
+  COLOR_FMT_NV12,
+};
+
+static inline unsigned int VENUS_EXTRADATA_SIZE(int width, int height) {
+  (void)height;
+  (void)width;
+  return 16 * 1024;
+}
+
+static inline unsigned int VENUS_Y_STRIDE(int color_fmt, int width) {
+  unsigned int stride = 0;
+  if (!width) goto invalid_input;
+
+  switch (color_fmt) {
+    case COLOR_FMT_NV12:
+      stride = MSM_MEDIA_ALIGN(width, 128);
+      break;
+    default:
+      break;
+  }
+invalid_input:
+  return stride;
+}
+
+static inline unsigned int VENUS_UV_STRIDE(int color_fmt, int width) {
+  unsigned int stride = 0;
+  if (!width) goto invalid_input;
+
+  switch (color_fmt) {
+    case COLOR_FMT_NV12:
+      stride = MSM_MEDIA_ALIGN(width, 128);
+      break;
+    default:
+      break;
+  }
+invalid_input:
+  return stride;
+}
+
+static inline unsigned int VENUS_Y_SCANLINES(int color_fmt, int height) {
+  unsigned int sclines = 0;
+  if (!height) goto invalid_input;
+
+  switch (color_fmt) {
+    case COLOR_FMT_NV12:
+      sclines = MSM_MEDIA_ALIGN(height, 32);
+      break;
+    default:
+      break;
+  }
+invalid_input:
+  return sclines;
+}
+
+static inline unsigned int VENUS_UV_SCANLINES(int color_fmt, int height) {
+  unsigned int sclines = 0;
+  if (!height) goto invalid_input;
+
+  switch (color_fmt) {
+    case COLOR_FMT_NV12:
+      sclines = MSM_MEDIA_ALIGN((height + 1) >> 1, 16);
+      break;
+    default:
+      break;
+  }
+invalid_input:
+  return sclines;
+}
+
+static inline unsigned int VENUS_BUFFER_SIZE(int color_fmt, int width, int height) {
+  const unsigned int extra_size = VENUS_EXTRADATA_SIZE(width, height);
+  unsigned int size = 0;
+  unsigned int y_stride = 0, uv_stride = 0, y_sclines = 0, uv_sclines = 0;
+
+  if (!width || !height) goto invalid_input;
+
+  y_stride = VENUS_Y_STRIDE(color_fmt, width);
+  uv_stride = VENUS_UV_STRIDE(color_fmt, width);
+  y_sclines = VENUS_Y_SCANLINES(color_fmt, height);
+  uv_sclines = VENUS_UV_SCANLINES(color_fmt, height);
+
+  switch (color_fmt) {
+    case COLOR_FMT_NV12: {
+      const unsigned int y_plane = y_stride * y_sclines;
+      const unsigned int uv_plane = uv_stride * uv_sclines + 4096;
+      size = y_plane + uv_plane + MSM_MEDIA_MAX(extra_size, 8 * y_stride);
+      size = MSM_MEDIA_ALIGN(size, 4096);
+      size += MSM_MEDIA_ALIGN(width, 512) * 512;
+      size = MSM_MEDIA_ALIGN(size, 4096);
+      break;
+    }
+    default:
+      break;
+  }
+invalid_input:
+  return size;
+}
+
+// Returns NV12 aligned (stride, y_height, uv_height, buffer_size) for the given frame dimensions.
+inline std::tuple<uint32_t, uint32_t, uint32_t, uint32_t> get_nv12_info(int width, int height) {
+  const uint32_t stride = VENUS_Y_STRIDE(COLOR_FMT_NV12, width);
+  const uint32_t y_height = VENUS_Y_SCANLINES(COLOR_FMT_NV12, height);
+  const uint32_t uv_height = VENUS_UV_SCANLINES(COLOR_FMT_NV12, height);
+  const uint32_t size = VENUS_BUFFER_SIZE(COLOR_FMT_NV12, width, height);
+
+  // Sanity checks for NV12 format assumptions
+  assert(stride == VENUS_UV_STRIDE(COLOR_FMT_NV12, width));
+  assert(y_height / 2 == uv_height);
+  assert((stride * y_height) % 0x1000 == 0);  // uv_offset must be page-aligned
+
+  return {stride, y_height, uv_height, size};
+}
@@ -0,0 +1,21 @@
+# Python version of openpilot/system/camerad/cameras/nv12_info.h
+# Calculations from media/msm_media_info.h (VENUS_BUFFER_SIZE)
+
+def align(val: int, alignment: int) -> int:
+  return ((val + alignment - 1) // alignment) * alignment
+
+def get_nv12_info(width: int, height: int) -> tuple[int, int, int, int]:
+  """Returns (stride, y_height, uv_height, buffer_size) for NV12 frame dimensions."""
+  stride = align(width, 128)
+  y_height = align(height, 32)
+  uv_height = align(height // 2, 16)
+
+  # VENUS_BUFFER_SIZE for NV12
+  y_plane = stride * y_height
+  uv_plane = stride * uv_height + 4096
+  size = y_plane + uv_plane + max(16 * 1024, 8 * stride)
+  size = align(size, 4096)
+  size += align(width, 512) * 512  # kernel padding for non-aligned frames
+  size = align(size, 4096)
+
+  return stride, y_height, uv_height, size
@@ -0,0 +1,245 @@
+#pragma once
+
+#include <sys/mman.h>
+#include <functional>
+#include <memory>
+#include <queue>
+#include <optional>
+#include <utility>
+
+#include "media/cam_req_mgr.h"
+
+#include "common/util.h"
+#include "common/swaglog.h"
+#include "system/camerad/cameras/hw.h"
+#include "system/camerad/cameras/camera_common.h"
+#include "system/camerad/sensors/sensor.h"
+
+#define MAX_IFE_BUFS 20
+
+const int MIPI_SETTLE_CNT = 33;  // Calculated by camera_freqs.py
+
+// For use with the Titan 170 ISP in the SDM845
+// https://github.com/commaai/agnos-kernel-sdm845
+
+// CSLDeviceType/CSLPacketOpcodesIFE from camx
+// cam_packet_header.op_code = (device << 24) | (opcode);
+#define CSLDeviceTypeImageSensor (0x01 << 24)
+#define CSLDeviceTypeIFE         (0x0F << 24)
+#define CSLDeviceTypeBPS         (0x10 << 24)
+#define OpcodesIFEInitialConfig  0x0
+#define OpcodesIFEUpdate         0x1
+
+// Sensor command values from the SDM845 kernel's private camera header:
+// drivers/media/platform/msm/camera/cam_sensor_module/cam_sensor_utils/cam_sensor_cmn_header.h
+// These are userspace-visible ioctl payload values, but Qualcomm did not export them through UAPI.
+enum {
+  CAMERA_SENSOR_CMD_TYPE_PROBE = 1,
+  CAMERA_SENSOR_CMD_TYPE_PWR_UP = 2,
+  CAMERA_SENSOR_CMD_TYPE_PWR_DOWN = 3,
+  CAMERA_SENSOR_CMD_TYPE_I2C_INFO = 4,
+  CAMERA_SENSOR_CMD_TYPE_I2C_RNDM_WR = 5,
+  CAMERA_SENSOR_CMD_TYPE_WAIT = 9,
+
+  CAMERA_SENSOR_WAIT_OP_SW_UCND = 3,
+
+  CAMERA_SENSOR_I2C_TYPE_BYTE = 1,
+  CAMERA_SENSOR_I2C_TYPE_WORD = 2,
+
+  I2C_FAST_MODE = 1,
+
+  CAM_SENSOR_PACKET_OPCODE_SENSOR_PROBE = 3,
+  CAM_SENSOR_PACKET_OPCODE_SENSOR_CONFIG = 4,
+  CAM_SENSOR_PACKET_OPCODE_SENSOR_NOP = 127,
+};
+
+std::optional<int32_t> device_acquire(int fd, int32_t session_handle, void *data, uint32_t num_resources=1);
+int device_config(int fd, int32_t session_handle, int32_t dev_handle, uint64_t packet_handle);
+int device_control(int fd, int op_code, int session_handle, int dev_handle);
+int do_cam_control(int fd, int op_code, void *handle, int size);
+void *alloc_w_mmu_hdl(int video0_fd, int len, uint32_t *handle, int align = 8, int flags = CAM_MEM_FLAG_KMD_ACCESS | CAM_MEM_FLAG_UMD_ACCESS | CAM_MEM_FLAG_CMD_BUF_TYPE,
+                      int mmu_hdl = 0, int mmu_hdl2 = 0);
+void release(int video0_fd, uint32_t handle);
+
+class MemoryManager {
+public:
+  void init(int _video0_fd) { video0_fd = _video0_fd; }
+  ~MemoryManager();
+
+  template <class T>
+  auto alloc(int len, uint32_t *handle) {
+    return std::unique_ptr<T, std::function<void(void *)>>((T*)alloc_buf(len, handle), [this](void *ptr) { this->free(ptr); });
+  }
+
+private:
+  void *alloc_buf(int len, uint32_t *handle);
+  void free(void *ptr);
+
+  std::map<void *, uint32_t> handle_lookup;
+  std::map<void *, int> size_lookup;
+  std::map<int, std::queue<void *> > cached_allocations;
+  int video0_fd;
+};
+
+class SpectraMaster {
+public:
+  void init();
+
+  unique_fd video0_fd;
+  unique_fd cam_sync_fd;
+  unique_fd isp_fd;
+  unique_fd icp_fd;
+  int device_iommu = -1;
+  int cdm_iommu = -1;
+  int icp_device_iommu = -1;
+  MemoryManager mem_mgr;
+};
+
+class SpectraBuf {
+public:
+  SpectraBuf() = default;
+
+  ~SpectraBuf() {
+    if (video_fd >= 0 && ptr) {
+      munmap(ptr, mmap_size);
+      release(video_fd, handle);
+    }
+  }
+
+  void init(SpectraMaster *m, int s, int a, bool shared_access, int mmu_hdl = 0, int mmu_hdl2 = 0, int count = 1) {
+    video_fd = m->video0_fd;
+    size = s;
+    alignment = a;
+    mmap_size = aligned_size() * count;
+
+    uint32_t flags = CAM_MEM_FLAG_HW_READ_WRITE | CAM_MEM_FLAG_KMD_ACCESS | CAM_MEM_FLAG_UMD_ACCESS | CAM_MEM_FLAG_CMD_BUF_TYPE;
+    if (shared_access) {
+      flags |= CAM_MEM_FLAG_HW_SHARED_ACCESS;
+    }
+
+    void *p = alloc_w_mmu_hdl(video_fd, mmap_size, (uint32_t*)&handle, alignment, flags, mmu_hdl, mmu_hdl2);
+    ptr = (unsigned char*)p;
+    assert(ptr != NULL);
+  };
+
+  uint32_t aligned_size() {
+    return ALIGNED_SIZE(size, alignment);
+  };
+
+  int video_fd = -1;
+  unsigned char *ptr = nullptr;
+  int size = 0, alignment = 0, handle = 0, mmap_size = 0;
+};
+
+class SpectraCamera {
+public:
+  SpectraCamera(SpectraMaster *master, const CameraConfig &config);
+  ~SpectraCamera();
+
+  void camera_open(VisionIpcServer *v);
+  bool handle_camera_event(const cam_req_mgr_message *event_data);
+  void camera_close();
+  void camera_map_bufs();
+  void config_bps(int idx, int request_id);
+  void config_ife(int idx, int request_id, bool init=false);
+
+  int clear_req_queue();
+  void enqueue_frame(uint64_t request_id);
+
+  int sensors_init();
+  void sensors_start();
+  void sensors_poke(int request_id);
+  void sensors_i2c(const struct i2c_random_wr_payload* dat, int len, int op_code, bool data_word);
+
+  bool openSensor();
+  void configISP();
+  void configICP();
+  void configCSIPHY();
+  void linkDevices();
+  void destroySyncObjectAt(int index);
+
+  // *** state ***
+
+  int ife_buf_depth = -1;
+  bool open = false;
+  bool enabled = true;
+  CameraConfig cc;
+  std::unique_ptr<const SensorInfo> sensor;
+
+  // YUV image size
+  uint32_t stride;
+  uint32_t y_height;
+  uint32_t uv_height;
+  uint32_t uv_offset;
+  uint32_t yuv_size;
+
+  unique_fd sensor_fd;
+  unique_fd csiphy_fd;
+
+  int32_t session_handle = -1;
+  int32_t sensor_dev_handle = -1;
+  int32_t isp_dev_handle = -1;
+  int32_t icp_dev_handle = -1;
+  int32_t csiphy_dev_handle = -1;
+
+  int32_t link_handle = -1;
+
+  SpectraBuf ife_cmd;
+  SpectraBuf ife_gamma_lut;
+  SpectraBuf ife_linearization_lut;
+  SpectraBuf ife_vignetting_lut;
+
+  SpectraBuf bps_cmd;
+  SpectraBuf bps_cdm_buffer;
+  SpectraBuf bps_cdm_program_array;
+  SpectraBuf bps_cdm_striping_bl;
+  SpectraBuf bps_iq;
+  SpectraBuf bps_striping;
+  SpectraBuf bps_linearization_lut;
+  SpectraBuf bps_gamma_lut;
+  SpectraBuf bps_fullres_dummy;
+  std::vector<uint32_t> bps_lin_reg;
+  std::vector<uint32_t> bps_ccm_reg;
+
+  int buf_handle_yuv[MAX_IFE_BUFS] = {};
+  int buf_handle_raw[MAX_IFE_BUFS] = {};
+  int sync_objs_ife[MAX_IFE_BUFS] = {};
+  int sync_objs_bps[MAX_IFE_BUFS] = {};
+  uint64_t last_valid_request_id = 0;
+  uint64_t last_requeue_ts = 0;
+  uint64_t last_valid_ife_frame_id = 0;
+  int invalid_request_count = 0;
+  bool skip_expected = true;
+
+  CameraBuf buf;
+  SpectraMaster *m;
+
+private:
+  void clearAndRequeue(uint64_t from_request_id);
+  bool validateEvent(uint64_t request_id, uint64_t ife_frame_id);
+  bool waitForFrameReady(uint64_t request_id);
+  bool processFrame(int buf_idx, uint64_t request_id, uint64_t ife_frame_id, uint64_t timestamp);
+  static bool syncFirstFrame(int camera_id, uint64_t request_id, uint64_t ife_frame_id, uint64_t timestamp, bool staggered);
+  struct SyncData {
+    uint64_t timestamp;
+    uint64_t frame_id_offset = 0;
+    bool staggered = false;
+  };
+  inline static std::map<int, SyncData> camera_sync_data;
+  inline static bool first_frame_synced = false;
+
+  // a mode for stressing edge cases: realignment, sync failures, etc.
+  inline bool stress_test(std::string log) {
+    static double last_trigger = 0;
+    static double prob = std::stod(util::getenv("SPECTRA_ERROR_PROB", "-1"));
+    static double dt = std::stod(util::getenv("SPECTRA_ERROR_DT", "1"));
+    bool triggered = (prob > 0) && \
+                     ((static_cast<double>(rand()) / RAND_MAX) < prob) && \
+                     (millis_since_boot() - last_trigger) > dt;
+    if (triggered) {
+      last_trigger = millis_since_boot();
+      LOGE("stress test (cam %d): %s", cc.camera_num, log.c_str());
+    }
+    return triggered;
+  }
+};
@@ -0,0 +1,15 @@
+#include "system/camerad/cameras/camera_common.h"
+
+#include <cassert>
+
+#include "common/params.h"
+#include "common/util.h"
+
+int main(int argc, char *argv[]) {
+  // doesn't need RT priority since we're using isolcpus
+  int ret = util::set_core_affinity({6});
+  assert(ret == 0 || Params().getBool("IsOffroad")); // failure ok while offroad due to offlining cores
+
+  camerad_thread();
+  return 0;
+}
@@ -0,0 +1,135 @@
+#include <cmath>
+
+#include "system/camerad/sensors/sensor.h"
+#include <media/msm_camsensor_sdk.h>
+
+namespace {
+
+const float sensor_analog_gains_OS04C10[] = {
+    1.0, 1.0625, 1.125, 1.1875, 1.25, 1.3125, 1.375, 1.4375, 1.5, 1.5625, 1.6875,
+    1.8125, 1.9375, 2.0, 2.125, 2.25, 2.375, 2.5, 2.625, 2.75, 2.875, 3.0,
+    3.125, 3.375, 3.625, 3.875, 4.0, 4.25, 4.5, 4.75, 5.0, 5.25, 5.5,
+    5.75, 6.0, 6.25, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10.0,
+    10.5, 11.0, 11.5, 12.0, 12.5, 13.0, 13.5, 14.0, 14.5, 15.0, 15.5};
+
+const uint32_t os04c10_analog_gains_reg[] = {
+    0x080, 0x088, 0x090, 0x098, 0x0A0, 0x0A8, 0x0B0, 0x0B8, 0x0C0, 0x0C8, 0x0D8,
+    0x0E8, 0x0F8, 0x100, 0x110, 0x120, 0x130, 0x140, 0x150, 0x160, 0x170, 0x180,
+    0x190, 0x1B0, 0x1D0, 0x1F0, 0x200, 0x220, 0x240, 0x260, 0x280, 0x2A0, 0x2C0,
+    0x2E0, 0x300, 0x320, 0x340, 0x380, 0x3C0, 0x400, 0x440, 0x480, 0x4C0, 0x500,
+    0x540, 0x580, 0x5C0, 0x600, 0x640, 0x680, 0x6C0, 0x700, 0x740, 0x780, 0x7C0};
+
+}  // namespace
+
+OS04C10::OS04C10() {
+  image_sensor = cereal::FrameData::ImageSensor::OS04C10;
+  bayer_pattern = CAM_ISP_PATTERN_BAYER_BGBGBG;
+  pixel_size_mm = 0.002;
+  data_word = false;
+
+  out_scale = 2;
+  frame_width = 2688;
+  frame_height = 1520;
+  frame_stride = frame_width * 12 / 8;
+
+  extra_height = 0;
+  frame_offset = 0;
+
+  start_reg_array.assign(std::begin(start_reg_array_os04c10), std::end(start_reg_array_os04c10));
+  init_reg_array.assign(std::begin(init_array_os04c10), std::end(init_array_os04c10));
+  probe_reg_addr = 0x300a;
+  probe_expected_data = 0x5304;
+  bits_per_pixel = 12;
+  mipi_format = CAM_FORMAT_MIPI_RAW_12;
+  frame_data_type = CSI_RAW12;
+  mclk_frequency = 24000000; // Hz
+
+  // TODO: this was set from logs. actually calculate it out
+  readout_time_ns = 11000000;
+
+  ev_scale = 150.0;
+  dc_gain_factor = 1;
+  dc_gain_min_weight = 1;  // always on is fine
+  dc_gain_max_weight = 1;
+  dc_gain_on_grey = 0.9;
+  dc_gain_off_grey = 1.0;
+  exposure_time_min = 2;
+  exposure_time_max = 2352;
+  analog_gain_min_idx = 0x0;
+  analog_gain_rec_idx = 0x0;  // 1x
+  analog_gain_max_idx = 0x28;
+  analog_gain_cost_delta = -1;
+  analog_gain_cost_low = 0.4;
+  analog_gain_cost_high = 6.4;
+  for (int i = 0; i <= analog_gain_max_idx; i++) {
+    sensor_analog_gains[i] = sensor_analog_gains_OS04C10[i];
+  }
+  min_ev = exposure_time_min * sensor_analog_gains[analog_gain_min_idx];
+  max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
+  target_grey_factor = 0.01;
+
+  black_level = 48;
+  color_correct_matrix = {
+    0x000000c2, 0x00000fe0, 0x00000fde,
+    0x00000fa7, 0x000000d9, 0x00001000,
+    0x00000fca, 0x00000fef, 0x000000c7,
+  };
+  for (int i = 0; i < 65; i++) {
+    float fx = i / 64.0;
+    gamma_lut_rgb.push_back((uint32_t)((10*fx)/(1+9*fx)*1023.0 + 0.5));
+  }
+  prepare_gamma_lut();
+  linearization_lut = {
+    0x02000000, 0x02000000, 0x02000000, 0x02000000,
+    0x020007ff, 0x020007ff, 0x020007ff, 0x020007ff,
+    0x02000bff, 0x02000bff, 0x02000bff, 0x02000bff,
+    0x020017ff, 0x020017ff, 0x020017ff, 0x020017ff,
+    0x02001bff, 0x02001bff, 0x02001bff, 0x02001bff,
+    0x020023ff, 0x020023ff, 0x020023ff, 0x020023ff,
+    0x00003fff, 0x00003fff, 0x00003fff, 0x00003fff,
+    0x00003fff, 0x00003fff, 0x00003fff, 0x00003fff,
+    0x00003fff, 0x00003fff, 0x00003fff, 0x00003fff,
+  };
+  linearization_pts = {0x07ff0bff, 0x17ff1bff, 0x23ff3fff, 0x3fff3fff};
+  vignetting_lut = {
+    0x01064832, 0x00da26d1, 0x00bb25d9, 0x00aac556, 0x00a06503, 0x009a64d3, 0x009744ba, 0x009744ba, 0x009a24d1, 0x00a00500, 0x00aa2551, 0x00ba45d2, 0x00d826c1, 0x01040820, 0x013729b9, 0x0171ab8d, 0x01b36d9b,
+    0x00eee777, 0x00c2c616, 0x00ae2571, 0x009fe4ff, 0x0096e4b7, 0x0090e487, 0x008d446a, 0x008d2469, 0x0090a485, 0x009684b4, 0x009f64fb, 0x00ad456a, 0x00c1a60d, 0x00eca765, 0x011fc8fe, 0x015a4ad2, 0x019c0ce0,
+    0x00dee6f7, 0x00b9c5ce, 0x00a5652b, 0x009964cb, 0x00904482, 0x00892449, 0x0085842c, 0x0085642b, 0x0088e447, 0x008fe47f, 0x0098e4c7, 0x00a4c526, 0x00b8a5c5, 0x00dc86e4, 0x010fc87e, 0x014a2a51, 0x018c0c60,
+    0x00d626b1, 0x00b4e5a7, 0x00a1e50f, 0x0095e4af, 0x008c2461, 0x00850428, 0x0081640b, 0x0081440a, 0x0084a425, 0x008ba45d, 0x009564ab, 0x00a1450a, 0x00b3c59e, 0x00d3e69f, 0x01070838, 0x01418a0c, 0x01834c1a,
+    0x00d4c6a6, 0x00b425a1, 0x00a1450a, 0x009544aa, 0x008b645b, 0x00844422, 0x0080a405, 0x0080a405, 0x00840420, 0x008b0458, 0x0094c4a6, 0x00a0a505, 0x00b30598, 0x00d26693, 0x0105a82d, 0x01402a01, 0x0181ec0f,
+    0x00daa6d5, 0x00b765bb, 0x00a3c51e, 0x0097a4bd, 0x008e4472, 0x00872439, 0x0083841c, 0x0083641b, 0x0086e437, 0x008de46f, 0x009724b9, 0x00a30518, 0x00b665b3, 0x00d866c3, 0x010b885c, 0x01460a30, 0x0187ec3f,
+    0x00e80740, 0x00bec5f6, 0x00aa6553, 0x009d24e9, 0x009404a0, 0x008d846c, 0x0089e44f, 0x0089e44f, 0x008d446a, 0x0093c49e, 0x009ca4e5, 0x00a9854c, 0x00bdc5ee, 0x00e5a72d, 0x0118c8c6, 0x01534a9a, 0x01952ca9,
+    0x00fca7e5, 0x00d06683, 0x00b5c5ae, 0x00a5852c, 0x009c84e4, 0x009664b3, 0x0093649b, 0x0093449a, 0x009624b1, 0x009c24e1, 0x00a50528, 0x00b4e5a7, 0x00ce8674, 0x00fa47d2, 0x012d696b, 0x0167eb3f, 0x01a9cd4e,
+    0x011888c4, 0x00ec6763, 0x00c7863c, 0x00b4e5a7, 0x00a8a545, 0x00a1c50e, 0x009ec4f6, 0x009ea4f5, 0x00a1a50d, 0x00a82541, 0x00b445a2, 0x00c5e62f, 0x00ea6753, 0x011648b2, 0x01496a4b, 0x0183ec1f, 0x01c5ae2d,
+    0x013bc9de, 0x010fa87d, 0x00eac756, 0x00cd466a, 0x00bc25e1, 0x00b405a0, 0x00afc57e, 0x00afa57d, 0x00b3a59d, 0x00bbc5de, 0x00cc0660, 0x00e92749, 0x010da86d, 0x013989cc, 0x016cab65, 0x01a72d39, 0x01e8ef47,
+    0x01666b33, 0x013a49d2, 0x011568ab, 0x00f7e7bf, 0x00e1c70e, 0x00d2e697, 0x00cb665b, 0x00cb2659, 0x00d26693, 0x00e0c706, 0x00f6a7b5, 0x0113c89e, 0x013849c2, 0x01642b21, 0x01974cba, 0x01d1ce8e, 0x0213909c,
+    0x01986cc3, 0x016c2b61, 0x01476a3b, 0x0129e94f, 0x0113a89d, 0x0104c826, 0x00fd47ea, 0x00fd27e9, 0x01044822, 0x0112c896, 0x0128a945, 0x0145ca2e, 0x016a4b52, 0x01960cb0, 0x01c92e49, 0x0203b01d, 0x0245922c,
+    0x01d1ae8d, 0x01a58d2c, 0x0180ac05, 0x01632b19, 0x014cea67, 0x013e29f1, 0x013689b4, 0x013669b3, 0x013d89ec, 0x014c0a60, 0x0161eb0f, 0x017f0bf8, 0x01a38d1c, 0x01cf4e7a, 0x02029014, 0x023d11e8, 0x027ed3f6,
+  };
+}
+
+std::vector<i2c_random_wr_payload> OS04C10::getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const {
+  uint32_t long_time = exposure_time;
+  uint32_t real_gain = os04c10_analog_gains_reg[new_exp_g];
+
+  return {
+    {0x3501, long_time>>8}, {0x3502, long_time&0xFF},
+    {0x3508, real_gain>>8}, {0x3509, real_gain&0xFF},
+    {0x350c, real_gain>>8}, {0x350d, real_gain&0xFF},
+  };
+}
+
+int OS04C10::getSlaveAddress(int port) const {
+  assert(port >= 0 && port <= 2);
+  return (int[]){0x6C, 0x20, 0x6C}[port];
+}
+
+float OS04C10::getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const {
+  float score = std::abs(desired_ev - (exp_t * exp_gain));
+  float m = exp_g_idx > analog_gain_rec_idx ? analog_gain_cost_high : analog_gain_cost_low;
+  score += std::abs(exp_g_idx - (int)analog_gain_rec_idx) * m;
+  score += ((1 - analog_gain_cost_delta) +
+            analog_gain_cost_delta * (exp_g_idx - analog_gain_min_idx) / (analog_gain_max_idx - analog_gain_min_idx)) *
+           std::abs(exp_g_idx - gain_idx) * 3.0;
+  return score;
+}
@@ -0,0 +1,332 @@
+#pragma once
+
+const struct i2c_random_wr_payload start_reg_array_os04c10[] = {{0x100, 1}};
+const struct i2c_random_wr_payload stop_reg_array_os04c10[] = {{0x100, 0}};
+
+const struct i2c_random_wr_payload init_array_os04c10[] = {
+  // OS04C10_AA_00_02_17_wAO_2688x1524_MIPI728Mbps_Linear12bit_20FPS_4Lane_MCLK24MHz
+  {0x0103, 0x01}, // software reset
+
+  // PLL + clocks
+  {0x0301, 0xe4},
+  {0x0303, 0x01},
+  {0x0305, 0xb6},
+  {0x0306, 0x01},
+  {0x0307, 0x17},
+  {0x0323, 0x04},
+  {0x0324, 0x01},
+  {0x0325, 0x62},
+
+  {0x3012, 0x06},
+  {0x3013, 0x02},
+  {0x3016, 0x72},
+  {0x3021, 0x03},
+  {0x3106, 0x21},
+  {0x3107, 0xa1},
+
+  // Analog/timing fine-tuning block
+  {0x3624, 0x00},
+  {0x3625, 0x4c},
+  {0x3660, 0x04},
+  {0x3666, 0xa5},
+  {0x3667, 0xa5},
+  {0x366a, 0x50},
+  {0x3673, 0x0d},
+  {0x3672, 0x0d},
+  {0x3671, 0x0d},
+  {0x3670, 0x0d},
+  {0x3685, 0x00},
+  {0x3694, 0x0d},
+  {0x3693, 0x0d},
+  {0x3692, 0x0d},
+  {0x3691, 0x0d},
+  {0x3696, 0x4c},
+  {0x3697, 0x4c},
+  {0x3698, 0x00},
+  {0x3699, 0x80},
+  {0x369a, 0x80},
+  {0x369b, 0x1f},
+  {0x369c, 0x1f},
+  {0x369d, 0x80},
+  {0x369e, 0x40},
+  {0x369f, 0x21},
+  {0x36a0, 0x12},
+  {0x36a1, 0xdd},
+  {0x36a2, 0x66},
+  {0x370a, 0x02},
+  {0x370e, 0x00},
+  {0x3710, 0x00},
+  {0x3713, 0x04},
+  {0x3725, 0x02},
+  {0x372a, 0x03},
+  {0x3738, 0xce},
+  {0x3748, 0x02},
+  {0x374a, 0x02},
+  {0x374c, 0x02},
+  {0x374e, 0x02},
+  {0x3756, 0x00},
+  {0x3757, 0x00},
+  {0x3767, 0x00},
+  {0x3771, 0x00},
+  {0x377b, 0x28},
+  {0x377c, 0x00},
+  {0x377d, 0x0c},
+  {0x3781, 0x03},
+  {0x3782, 0x00},
+  {0x3789, 0x14},
+  {0x3795, 0x02},
+  {0x379c, 0x00},
+  {0x379d, 0x00},
+  {0x37b8, 0x04},
+  {0x37ba, 0x03},
+  {0x37bb, 0x00},
+  {0x37bc, 0x04},
+  {0x37be, 0x26},
+  {0x37c4, 0x11},
+  {0x37c5, 0x80},
+  {0x37c6, 0x14},
+  {0x37c7, 0xa8},
+  {0x37da, 0x11},
+  {0x381f, 0x08},
+  {0x3881, 0x00},
+  {0x3888, 0x04},
+  {0x388b, 0x00},
+  {0x3c80, 0x10},
+  {0x3c86, 0x00},
+  {0x3c8c, 0x40},
+  {0x3c9f, 0x01},
+  {0x3d85, 0x1b},
+  {0x3d8c, 0x71},
+  {0x3d8d, 0xe2},
+  {0x3f00, 0x0b},
+  {0x3f06, 0x04},
+
+  // BLC - black level correction
+  {0x400a, 0x01},
+  {0x400b, 0x50},
+  {0x400e, 0x08},
+  {0x4043, 0x7e},
+  {0x4045, 0x7e},
+  {0x4047, 0x7e},
+  {0x4049, 0x7e},
+  {0x4090, 0x04},
+  {0x40b0, 0x00},
+  {0x40b1, 0x00},
+  {0x40b2, 0x00},
+  {0x40b3, 0x00},
+  {0x40b4, 0x00},
+  {0x40b5, 0x00},
+  {0x40b7, 0x00},
+  {0x40b8, 0x00},
+  {0x40b9, 0x00},
+  {0x40ba, 0x01},
+
+  {0x4301, 0x00},
+  {0x4303, 0x00},
+  {0x4502, 0x04},
+  {0x4503, 0x00},
+  {0x4504, 0x06},
+  {0x4506, 0x00},
+  {0x4507, 0x47},
+  {0x4803, 0x00},
+  {0x480c, 0x32},
+  {0x480e, 0x04},
+  {0x4813, 0xe4},
+  {0x4819, 0x70},
+  {0x481f, 0x30},
+  {0x4823, 0x3f},
+  {0x4825, 0x30},
+  {0x4833, 0x10},
+  {0x484b, 0x27},
+  {0x488b, 0x00},
+  {0x4d00, 0x04},
+  {0x4d01, 0xad},
+  {0x4d02, 0xbc},
+  {0x4d03, 0xa1},
+  {0x4d04, 0x1f},
+  {0x4d05, 0x4c},
+  {0x4d0b, 0x01},
+  {0x4e00, 0x2a},
+  {0x4e0d, 0x00},
+
+  // ISP
+  {0x5001, 0x09},
+  {0x5004, 0x00},
+  {0x5080, 0x04},
+  {0x5036, 0x80},
+  {0x5180, 0x70},
+  {0x5181, 0x10},
+
+  // DPC - defective pixel correction
+  {0x520a, 0x03},
+  {0x520b, 0x06},
+  {0x520c, 0x0c},
+
+  {0x580b, 0x0f},
+  {0x580d, 0x00},
+  {0x580f, 0x00},
+  {0x5820, 0x00},
+  {0x5821, 0x00},
+
+  {0x301c, 0xf8},
+  {0x301e, 0xb4},
+  {0x301f, 0xf0},
+  {0x3022, 0x61},
+  {0x3109, 0xe7},
+  {0x3600, 0x00},
+  {0x3610, 0x65},
+  {0x3611, 0x85},
+  {0x3613, 0x3a},
+  {0x3615, 0x60},
+  {0x3621, 0xb0},
+  {0x3620, 0x0c},
+  {0x3629, 0x00},
+  {0x3661, 0x04},
+  {0x3664, 0x70},
+  {0x3665, 0x00},
+  {0x3681, 0x80},
+  {0x3682, 0x40},
+  {0x3683, 0x21},
+  {0x3684, 0x12},
+  {0x3700, 0x2a},
+  {0x3701, 0x12},
+  {0x3703, 0x28},
+  {0x3704, 0x0e},
+  {0x3706, 0x9d},
+  {0x3709, 0x4a},
+  {0x370b, 0x48},
+  {0x370c, 0x01},
+  {0x370f, 0x00},
+  {0x3714, 0x24},
+  {0x3716, 0x04},
+  {0x3719, 0x11},
+  {0x371a, 0x1e},
+  {0x3720, 0x00},
+  {0x3724, 0x13},
+  {0x373f, 0xb0},
+  {0x3741, 0x9d},
+  {0x3743, 0x9d},
+  {0x3745, 0x9d},
+  {0x3747, 0x9d},
+  {0x3749, 0x48},
+  {0x374b, 0x48},
+  {0x374d, 0x48},
+  {0x374f, 0x48},
+  {0x3755, 0x10},
+  {0x376c, 0x00},
+  {0x378d, 0x3c},
+  {0x3790, 0x01},
+  {0x3791, 0x01},
+  {0x3798, 0x40},
+  {0x379e, 0x00},
+  {0x379f, 0x04},
+  {0x37a1, 0x10},
+  {0x37a2, 0x1e},
+  {0x37a8, 0x10},
+  {0x37a9, 0x1e},
+  {0x37ac, 0xa0},
+  {0x37b9, 0x01},
+  {0x37bd, 0x01},
+  {0x37bf, 0x26},
+  {0x37c0, 0x11},
+  {0x37c2, 0x04},
+  {0x37cd, 0x19},
+  {0x37e0, 0x08},
+  {0x37e6, 0x04},
+  {0x37e5, 0x02},
+  {0x37e1, 0x0c},
+  {0x3737, 0x04},
+  {0x37d8, 0x02},
+  {0x37e2, 0x10},
+  {0x3739, 0x10},
+  {0x3662, 0x10},
+  {0x37e4, 0x20},
+  {0x37e3, 0x08},
+  {0x37d9, 0x08},
+  {0x4040, 0x00},
+  {0x4041, 0x07},
+  {0x4008, 0x02},
+  {0x4009, 0x0d},
+
+  // FSIN - frame sync
+  {0x3002, 0x22},
+  {0x3663, 0x22},
+  {0x368a, 0x04},
+  {0x3822, 0x44},
+  {0x3823, 0x00},
+  {0x3829, 0x03},
+  {0x3832, 0xf8},
+  {0x382c, 0x00},
+  {0x3844, 0x06},
+  {0x3843, 0x00},
+  {0x382a, 0x00},
+  {0x382b, 0x0c},
+
+  // 2704x1536 -> 2688x1520 out
+  {0x3800, 0x00}, {0x3801, 0x00},
+  {0x3802, 0x00}, {0x3803, 0x00},
+  {0x3804, 0x0a}, {0x3805, 0x8f},
+  {0x3806, 0x05}, {0x3807, 0xff},
+  {0x3808, 0x0a}, {0x3809, 0x80},
+  {0x380a, 0x05}, {0x380b, 0xf0},
+  {0x3811, 0x08},
+  {0x3813, 0x08},
+  {0x3814, 0x01},
+  {0x3815, 0x01},
+  {0x3816, 0x01},
+  {0x3817, 0x01},
+
+  {0x380c, 0x08}, {0x380d, 0x5c}, // HTS (line length)
+  {0x380e, 0x09}, {0x380f, 0x38}, // VTS (frame length)
+
+  {0x3820, 0xb0},
+  {0x3821, 0x00},
+  {0x3880, 0x00},
+  {0x3882, 0x20},
+  {0x3c91, 0x0b},
+  {0x3c94, 0x45},
+  {0x3cad, 0x00},
+  {0x3cae, 0x00},
+  {0x4000, 0xf3},
+  {0x4001, 0x60},
+  {0x4003, 0x40},
+  {0x4300, 0xff},
+  {0x4302, 0x0f},
+  {0x4305, 0x83},
+  {0x4505, 0x84},
+  {0x4809, 0x0e},
+  {0x480a, 0x04},
+  {0x4837, 0x15},
+  {0x4c00, 0x08},
+  {0x4c01, 0x08},
+  {0x4c04, 0x00},
+  {0x4c05, 0x00},
+  {0x5000, 0xf9},
+  // {0x0100, 0x01},
+  // {0x320d, 0x00},
+  // {0x3208, 0xa0},
+
+  // initialize exposure
+  {0x3503, 0x88},
+
+  // long exposure
+  {0x3500, 0x00}, {0x3501, 0x00}, {0x3502, 0x10},
+  {0x3508, 0x00}, {0x3509, 0x80},
+  {0x350a, 0x04}, {0x350b, 0x00},
+
+  // short exposure
+  {0x3510, 0x00}, {0x3511, 0x00}, {0x3512, 0x40},
+  {0x350c, 0x00}, {0x350d, 0x80},
+  {0x350e, 0x04}, {0x350f, 0x00},
+
+  // white balance
+  // b
+  {0x5100, 0x06}, {0x5101, 0x7e},
+  {0x5140, 0x06}, {0x5141, 0x7e},
+  // g
+  {0x5102, 0x04}, {0x5103, 0x00},
+  {0x5142, 0x04}, {0x5143, 0x00},
+  // r
+  {0x5104, 0x08}, {0x5105, 0xd6},
+  {0x5144, 0x08}, {0x5145, 0xd6},
+};
@@ -0,0 +1,142 @@
+#include <cmath>
+
+#include "system/camerad/sensors/sensor.h"
+#include <media/msm_camsensor_sdk.h>
+
+namespace {
+
+const float sensor_analog_gains_OX03C10[] = {
+    1.0, 1.0625, 1.125, 1.1875, 1.25, 1.3125, 1.375, 1.4375, 1.5, 1.5625, 1.6875,
+    1.8125, 1.9375, 2.0, 2.125, 2.25, 2.375, 2.5, 2.625, 2.75, 2.875, 3.0,
+    3.125, 3.375, 3.625, 3.875, 4.0, 4.25, 4.5, 4.75, 5.0, 5.25, 5.5,
+    5.75, 6.0, 6.25, 6.5, 7.0, 7.5, 8.0, 8.5, 9.0, 9.5, 10.0,
+    10.5, 11.0, 11.5, 12.0, 12.5, 13.0, 13.5, 14.0, 14.5, 15.0, 15.5};
+
+const uint32_t ox03c10_analog_gains_reg[] = {
+    0x100, 0x110, 0x120, 0x130, 0x140, 0x150, 0x160, 0x170, 0x180, 0x190, 0x1B0,
+    0x1D0, 0x1F0, 0x200, 0x220, 0x240, 0x260, 0x280, 0x2A0, 0x2C0, 0x2E0, 0x300,
+    0x320, 0x360, 0x3A0, 0x3E0, 0x400, 0x440, 0x480, 0x4C0, 0x500, 0x540, 0x580,
+    0x5C0, 0x600, 0x640, 0x680, 0x700, 0x780, 0x800, 0x880, 0x900, 0x980, 0xA00,
+    0xA80, 0xB00, 0xB80, 0xC00, 0xC80, 0xD00, 0xD80, 0xE00, 0xE80, 0xF00, 0xF80};
+
+const uint32_t VS_TIME_MIN_OX03C10 = 1;
+const uint32_t VS_TIME_MAX_OX03C10 = 34;  // vs < 35
+
+}  // namespace
+
+OX03C10::OX03C10() {
+  image_sensor = cereal::FrameData::ImageSensor::OX03C10;
+  bayer_pattern = CAM_ISP_PATTERN_BAYER_GRGRGR;
+  pixel_size_mm = 0.003;
+  data_word = false;
+  frame_width = 1928;
+  frame_height = 1208;
+  frame_stride = (frame_width * 12 / 8) + 4;
+  extra_height = 16;            // top 2 + bot 14
+  frame_offset = 2;
+
+  start_reg_array.assign(std::begin(start_reg_array_ox03c10), std::end(start_reg_array_ox03c10));
+  init_reg_array.assign(std::begin(init_array_ox03c10), std::end(init_array_ox03c10));
+  probe_reg_addr = 0x300a;
+  probe_expected_data = 0x5803;
+  bits_per_pixel = 12;
+  mipi_format = CAM_FORMAT_MIPI_RAW_12;
+  frame_data_type = CSI_RAW12;
+  mclk_frequency = 24000000; // Hz
+
+  readout_time_ns = 14697000;
+
+  dc_gain_factor = 7.32;
+  dc_gain_min_weight = 1;  // always on is fine
+  dc_gain_max_weight = 1;
+  dc_gain_on_grey = 0.9;
+  dc_gain_off_grey = 1.0;
+  exposure_time_min = 2;  // 1x
+  exposure_time_max = 2016;
+  analog_gain_min_idx = 0x0;
+  analog_gain_rec_idx = 0x0;  // 1x
+  analog_gain_max_idx = 0x36;
+  analog_gain_cost_delta = -1;
+  analog_gain_cost_low = 0.4;
+  analog_gain_cost_high = 6.4;
+  for (int i = 0; i <= analog_gain_max_idx; i++) {
+    sensor_analog_gains[i] = sensor_analog_gains_OX03C10[i];
+  }
+  min_ev = (exposure_time_min + VS_TIME_MIN_OX03C10) * sensor_analog_gains[analog_gain_min_idx];
+  max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
+  target_grey_factor = 0.01;
+
+  black_level = 0;
+  color_correct_matrix = {
+    0x000000b6, 0x00000ff1, 0x00000fda,
+    0x00000fcc, 0x000000b9, 0x00000ffb,
+    0x00000fc2, 0x00000ff6, 0x000000c9,
+  };
+  for (int i = 0; i < 65; i++) {
+    float fx = i / 64.0;
+    fx = -0.507089*exp(-12.54124638*fx) + 0.9655*pow(fx, 0.5) - 0.472597*fx + 0.507089;
+    gamma_lut_rgb.push_back((uint32_t)(fx*1023.0 + 0.5));
+  }
+  prepare_gamma_lut();
+  linearization_lut = {
+    0x00200000, 0x00200000, 0x00200000, 0x00200000,
+    0x00404080, 0x00404080, 0x00404080, 0x00404080,
+    0x00804100, 0x00804100, 0x00804100, 0x00804100,
+    0x02014402, 0x02014402, 0x02014402, 0x02014402,
+    0x0402c804, 0x0402c804, 0x0402c804, 0x0402c804,
+    0x0805d00a, 0x0805d00a, 0x0805d00a, 0x0805d00a,
+    0x100ba015, 0x100ba015, 0x100ba015, 0x100ba015,
+    0x00003fff, 0x00003fff, 0x00003fff, 0x00003fff,
+    0x00003fff, 0x00003fff, 0x00003fff, 0x00003fff,
+  };
+  linearization_pts = {0x07ff0bff, 0x17ff1bff, 0x1fff23ff, 0x27ff3fff};
+  vignetting_lut = {
+    0x00eaa755, 0x00cf2679, 0x00bc05e0, 0x00acc566, 0x00a1450a, 0x009984cc, 0x0095a4ad, 0x009584ac, 0x009944ca, 0x00a0c506, 0x00ac0560, 0x00bb25d9, 0x00ce2671, 0x00e90748, 0x01112889, 0x014a2a51, 0x01984cc2,
+    0x00db06d8, 0x00c30618, 0x00afe57f, 0x00a0a505, 0x009524a9, 0x008d646b, 0x0089844c, 0x0089644b, 0x008d2469, 0x0094a4a5, 0x009fe4ff, 0x00af0578, 0x00c20610, 0x00d986cc, 0x00fda7ed, 0x01320990, 0x017aebd7,
+    0x00d1868c, 0x00baa5d5, 0x00a7853c, 0x009844c2, 0x008cc466, 0x0085a42d, 0x0083641b, 0x0083641b, 0x0085842c, 0x008c4462, 0x0097a4bd, 0x00a6c536, 0x00b9a5cd, 0x00d06683, 0x00f1678b, 0x01226913, 0x0167ab3d,
+    0x00cd0668, 0x00b625b1, 0x00a30518, 0x0093c49e, 0x00884442, 0x00830418, 0x0080e407, 0x0080c406, 0x0082e417, 0x0087c43e, 0x00932499, 0x00a22511, 0x00b525a9, 0x00cbe65f, 0x00eb0758, 0x011a68d3, 0x015daaed,
+    0x00cc4662, 0x00b565ab, 0x00a24512, 0x00930498, 0x0087843c, 0x0082a415, 0x00806403, 0x00806403, 0x00828414, 0x00870438, 0x00926493, 0x00a1850c, 0x00b465a3, 0x00cb2659, 0x00ea2751, 0x011928c9, 0x015c2ae1,
+    0x00cf667b, 0x00b885c4, 0x00a5652b, 0x009624b1, 0x008aa455, 0x00846423, 0x00822411, 0x00822411, 0x00844422, 0x008a2451, 0x009564ab, 0x00a48524, 0x00b785bc, 0x00ce4672, 0x00ee6773, 0x011e88f4, 0x0162eb17,
+    0x00d6c6b6, 0x00bf65fb, 0x00ac4562, 0x009d04e8, 0x0091848c, 0x0089c44e, 0x00862431, 0x00860430, 0x0089844c, 0x00910488, 0x009c64e3, 0x00ab655b, 0x00be65f3, 0x00d566ab, 0x00f847c2, 0x012b2959, 0x01726b93,
+    0x00e3e71f, 0x00ca0650, 0x00b705b8, 0x00a7a53d, 0x009c24e1, 0x009484a4, 0x00908484, 0x00908484, 0x009424a1, 0x009bc4de, 0x00a70538, 0x00b625b1, 0x00c90648, 0x00e26713, 0x0108e847, 0x013fe9ff, 0x018bcc5e,
+    0x00f807c0, 0x00d966cb, 0x00c5862c, 0x00b625b1, 0x00aaa555, 0x00a30518, 0x009f04f8, 0x009f04f8, 0x00a2a515, 0x00aa2551, 0x00b585ac, 0x00c4a625, 0x00d846c2, 0x00f647b2, 0x0121a90d, 0x015e4af2, 0x01b8cdc6,
+    0x011548aa, 0x00f1678b, 0x00d886c4, 0x00c86643, 0x00bce5e7, 0x00b545aa, 0x00b1658b, 0x00b1458a, 0x00b505a8, 0x00bc85e4, 0x00c7c63e, 0x00d786bc, 0x00efe77f, 0x0113489a, 0x0144ea27, 0x01888c44, 0x01fdcfee,
+    0x013e49f2, 0x0113e89f, 0x00f5a7ad, 0x00e0c706, 0x00d30698, 0x00cb665b, 0x00c7663b, 0x00c7663b, 0x00cb0658, 0x00d2a695, 0x00dfe6ff, 0x00f467a3, 0x01122891, 0x013be9df, 0x01750ba8, 0x01cfae7d, 0x025912c8,
+    0x01766bb3, 0x01446a23, 0x011fc8fe, 0x0105e82f, 0x00f467a3, 0x00e9874c, 0x00e46723, 0x00e44722, 0x00e92749, 0x00f3a79d, 0x0104c826, 0x011e48f2, 0x01424a12, 0x01738b9c, 0x01bf6dfb, 0x023611b0, 0x02ced676,
+    0x01cf8e7c, 0x01866c33, 0x015aaad5, 0x013ae9d7, 0x01250928, 0x011768bb, 0x0110a885, 0x01108884, 0x0116e8b7, 0x01242921, 0x0139a9cd, 0x0158eac7, 0x01840c20, 0x01cb0e58, 0x0233719b, 0x02b9d5ce, 0x03645b22,
+  };
+}
+
+std::vector<i2c_random_wr_payload> OX03C10::getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const {
+ // t_HCG&t_LCG + t_VS on LPD, t_SPD on SPD
+  uint32_t hcg_time = exposure_time;
+  uint32_t lcg_time = hcg_time;
+  uint32_t spd_time = std::min(std::max((uint32_t)exposure_time, (exposure_time_max + VS_TIME_MAX_OX03C10) / 3), exposure_time_max + VS_TIME_MAX_OX03C10);
+  uint32_t vs_time = std::min(std::max((uint32_t)exposure_time / 40, VS_TIME_MIN_OX03C10), VS_TIME_MAX_OX03C10);
+
+  uint32_t real_gain = ox03c10_analog_gains_reg[new_exp_g];
+
+  return {
+    {0x3501, hcg_time>>8}, {0x3502, hcg_time&0xFF},
+    {0x3581, lcg_time>>8}, {0x3582, lcg_time&0xFF},
+    {0x3541, spd_time>>8}, {0x3542, spd_time&0xFF},
+    {0x35c2, vs_time&0xFF},
+
+    {0x3508, real_gain>>8}, {0x3509, real_gain&0xFF},
+  };
+}
+
+int OX03C10::getSlaveAddress(int port) const {
+  assert(port >= 0 && port <= 2);
+  return (int[]){0x6C, 0x20, 0x6C}[port];
+}
+
+float OX03C10::getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const {
+  float score = std::abs(desired_ev - (exp_t * exp_gain));
+  float m = exp_g_idx > analog_gain_rec_idx ? analog_gain_cost_high : analog_gain_cost_low;
+  score += std::abs(exp_g_idx - (int)analog_gain_rec_idx) * m;
+  score += ((1 - analog_gain_cost_delta) +
+            analog_gain_cost_delta * (exp_g_idx - analog_gain_min_idx) / (analog_gain_max_idx - analog_gain_min_idx)) *
+           std::abs(exp_g_idx - gain_idx) * 5.0;
+  return score;
+}
@@ -0,0 +1,751 @@
+#pragma once
+
+const struct i2c_random_wr_payload start_reg_array_ox03c10[] = {{0x100, 1}};
+const struct i2c_random_wr_payload stop_reg_array_ox03c10[] = {{0x100, 0}};
+
+const struct i2c_random_wr_payload init_array_ox03c10[] = {
+  {0x103, 1},
+  {0x107, 1},
+
+  // X3C_1920x1280_60fps_HDR4_LFR_PWL12_mipi1200
+
+  // TPM
+  {0x4d5a, 0x1a}, {0x4d09, 0xff}, {0x4d09, 0xdf},
+
+  /*)
+  // group 4
+  {0x3208, 0x04},
+  {0x4620, 0x04},
+  {0x3208, 0x14},
+
+  // group 5
+  {0x3208, 0x05},
+  {0x4620, 0x04},
+  {0x3208, 0x15},
+
+  // group 2
+  {0x3208, 0x02},
+  {0x3507, 0x00},
+  {0x3208, 0x12},
+
+  // delay launch group 2
+  {0x3208, 0xa2},*/
+
+  // **NOTE**: if this is changed, readout_time_ns must be updated in the Sensor config
+  // PLL setup
+  {0x0301, 0xc8}, // pll1_divs, pll1_predivp, pll1_divpix
+  {0x0303, 0x01}, // pll1_prediv
+  {0x0304, 0x01}, {0x0305, 0x2c}, // pll1_loopdiv = 300
+  {0x0306, 0x04}, // pll1_divmipi = 4
+  {0x0307, 0x01}, // pll1_divm = 1
+  {0x0316, 0x00},
+  {0x0317, 0x00},
+  {0x0318, 0x00},
+  {0x0323, 0x05}, // pll2_prediv
+  {0x0324, 0x01}, {0x0325, 0x2c}, // pll2_divp = 300
+
+  // SCLK/PCLK
+  {0x0400, 0xe0}, {0x0401, 0x80},
+  {0x0403, 0xde}, {0x0404, 0x34},
+  {0x0405, 0x3b}, {0x0406, 0xde},
+  {0x0407, 0x08},
+  {0x0408, 0xe0}, {0x0409, 0x7f},
+  {0x040a, 0xde}, {0x040b, 0x34},
+  {0x040c, 0x47}, {0x040d, 0xd8},
+  {0x040e, 0x08},
+
+  // xchk
+  {0x2803, 0xfe}, {0x280b, 0x00}, {0x280c, 0x79},
+
+  // SC ctrl
+  {0x3001, 0x03}, // io_pad_oen
+  {0x3002, 0xfc}, // io_pad_oen
+  {0x3005, 0x80}, // io_pad_out
+  {0x3007, 0x01}, // io_pad_sel
+  {0x3008, 0x80}, // io_pad_sel
+
+  // FSIN (frame sync) with external pulses
+  {0x3009, 0x2},
+  {0x3015, 0x2},
+  {0x383E, 0x80},
+  {0x3881, 0x4},
+  {0x3882, 0x8}, {0x3883, 0x0D},
+  {0x3836, 0x1F}, {0x3837, 0x40},
+
+  // causes issues on some devices
+  //{0x3822, 0x33},  // wait for pulse before first frame
+
+  {0x3892, 0x44},
+  {0x3823, 0x41},
+
+  {0x3012, 0x41}, // SC_PHY_CTRL = 4 lane MIPI
+  {0x3020, 0x05}, // SC_CTRL_20
+
+  // this is not in the datasheet, listed as RSVD
+  // but the camera doesn't work without it
+  {0x3700, 0x28}, {0x3701, 0x15}, {0x3702, 0x19}, {0x3703, 0x23},
+  {0x3704, 0x0a}, {0x3705, 0x00}, {0x3706, 0x3e}, {0x3707, 0x0d},
+  {0x3708, 0x50}, {0x3709, 0x5a}, {0x370a, 0x00}, {0x370b, 0x96},
+  {0x3711, 0x11}, {0x3712, 0x13}, {0x3717, 0x02}, {0x3718, 0x73},
+  {0x372c, 0x40}, {0x3733, 0x01}, {0x3738, 0x36}, {0x3739, 0x36},
+  {0x373a, 0x25}, {0x373b, 0x25}, {0x373f, 0x21}, {0x3740, 0x21},
+  {0x3741, 0x21}, {0x3742, 0x21}, {0x3747, 0x28}, {0x3748, 0x28},
+  {0x3749, 0x19}, {0x3755, 0x1a}, {0x3756, 0x0a}, {0x3757, 0x1c},
+  {0x3765, 0x19}, {0x3766, 0x05}, {0x3767, 0x05}, {0x3768, 0x13},
+  {0x376c, 0x07}, {0x3778, 0x20}, {0x377c, 0xc8}, {0x3781, 0x02},
+  {0x3783, 0x02}, {0x379c, 0x58}, {0x379e, 0x00}, {0x379f, 0x00},
+  {0x37a0, 0x00}, {0x37bc, 0x22}, {0x37c0, 0x01}, {0x37c4, 0x3e},
+  {0x37c5, 0x3e}, {0x37c6, 0x2a}, {0x37c7, 0x28}, {0x37c8, 0x02},
+  {0x37c9, 0x12}, {0x37cb, 0x29}, {0x37cd, 0x29}, {0x37d2, 0x00},
+  {0x37d3, 0x73}, {0x37d6, 0x00}, {0x37d7, 0x6b}, {0x37dc, 0x00},
+  {0x37df, 0x54}, {0x37e2, 0x00}, {0x37e3, 0x00}, {0x37f8, 0x00},
+  {0x37f9, 0x01}, {0x37fa, 0x00}, {0x37fb, 0x19},
+
+  // also RSVD
+  {0x3c03, 0x01}, {0x3c04, 0x01}, {0x3c06, 0x21}, {0x3c08, 0x01},
+  {0x3c09, 0x01}, {0x3c0a, 0x01}, {0x3c0b, 0x21}, {0x3c13, 0x21},
+  {0x3c14, 0x82}, {0x3c16, 0x13}, {0x3c21, 0x00}, {0x3c22, 0xf3},
+  {0x3c37, 0x12}, {0x3c38, 0x31}, {0x3c3c, 0x00}, {0x3c3d, 0x03},
+  {0x3c44, 0x16}, {0x3c5c, 0x8a}, {0x3c5f, 0x03}, {0x3c61, 0x80},
+  {0x3c6f, 0x2b}, {0x3c70, 0x5f}, {0x3c71, 0x2c}, {0x3c72, 0x2c},
+  {0x3c73, 0x2c}, {0x3c76, 0x12},
+
+  // PEC checks
+  {0x3182, 0x12},
+
+  {0x320e, 0x00}, {0x320f, 0x00}, // RSVD
+  {0x3211, 0x61},
+  {0x3215, 0xcd},
+  {0x3219, 0x08},
+
+  {0x3506, 0x20}, {0x3507, 0x00}, // hcg fine exposure
+  {0x350a, 0x01}, {0x350b, 0x00}, {0x350c, 0x00}, // hcg digital gain
+
+  {0x3586, 0x40}, {0x3587, 0x00}, // lcg fine exposure
+  {0x358a, 0x01}, {0x358b, 0x00}, {0x358c, 0x00}, // lcg digital gain
+
+  {0x3546, 0x20}, {0x3547, 0x00}, // spd fine exposure
+  {0x354a, 0x01}, {0x354b, 0x00}, {0x354c, 0x00}, // spd digital gain
+
+  {0x35c6, 0xb0}, {0x35c7, 0x00}, // vs fine exposure
+  {0x35ca, 0x01}, {0x35cb, 0x00}, {0x35cc, 0x00}, // vs digital gain
+
+  // also RSVD
+  {0x3600, 0x8f}, {0x3605, 0x16}, {0x3609, 0xf0}, {0x360a, 0x01},
+  {0x360e, 0x1d}, {0x360f, 0x10}, {0x3610, 0x70}, {0x3611, 0x3a},
+  {0x3612, 0x28}, {0x361a, 0x29}, {0x361b, 0x6c}, {0x361c, 0x0b},
+  {0x361d, 0x00}, {0x361e, 0xfc}, {0x362a, 0x00}, {0x364d, 0x0f},
+  {0x364e, 0x18}, {0x364f, 0x12}, {0x3653, 0x1c}, {0x3654, 0x00},
+  {0x3655, 0x1f}, {0x3656, 0x1f}, {0x3657, 0x0c}, {0x3658, 0x0a},
+  {0x3659, 0x14}, {0x365a, 0x18}, {0x365b, 0x14}, {0x365c, 0x10},
+  {0x365e, 0x12}, {0x3674, 0x08}, {0x3677, 0x3a}, {0x3678, 0x3a},
+  {0x3679, 0x19},
+
+  // Y_ADDR_START = 4
+  {0x3802, 0x00}, {0x3803, 0x04},
+  // Y_ADDR_END = 0x50b
+  {0x3806, 0x05}, {0x3807, 0x0b},
+
+  // X_OUTPUT_SIZE = 0x780 = 1920 (changed to 1928)
+  {0x3808, 0x07}, {0x3809, 0x88},
+
+  // Y_OUTPUT_SIZE = 0x500 = 1280 (changed to 1208)
+  {0x380a, 0x04}, {0x380b, 0xb8},
+
+  // horizontal timing 0x447
+  {0x380c, 0x04}, {0x380d, 0x47},
+
+  // rows per frame (was 0x2ae)
+  // 0x8ae = 53.65 ms
+  {0x380e, 0x08}, {0x380f, 0x15},
+  // this should be triggered by FSIN, not free running
+
+  {0x3810, 0x00}, {0x3811, 0x08}, // x cutoff
+  {0x3812, 0x00}, {0x3813, 0x04}, // y cutoff
+  {0x3816, 0x01},
+  {0x3817, 0x01},
+  {0x381c, 0x18},
+  {0x381e, 0x01},
+  {0x381f, 0x01},
+
+  // don't mirror, just flip
+  {0x3820, 0x04},
+
+  {0x3821, 0x19},
+  {0x3832, 0xF0},
+  {0x3834, 0xF0},
+  {0x384c, 0x02},
+  {0x384d, 0x0d},
+  {0x3850, 0x00},
+  {0x3851, 0x42},
+  {0x3852, 0x00},
+  {0x3853, 0x40},
+  {0x3858, 0x04},
+  {0x388c, 0x02},
+  {0x388d, 0x2b},
+
+  // APC
+  {0x3b40, 0x05}, {0x3b41, 0x40}, {0x3b42, 0x00}, {0x3b43, 0x90},
+  {0x3b44, 0x00}, {0x3b45, 0x20}, {0x3b46, 0x00}, {0x3b47, 0x20},
+  {0x3b48, 0x19}, {0x3b49, 0x12}, {0x3b4a, 0x16}, {0x3b4b, 0x2e},
+  {0x3b4c, 0x00}, {0x3b4d, 0x00},
+  {0x3b86, 0x00}, {0x3b87, 0x34}, {0x3b88, 0x00}, {0x3b89, 0x08},
+  {0x3b8a, 0x05}, {0x3b8b, 0x00}, {0x3b8c, 0x07}, {0x3b8d, 0x80},
+  {0x3b8e, 0x00}, {0x3b8f, 0x00}, {0x3b92, 0x05}, {0x3b93, 0x00},
+  {0x3b94, 0x07}, {0x3b95, 0x80}, {0x3b9e, 0x09},
+
+  // OTP
+  {0x3d82, 0x73},
+  {0x3d85, 0x05},
+  {0x3d8a, 0x03},
+  {0x3d8b, 0xff},
+  {0x3d99, 0x00},
+  {0x3d9a, 0x9f},
+  {0x3d9b, 0x00},
+  {0x3d9c, 0xa0},
+  {0x3da4, 0x00},
+  {0x3da7, 0x50},
+
+  // DTR
+  {0x420e, 0x6b},
+  {0x420f, 0x6e},
+  {0x4210, 0x06},
+  {0x4211, 0xc1},
+  {0x421e, 0x02},
+  {0x421f, 0x45},
+  {0x4220, 0xe1},
+  {0x4221, 0x01},
+  {0x4301, 0xff},
+  {0x4307, 0x03},
+  {0x4308, 0x13},
+  {0x430a, 0x13},
+  {0x430d, 0x93},
+  {0x430f, 0x57},
+  {0x4310, 0x95},
+  {0x4311, 0x16},
+  {0x4316, 0x00},
+
+  {0x4317, 0x38}, // both embedded rows are enabled
+
+  {0x4319, 0x03}, // spd dcg
+  {0x431a, 0x00}, // 8 bit mipi
+  {0x431b, 0x00},
+  {0x431d, 0x2a},
+  {0x431e, 0x11},
+
+  {0x431f, 0x20}, // enable PWL (pwl0_en), 12 bits
+  //{0x431f, 0x00}, // disable PWL
+
+  {0x4320, 0x19},
+  {0x4323, 0x80},
+  {0x4324, 0x00},
+  {0x4503, 0x4e},
+  {0x4505, 0x00},
+  {0x4509, 0x00},
+  {0x450a, 0x00},
+  {0x4580, 0xf8},
+  {0x4583, 0x07},
+  {0x4584, 0x6a},
+  {0x4585, 0x08},
+  {0x4586, 0x05},
+  {0x4587, 0x04},
+  {0x4588, 0x73},
+  {0x4589, 0x05},
+  {0x458a, 0x1f},
+  {0x458b, 0x02},
+  {0x458c, 0xdc},
+  {0x458d, 0x03},
+  {0x458e, 0x02},
+  {0x4597, 0x07},
+  {0x4598, 0x40},
+  {0x4599, 0x0e},
+  {0x459a, 0x0e},
+  {0x459b, 0xfb},
+  {0x459c, 0xf3},
+  {0x4602, 0x00},
+  {0x4603, 0x13},
+  {0x4604, 0x00},
+  {0x4609, 0x0a},
+  {0x460a, 0x30},
+  {0x4610, 0x00},
+  {0x4611, 0x70},
+  {0x4612, 0x01},
+  {0x4613, 0x00},
+  {0x4614, 0x00},
+  {0x4615, 0x70},
+  {0x4616, 0x01},
+  {0x4617, 0x00},
+
+  {0x4800, 0x04}, // invert output PCLK
+  {0x480a, 0x22},
+  {0x4813, 0xe4},
+
+  // mipi
+  {0x4814, 0x2a},
+  {0x4837, 0x0d},
+  {0x484b, 0x47},
+  {0x484f, 0x00},
+  {0x4887, 0x51},
+  {0x4d00, 0x4a},
+  {0x4d01, 0x18},
+  {0x4d05, 0xff},
+  {0x4d06, 0x88},
+  {0x4d08, 0x63},
+  {0x4d09, 0xdf},
+  {0x4d15, 0x7d},
+  {0x4d1a, 0x20},
+  {0x4d30, 0x0a},
+  {0x4d31, 0x00},
+  {0x4d34, 0x7d},
+  {0x4d3c, 0x7d},
+  {0x4f00, 0x00},
+  {0x4f01, 0x00},
+  {0x4f02, 0x00},
+  {0x4f03, 0x20},
+  {0x4f04, 0xe0},
+  {0x6a00, 0x00},
+  {0x6a01, 0x20},
+  {0x6a02, 0x00},
+  {0x6a03, 0x20},
+  {0x6a04, 0x02},
+  {0x6a05, 0x80},
+  {0x6a06, 0x01},
+  {0x6a07, 0xe0},
+  {0x6a08, 0xcf},
+  {0x6a09, 0x01},
+  {0x6a0a, 0x40},
+  {0x6a20, 0x00},
+  {0x6a21, 0x02},
+  {0x6a22, 0x00},
+  {0x6a23, 0x00},
+  {0x6a24, 0x00},
+  {0x6a25, 0x00},
+  {0x6a26, 0x00},
+  {0x6a27, 0x00},
+  {0x6a28, 0x00},
+
+  // isp
+  {0x5000, 0x8f},
+  {0x5001, 0x75},
+  {0x5002, 0x7f}, // PWL0
+  //{0x5002, 0x3f}, // PWL disable
+  {0x5003, 0x7a},
+
+  {0x5004, 0x3e},
+  {0x5005, 0x1e},
+  {0x5006, 0x1e},
+  {0x5007, 0x1e},
+
+  {0x5008, 0x00},
+  {0x500c, 0x00},
+  {0x502c, 0x00},
+  {0x502e, 0x00},
+  {0x502f, 0x00},
+  {0x504b, 0x00},
+  {0x5053, 0x00},
+  {0x505b, 0x00},
+  {0x5063, 0x00},
+  {0x5070, 0x00},
+  {0x5074, 0x04},
+  {0x507a, 0x04},
+  {0x507b, 0x09},
+  {0x5500, 0x02},
+  {0x5700, 0x02},
+  {0x5900, 0x02},
+  {0x6007, 0x04},
+  {0x6008, 0x05},
+  {0x6009, 0x02},
+  {0x600b, 0x08},
+  {0x600c, 0x07},
+  {0x600d, 0x88},
+  {0x6016, 0x00},
+  {0x6027, 0x04},
+  {0x6028, 0x05},
+  {0x6029, 0x02},
+  {0x602b, 0x08},
+  {0x602c, 0x07},
+  {0x602d, 0x88},
+  {0x6047, 0x04},
+  {0x6048, 0x05},
+  {0x6049, 0x02},
+  {0x604b, 0x08},
+  {0x604c, 0x07},
+  {0x604d, 0x88},
+  {0x6067, 0x04},
+  {0x6068, 0x05},
+  {0x6069, 0x02},
+  {0x606b, 0x08},
+  {0x606c, 0x07},
+  {0x606d, 0x88},
+  {0x6087, 0x04},
+  {0x6088, 0x05},
+  {0x6089, 0x02},
+  {0x608b, 0x08},
+  {0x608c, 0x07},
+  {0x608d, 0x88},
+
+  // 12-bit PWL0
+  {0x5e00, 0x00},
+
+  // m_ndX_exp[0:32]
+  // 9*2+0xa*3+0xb*2+0xc*2+0xd*2+0xe*2+0xf*2+0x10*2+0x11*2+0x12*4+0x13*3+0x14*3+0x15*3+0x16 = 518
+  {0x5e01, 0x09},
+  {0x5e02, 0x09},
+  {0x5e03, 0x0a},
+  {0x5e04, 0x0a},
+  {0x5e05, 0x0a},
+  {0x5e06, 0x0b},
+  {0x5e07, 0x0b},
+  {0x5e08, 0x0c},
+  {0x5e09, 0x0c},
+  {0x5e0a, 0x0d},
+  {0x5e0b, 0x0d},
+  {0x5e0c, 0x0e},
+  {0x5e0d, 0x0e},
+  {0x5e0e, 0x0f},
+  {0x5e0f, 0x0f},
+  {0x5e10, 0x10},
+  {0x5e11, 0x10},
+  {0x5e12, 0x11},
+  {0x5e13, 0x11},
+  {0x5e14, 0x12},
+  {0x5e15, 0x12},
+  {0x5e16, 0x12},
+  {0x5e17, 0x12},
+  {0x5e18, 0x13},
+  {0x5e19, 0x13},
+  {0x5e1a, 0x13},
+  {0x5e1b, 0x14},
+  {0x5e1c, 0x14},
+  {0x5e1d, 0x14},
+  {0x5e1e, 0x15},
+  {0x5e1f, 0x15},
+  {0x5e20, 0x15},
+  {0x5e21, 0x16},
+
+  // m_ndY_val[0:32]
+  // 0x200+0xff+0x100*3+0x80*12+0x40*16 = 4095
+  {0x5e22, 0x00}, {0x5e23, 0x02}, {0x5e24, 0x00},
+  {0x5e25, 0x00}, {0x5e26, 0x00}, {0x5e27, 0xff},
+  {0x5e28, 0x00}, {0x5e29, 0x01}, {0x5e2a, 0x00},
+  {0x5e2b, 0x00}, {0x5e2c, 0x01}, {0x5e2d, 0x00},
+  {0x5e2e, 0x00}, {0x5e2f, 0x01}, {0x5e30, 0x00},
+  {0x5e31, 0x00}, {0x5e32, 0x00}, {0x5e33, 0x80},
+  {0x5e34, 0x00}, {0x5e35, 0x00}, {0x5e36, 0x80},
+  {0x5e37, 0x00}, {0x5e38, 0x00}, {0x5e39, 0x80},
+  {0x5e3a, 0x00}, {0x5e3b, 0x00}, {0x5e3c, 0x80},
+  {0x5e3d, 0x00}, {0x5e3e, 0x00}, {0x5e3f, 0x80},
+  {0x5e40, 0x00}, {0x5e41, 0x00}, {0x5e42, 0x80},
+  {0x5e43, 0x00}, {0x5e44, 0x00}, {0x5e45, 0x80},
+  {0x5e46, 0x00}, {0x5e47, 0x00}, {0x5e48, 0x80},
+  {0x5e49, 0x00}, {0x5e4a, 0x00}, {0x5e4b, 0x80},
+  {0x5e4c, 0x00}, {0x5e4d, 0x00}, {0x5e4e, 0x80},
+  {0x5e4f, 0x00}, {0x5e50, 0x00}, {0x5e51, 0x80},
+  {0x5e52, 0x00}, {0x5e53, 0x00}, {0x5e54, 0x80},
+  {0x5e55, 0x00}, {0x5e56, 0x00}, {0x5e57, 0x40},
+  {0x5e58, 0x00}, {0x5e59, 0x00}, {0x5e5a, 0x40},
+  {0x5e5b, 0x00}, {0x5e5c, 0x00}, {0x5e5d, 0x40},
+  {0x5e5e, 0x00}, {0x5e5f, 0x00}, {0x5e60, 0x40},
+  {0x5e61, 0x00}, {0x5e62, 0x00}, {0x5e63, 0x40},
+  {0x5e64, 0x00}, {0x5e65, 0x00}, {0x5e66, 0x40},
+  {0x5e67, 0x00}, {0x5e68, 0x00}, {0x5e69, 0x40},
+  {0x5e6a, 0x00}, {0x5e6b, 0x00}, {0x5e6c, 0x40},
+  {0x5e6d, 0x00}, {0x5e6e, 0x00}, {0x5e6f, 0x40},
+  {0x5e70, 0x00}, {0x5e71, 0x00}, {0x5e72, 0x40},
+  {0x5e73, 0x00}, {0x5e74, 0x00}, {0x5e75, 0x40},
+  {0x5e76, 0x00}, {0x5e77, 0x00}, {0x5e78, 0x40},
+  {0x5e79, 0x00}, {0x5e7a, 0x00}, {0x5e7b, 0x40},
+  {0x5e7c, 0x00}, {0x5e7d, 0x00}, {0x5e7e, 0x40},
+  {0x5e7f, 0x00}, {0x5e80, 0x00}, {0x5e81, 0x40},
+  {0x5e82, 0x00}, {0x5e83, 0x00}, {0x5e84, 0x40},
+
+  // disable PWL
+  /*{0x5e01, 0x18}, {0x5e02, 0x00}, {0x5e03, 0x00}, {0x5e04, 0x00},
+  {0x5e05, 0x00}, {0x5e06, 0x00}, {0x5e07, 0x00}, {0x5e08, 0x00},
+  {0x5e09, 0x00}, {0x5e0a, 0x00}, {0x5e0b, 0x00}, {0x5e0c, 0x00},
+  {0x5e0d, 0x00}, {0x5e0e, 0x00}, {0x5e0f, 0x00}, {0x5e10, 0x00},
+  {0x5e11, 0x00}, {0x5e12, 0x00}, {0x5e13, 0x00}, {0x5e14, 0x00},
+  {0x5e15, 0x00}, {0x5e16, 0x00}, {0x5e17, 0x00}, {0x5e18, 0x00},
+  {0x5e19, 0x00}, {0x5e1a, 0x00}, {0x5e1b, 0x00}, {0x5e1c, 0x00},
+  {0x5e1d, 0x00}, {0x5e1e, 0x00}, {0x5e1f, 0x00}, {0x5e20, 0x00},
+  {0x5e21, 0x00},
+
+  {0x5e22, 0x00}, {0x5e23, 0x0f}, {0x5e24, 0xFF},*/
+
+  {0x4001, 0x2b},  // BLC_CTRL_1
+  {0x4008, 0x02}, {0x4009, 0x03},
+  {0x4018, 0x12},
+  {0x4022, 0x40},
+  {0x4023, 0x20},
+
+  // all black level targets are 0x40
+  {0x4026, 0x00}, {0x4027, 0x40},
+  {0x4028, 0x00}, {0x4029, 0x40},
+  {0x402a, 0x00}, {0x402b, 0x40},
+  {0x402c, 0x00}, {0x402d, 0x40},
+
+  {0x407e, 0xcc},
+  {0x407f, 0x18},
+  {0x4080, 0xff},
+  {0x4081, 0xff},
+  {0x4082, 0x01},
+  {0x4083, 0x53},
+  {0x4084, 0x01},
+  {0x4085, 0x2b},
+  {0x4086, 0x00},
+  {0x4087, 0xb3},
+
+  {0x4640, 0x40},
+  {0x4641, 0x11},
+  {0x4642, 0x0e},
+  {0x4643, 0xee},
+  {0x4646, 0x0f},
+  {0x4648, 0x00},
+  {0x4649, 0x03},
+
+  {0x4f00, 0x00},
+  {0x4f01, 0x00},
+  {0x4f02, 0x80},
+  {0x4f03, 0x2c},
+  {0x4f04, 0xf8},
+
+  {0x4d09, 0xff},
+  {0x4d09, 0xdf},
+
+  {0x5003, 0x7a},
+  {0x5b80, 0x08},
+  {0x5c00, 0x08},
+  {0x5c80, 0x00},
+  {0x5bbe, 0x12},
+  {0x5c3e, 0x12},
+  {0x5cbe, 0x12},
+  {0x5b8a, 0x80},
+  {0x5b8b, 0x80},
+  {0x5b8c, 0x80},
+  {0x5b8d, 0x80},
+  {0x5b8e, 0x60},
+  {0x5b8f, 0x80},
+  {0x5b90, 0x80},
+  {0x5b91, 0x80},
+  {0x5b92, 0x80},
+  {0x5b93, 0x20},
+  {0x5b94, 0x80},
+  {0x5b95, 0x80},
+  {0x5b96, 0x80},
+  {0x5b97, 0x20},
+  {0x5b98, 0x00},
+  {0x5b99, 0x80},
+  {0x5b9a, 0x40},
+  {0x5b9b, 0x20},
+  {0x5b9c, 0x00},
+  {0x5b9d, 0x00},
+  {0x5b9e, 0x80},
+  {0x5b9f, 0x00},
+  {0x5ba0, 0x00},
+  {0x5ba1, 0x00},
+  {0x5ba2, 0x00},
+  {0x5ba3, 0x00},
+  {0x5ba4, 0x00},
+  {0x5ba5, 0x00},
+  {0x5ba6, 0x00},
+  {0x5ba7, 0x00},
+  {0x5ba8, 0x02},
+  {0x5ba9, 0x00},
+  {0x5baa, 0x02},
+  {0x5bab, 0x76},
+  {0x5bac, 0x03},
+  {0x5bad, 0x08},
+  {0x5bae, 0x00},
+  {0x5baf, 0x80},
+  {0x5bb0, 0x00},
+  {0x5bb1, 0xc0},
+  {0x5bb2, 0x01},
+  {0x5bb3, 0x00},
+
+  // m_nNormCombineWeight
+  {0x5c0a, 0x80}, {0x5c0b, 0x80}, {0x5c0c, 0x80}, {0x5c0d, 0x80}, {0x5c0e, 0x60},
+  {0x5c0f, 0x80}, {0x5c10, 0x80}, {0x5c11, 0x80}, {0x5c12, 0x60}, {0x5c13, 0x20},
+  {0x5c14, 0x80}, {0x5c15, 0x80}, {0x5c16, 0x80}, {0x5c17, 0x20}, {0x5c18, 0x00},
+  {0x5c19, 0x80}, {0x5c1a, 0x40}, {0x5c1b, 0x20}, {0x5c1c, 0x00}, {0x5c1d, 0x00},
+  {0x5c1e, 0x80}, {0x5c1f, 0x00}, {0x5c20, 0x00}, {0x5c21, 0x00}, {0x5c22, 0x00},
+  {0x5c23, 0x00}, {0x5c24, 0x00}, {0x5c25, 0x00}, {0x5c26, 0x00}, {0x5c27, 0x00},
+
+  // m_nCombinThreL
+  {0x5c28, 0x02}, {0x5c29, 0x00},
+  {0x5c2a, 0x02}, {0x5c2b, 0x76},
+  {0x5c2c, 0x03}, {0x5c2d, 0x08},
+
+  // m_nCombinThreS
+  {0x5c2e, 0x00}, {0x5c2f, 0x80},
+  {0x5c30, 0x00}, {0x5c31, 0xc0},
+  {0x5c32, 0x01}, {0x5c33, 0x00},
+
+  // m_nNormCombineWeight
+  {0x5c8a, 0x80}, {0x5c8b, 0x80}, {0x5c8c, 0x80}, {0x5c8d, 0x80}, {0x5c8e, 0x80},
+  {0x5c8f, 0x80}, {0x5c90, 0x80}, {0x5c91, 0x80}, {0x5c92, 0x80}, {0x5c93, 0x60},
+  {0x5c94, 0x80}, {0x5c95, 0x80}, {0x5c96, 0x80}, {0x5c97, 0x60}, {0x5c98, 0x40},
+  {0x5c99, 0x80}, {0x5c9a, 0x80}, {0x5c9b, 0x80}, {0x5c9c, 0x40}, {0x5c9d, 0x00},
+  {0x5c9e, 0x80}, {0x5c9f, 0x80}, {0x5ca0, 0x80}, {0x5ca1, 0x20}, {0x5ca2, 0x00},
+  {0x5ca3, 0x80}, {0x5ca4, 0x80}, {0x5ca5, 0x00}, {0x5ca6, 0x00}, {0x5ca7, 0x00},
+
+  {0x5ca8, 0x01}, {0x5ca9, 0x00},
+  {0x5caa, 0x02}, {0x5cab, 0x00},
+  {0x5cac, 0x03}, {0x5cad, 0x08},
+
+  {0x5cae, 0x01}, {0x5caf, 0x00},
+  {0x5cb0, 0x02}, {0x5cb1, 0x00},
+  {0x5cb2, 0x03}, {0x5cb3, 0x08},
+
+  // combine ISP
+  {0x5be7, 0x80},
+  {0x5bc9, 0x80},
+  {0x5bca, 0x80},
+  {0x5bcb, 0x80},
+  {0x5bcc, 0x80},
+  {0x5bcd, 0x80},
+  {0x5bce, 0x80},
+  {0x5bcf, 0x80},
+  {0x5bd0, 0x80},
+  {0x5bd1, 0x80},
+  {0x5bd2, 0x20},
+  {0x5bd3, 0x80},
+  {0x5bd4, 0x40},
+  {0x5bd5, 0x20},
+  {0x5bd6, 0x00},
+  {0x5bd7, 0x00},
+  {0x5bd8, 0x00},
+  {0x5bd9, 0x00},
+  {0x5bda, 0x00},
+  {0x5bdb, 0x00},
+  {0x5bdc, 0x00},
+  {0x5bdd, 0x00},
+  {0x5bde, 0x00},
+  {0x5bdf, 0x00},
+  {0x5be0, 0x00},
+  {0x5be1, 0x00},
+  {0x5be2, 0x00},
+  {0x5be3, 0x00},
+  {0x5be4, 0x00},
+  {0x5be5, 0x00},
+  {0x5be6, 0x00},
+
+  // m_nSPDCombineWeight
+  {0x5c49, 0x80}, {0x5c4a, 0x80}, {0x5c4b, 0x80}, {0x5c4c, 0x80}, {0x5c4d, 0x40},
+  {0x5c4e, 0x80}, {0x5c4f, 0x80}, {0x5c50, 0x80}, {0x5c51, 0x60}, {0x5c52, 0x20},
+  {0x5c53, 0x80}, {0x5c54, 0x80}, {0x5c55, 0x80}, {0x5c56, 0x20}, {0x5c57, 0x00},
+  {0x5c58, 0x80}, {0x5c59, 0x40}, {0x5c5a, 0x20}, {0x5c5b, 0x00}, {0x5c5c, 0x00},
+  {0x5c5d, 0x80}, {0x5c5e, 0x00}, {0x5c5f, 0x00}, {0x5c60, 0x00}, {0x5c61, 0x00},
+  {0x5c62, 0x00}, {0x5c63, 0x00}, {0x5c64, 0x00}, {0x5c65, 0x00}, {0x5c66, 0x00},
+
+  // m_nSPDCombineWeight
+  {0x5cc9, 0x80}, {0x5cca, 0x80}, {0x5ccb, 0x80}, {0x5ccc, 0x80}, {0x5ccd, 0x80},
+  {0x5cce, 0x80}, {0x5ccf, 0x80}, {0x5cd0, 0x80}, {0x5cd1, 0x80}, {0x5cd2, 0x60},
+  {0x5cd3, 0x80}, {0x5cd4, 0x80}, {0x5cd5, 0x80}, {0x5cd6, 0x60}, {0x5cd7, 0x40},
+  {0x5cd8, 0x80}, {0x5cd9, 0x80}, {0x5cda, 0x80}, {0x5cdb, 0x40}, {0x5cdc, 0x20},
+  {0x5cdd, 0x80}, {0x5cde, 0x80}, {0x5cdf, 0x80}, {0x5ce0, 0x20}, {0x5ce1, 0x00},
+  {0x5ce2, 0x80}, {0x5ce3, 0x80}, {0x5ce4, 0x80}, {0x5ce5, 0x00}, {0x5ce6, 0x00},
+
+  {0x5d74, 0x01},
+  {0x5d75, 0x00},
+
+  {0x5d1f, 0x81},
+  {0x5d11, 0x00},
+  {0x5d12, 0x10},
+  {0x5d13, 0x10},
+  {0x5d15, 0x05},
+  {0x5d16, 0x05},
+  {0x5d17, 0x05},
+  {0x5d08, 0x03},
+  {0x5d09, 0xb6},
+  {0x5d0a, 0x03},
+  {0x5d0b, 0xb6},
+  {0x5d18, 0x03},
+  {0x5d19, 0xb6},
+  {0x5d62, 0x01},
+  {0x5d40, 0x02},
+  {0x5d41, 0x01},
+  {0x5d63, 0x1f},
+  {0x5d64, 0x00},
+  {0x5d65, 0x80},
+  {0x5d56, 0x00},
+  {0x5d57, 0x20},
+  {0x5d58, 0x00},
+  {0x5d59, 0x20},
+  {0x5d5a, 0x00},
+  {0x5d5b, 0x0c},
+  {0x5d5c, 0x02},
+  {0x5d5d, 0x40},
+  {0x5d5e, 0x02},
+  {0x5d5f, 0x40},
+  {0x5d60, 0x03},
+  {0x5d61, 0x40},
+  {0x5d4a, 0x02},
+  {0x5d4b, 0x40},
+  {0x5d4c, 0x02},
+  {0x5d4d, 0x40},
+  {0x5d4e, 0x02},
+  {0x5d4f, 0x40},
+  {0x5d50, 0x18},
+  {0x5d51, 0x80},
+  {0x5d52, 0x18},
+  {0x5d53, 0x80},
+  {0x5d54, 0x18},
+  {0x5d55, 0x80},
+  {0x5d46, 0x20},
+  {0x5d47, 0x00},
+  {0x5d48, 0x22},
+  {0x5d49, 0x00},
+  {0x5d42, 0x20},
+  {0x5d43, 0x00},
+  {0x5d44, 0x22},
+  {0x5d45, 0x00},
+
+  {0x5004, 0x1e},
+  {0x4221, 0x03},  // this is changed from 1 -> 3
+
+  // DCG exposure coarse
+  // {0x3501, 0x01}, {0x3502, 0xc8},
+  // SPD exposure coarse
+  // {0x3541, 0x01}, {0x3542, 0xc8},
+  // VS exposure coarse
+  // {0x35c1, 0x00}, {0x35c2, 0x01},
+
+  // crc reference
+  {0x420e, 0x66}, {0x420f, 0x5d}, {0x4210, 0xa8}, {0x4211, 0x55},
+  // crc stat check
+  {0x507a, 0x5f}, {0x507b, 0x46},
+
+  // watchdog control
+  {0x4f00, 0x00}, {0x4f01, 0x01}, {0x4f02, 0x80}, {0x4f04, 0x2c},
+
+  // color balance gains
+  // blue
+  {0x5280, 0x06}, {0x5281, 0xCB}, // hcg
+  {0x5480, 0x06}, {0x5481, 0xCB}, // lcg
+  {0x5680, 0x06}, {0x5681, 0xCB}, // spd
+  {0x5880, 0x06}, {0x5881, 0xCB}, // vs
+
+  // green(blue)
+  {0x5282, 0x04}, {0x5283, 0x00},
+  {0x5482, 0x04}, {0x5483, 0x00},
+  {0x5682, 0x04}, {0x5683, 0x00},
+  {0x5882, 0x04}, {0x5883, 0x00},
+
+  // green(red)
+  {0x5284, 0x04}, {0x5285, 0x00},
+  {0x5484, 0x04}, {0x5485, 0x00},
+  {0x5684, 0x04}, {0x5685, 0x00},
+  {0x5884, 0x04}, {0x5885, 0x00},
+
+  // red
+  {0x5286, 0x08}, {0x5287, 0xDE},
+  {0x5486, 0x08}, {0x5487, 0xDE},
+  {0x5686, 0x08}, {0x5687, 0xDE},
+  {0x5886, 0x08}, {0x5887, 0xDE},
+
+  // fixed gains
+  {0x3588, 0x01}, {0x3589, 0x00},
+  {0x35c8, 0x01}, {0x35c9, 0x00},
+  {0x3548, 0x0F}, {0x3549, 0x00},
+  {0x35c1, 0x00},
+};
@@ -0,0 +1,104 @@
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <map>
+#include <utility>
+#include <vector>
+
+#include "media/cam_isp.h"
+#include "media/cam_sensor.h"
+
+#include "openpilot/cereal/gen/cpp/log.capnp.h"
+#include "system/camerad/sensors/ox03c10_registers.h"
+#include "system/camerad/sensors/os04c10_registers.h"
+
+#define ANALOG_GAIN_MAX_CNT 55
+
+class SensorInfo {
+public:
+  SensorInfo() = default;
+  virtual std::vector<i2c_random_wr_payload> getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const { return {}; }
+  virtual float getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const {return 0; }
+  virtual int getSlaveAddress(int port) const { assert(0); }
+
+  cereal::FrameData::ImageSensor image_sensor = cereal::FrameData::ImageSensor::UNKNOWN;
+  float pixel_size_mm;
+  uint32_t frame_width, frame_height;
+  uint32_t frame_stride;
+  uint32_t frame_offset = 0;
+  uint32_t extra_height = 0;
+  int out_scale = 1;
+  int registers_offset = -1;
+  int stats_offset = -1;
+  int hdr_offset = -1;
+
+  int exposure_time_min;
+  int exposure_time_max;
+
+  float dc_gain_factor;
+  int dc_gain_min_weight;
+  int dc_gain_max_weight;
+  float dc_gain_on_grey;
+  float dc_gain_off_grey;
+
+  float ev_scale = 1.0;
+  float sensor_analog_gains[ANALOG_GAIN_MAX_CNT];
+  int analog_gain_min_idx;
+  int analog_gain_max_idx;
+  int analog_gain_rec_idx;
+  int analog_gain_cost_delta;
+  float analog_gain_cost_low;
+  float analog_gain_cost_high;
+  float target_grey_factor;
+  float min_ev;
+  float max_ev;
+
+  bool data_word;
+  uint32_t probe_reg_addr;
+  uint32_t probe_expected_data;
+  std::vector<i2c_random_wr_payload> start_reg_array;
+  std::vector<i2c_random_wr_payload> init_reg_array;
+
+  uint32_t bits_per_pixel;
+  uint32_t bayer_pattern;
+  uint32_t mipi_format;
+  uint32_t mclk_frequency;
+  uint32_t frame_data_type;
+
+  uint32_t readout_time_ns;  // used to recover EOF from SOF
+
+  // ISP image processing params
+  uint32_t black_level;
+  std::vector<uint32_t> color_correct_matrix;  // 3x3
+  std::vector<uint32_t> gamma_lut_rgb;         // gamma LUTs are length 64 * sizeof(uint32_t); same for r/g/b here
+  void prepare_gamma_lut() {
+    for (int i = 0; i < 64; i++) {
+      gamma_lut_rgb[i] |= ((uint32_t)(gamma_lut_rgb[i+1] - gamma_lut_rgb[i]) << 10);
+    }
+    gamma_lut_rgb.pop_back();
+  }
+  std::vector<uint32_t> linearization_lut;     // length 36
+  std::vector<uint32_t> linearization_pts;     // length 4
+  std::vector<uint32_t> vignetting_lut;        // length 221
+
+  const int num() const {
+    return static_cast<int>(image_sensor);
+  };
+};
+
+class OX03C10 : public SensorInfo {
+public:
+  OX03C10();
+  std::vector<i2c_random_wr_payload> getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const override;
+  float getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const override;
+  int getSlaveAddress(int port) const override;
+};
+
+class OS04C10 : public SensorInfo {
+public:
+  OS04C10();
+  std::vector<i2c_random_wr_payload> getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const override;
+  float getExposureScore(float desired_ev, int exp_t, int exp_g_idx, float exp_gain, int gain_idx) const override;
+  int getSlaveAddress(int port) const override;
+};
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import numpy as np
+
+import openpilot.cereal.messaging as messaging
+from openpilot.cereal.visionipc import VisionStreamType
+from msgq.visionipc import VisionIpcClient
+from openpilot.common.realtime import DT_MDL
+
+
+VISION_STREAMS = {
+  "narrowRoadCameraState": VisionStreamType.VISION_STREAM_NARROW_ROAD,
+  "cabinCameraState": VisionStreamType.VISION_STREAM_CABIN,
+  "wideRoadCameraState": VisionStreamType.VISION_STREAM_WIDE_ROAD,
+}
+
+
+def yuv_to_rgb(y, u, v):
+  ul = np.repeat(np.repeat(u, 2).reshape(u.shape[0], y.shape[1]), 2, axis=0).reshape(y.shape)
+  vl = np.repeat(np.repeat(v, 2).reshape(v.shape[0], y.shape[1]), 2, axis=0).reshape(y.shape)
+
+  yuv = np.dstack((y, ul, vl)).astype(np.int16)
+  yuv[:, :, 1:] -= 128
+
+  m = np.array([
+    [1.00000,  1.00000, 1.00000],
+    [0.00000, -0.39465, 2.03211],
+    [1.13983, -0.58060, 0.00000],
+  ])
+  rgb = np.dot(yuv, m).clip(0, 255)
+  return rgb.astype(np.uint8)
+
+
+def extract_image(buf):
+  # NV12 format: Y plane followed by interleaved UV plane
+  # UV plane size is stride * uv_height, where uv_height = align(height/2, 16)
+  uv_height = ((buf.height // 2) + 15) // 16 * 16
+  uv_plane_size = buf.stride * uv_height
+
+  y = np.array(buf.data[:buf.uv_offset], dtype=np.uint8).reshape((-1, buf.stride))[:buf.height, :buf.width]
+  uv_data = buf.data[buf.uv_offset:buf.uv_offset + uv_plane_size]
+  u = np.array(uv_data[::2], dtype=np.uint8).reshape((-1, buf.stride//2))[:buf.height//2, :buf.width//2]
+  v = np.array(uv_data[1::2], dtype=np.uint8).reshape((-1, buf.stride//2))[:buf.height//2, :buf.width//2]
+
+  return yuv_to_rgb(y, u, v)
+
+
+def get_snapshots(frame="narrowRoadCameraState", front_frame="cabinCameraState"):
+  sockets = [s for s in (frame, front_frame) if s is not None]
+  sm = messaging.SubMaster(sockets)
+  vipc_clients = {s: VisionIpcClient("camerad", VISION_STREAMS[s], True) for s in sockets}
+
+  # wait 4 sec from camerad startup for focus and exposure
+  while sm[sockets[0]].frameId < int(4. / DT_MDL):
+    sm.update()
+
+  for client in vipc_clients.values():
+    client.connect(True)
+
+  # grab images
+  rear, front = None, None
+  if frame is not None:
+    c = vipc_clients[frame]
+    rear = extract_image(c.recv())
+  if front_frame is not None:
+    c = vipc_clients[front_frame]
+    front = extract_image(c.recv())
+  return rear, front
@@ -0,0 +1,2 @@
+jpegs/
+test_ae_gray
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+#echo 4294967295 | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
+
+# no CCI and UTIL, very spammy
+echo 0xfffdbfff | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
+#echo 0 | sudo tee /sys/module/cam_debug_util/parameters/debug_mdl
+
+sudo dmesg -C
+scons -u --minimal .
+export DEBUG_FRAMES=1
+export DISABLE_ROAD=1 DISABLE_WIDE_ROAD=1
+#export DISABLE_DRIVER=1
+export LOGPRINT=debug
+./camerad
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+cd /sys/kernel/debug/tracing
+echo "" > trace
+echo 1 > tracing_on
+#echo Y > /sys/kernel/debug/camera_icp/a5_debug_q
+echo 0x1 > /sys/kernel/debug/camera_icp/a5_debug_type
+echo 1 > /sys/kernel/debug/tracing/events/camera/enable
+echo 0xffffffff > /sys/kernel/debug/camera_icp/a5_debug_lvl
+echo 1 > /sys/kernel/debug/tracing/events/camera/cam_icp_fw_dbg/enable
+
+cat /sys/kernel/debug/tracing/trace_pipe
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+DISABLE_ROAD=1 DISABLE_WIDE_ROAD=1 DEBUG_FRAMES=1 LOGPRINT=debug LD_PRELOAD=/data/tici_test_scripts/isp/interceptor/tmpioctl.so ./camerad
@@ -0,0 +1,9 @@
+#!/bin/sh
+cd ..
+while :; do
+  ./camerad &
+  pid="$!"
+  sleep 2
+  kill -2 $pid
+  wait $pid
+done
@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+
+import os
+import time
+import unittest
+import numpy as np
+
+from openpilot.common.parameterized import parameterized
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.cereal.services import SERVICE_LIST
+from openpilot.tools.lib.log_time_series import msgs_to_time_series
+from openpilot.system.camerad.snapshot import get_snapshots
+from openpilot.selfdrive.test.helpers import collect_logs, log_collector, processes_context
+
+TEST_TIMESPAN = 10
+CAMERAS = ('narrowRoadCameraState', 'cabinCameraState', 'wideRoadCameraState')
+EXPOSURE_STABLE_COUNT = 3
+EXPOSURE_RANGE = (0.15, 0.35)
+MAX_TEST_TIME = 25
+
+
+def _numpy_rgb2gray(im):
+  return np.clip(im[:,:,2] * 0.114 + im[:,:,1] * 0.587 + im[:,:,0] * 0.299, 0, 255).astype(np.uint8)
+
+def _exposure_stats(im):
+  h, w = im.shape[:2]
+  gray = _numpy_rgb2gray(im[h//10:9*h//10, w//10:9*w//10])
+  return float(np.median(gray) / 255.), float(np.mean(gray) / 255.)
+
+def _in_range(median, mean):
+  lo, hi = EXPOSURE_RANGE
+  return lo < median < hi and lo < mean < hi
+
+def _exposure_stable(results):
+  return all(
+    len(v) >= EXPOSURE_STABLE_COUNT and all(_in_range(*s) for s in v[-EXPOSURE_STABLE_COUNT:])
+    for v in results.values()
+  )
+
+
+def run_and_log(procs, services, duration):
+  with processes_context(procs):
+    return collect_logs(services, duration)
+
+def _camera_session():
+  """Single camerad session that collects logs and exposure data.
+     Runs until exposure stabilizes (min TEST_TIMESPAN seconds for enough log data)."""
+  with processes_context(["camerad"]), log_collector(CAMERAS) as (raw_logs, lock):
+    exposure = {cam: [] for cam in CAMERAS}
+    start = time.monotonic()
+    while time.monotonic() - start < MAX_TEST_TIME:
+      rpic, dpic = get_snapshots(frame="narrowRoadCameraState", front_frame="cabinCameraState")
+      wpic, _ = get_snapshots(frame="wideRoadCameraState")
+      for cam, img in zip(CAMERAS, [rpic, dpic, wpic], strict=True):
+        exposure[cam].append(_exposure_stats(img))
+
+      if time.monotonic() - start >= TEST_TIMESPAN and _exposure_stable(exposure):
+        break
+
+    elapsed = time.monotonic() - start
+
+  with lock:
+    ts = msgs_to_time_series(raw_logs)
+
+  for cam in CAMERAS:
+    expected_frames = SERVICE_LIST[cam].frequency * elapsed
+    cnt = len(ts[cam]['t'])
+    assert expected_frames*0.8 < cnt < expected_frames*1.2, f"unexpected frame count {cam}: {expected_frames=}, got {cnt}"
+
+    dts = np.abs(np.diff([ts[cam]['timestampSof']/1e6]) - 1000/SERVICE_LIST[cam].frequency)
+    assert (dts < 1.0).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}"
+
+  return ts, exposure
+
+class TestCamerad(OpenpilotTestCase):
+  COMMA_HARDWARE_TEST = True
+
+  @classmethod
+  def setUpClass(cls):
+    super().setUpClass()
+    cls.logs, cls.exposure_data = _camera_session()
+
+  @parameterized.expand(CAMERAS, names=("cam",))
+  def test_camera_exposure(self, cam):
+    lo, hi = EXPOSURE_RANGE
+    checks = self.exposure_data[cam]
+    assert len(checks) >= EXPOSURE_STABLE_COUNT, f"{cam}: only got {len(checks)} samples"
+
+    # check that exposure converges into the valid range
+    passed = sum(_in_range(med, mean) for med, mean in checks)
+    assert passed >= EXPOSURE_STABLE_COUNT, \
+      f"{cam}: only {passed}/{len(checks)} checks in range. " + \
+      " | ".join(f"#{i+1}: med={m:.4f} mean={u:.4f}" for i, (m, u) in enumerate(checks))
+
+    # check that exposure is stable once converged (no regressions)
+    in_range = False
+    for i, (median, mean) in enumerate(checks):
+      ok = _in_range(median, mean)
+      if in_range and not ok:
+        self.fail(f"{cam}: exposure regressed on sample {i+1} " +
+                    f"(median={median:.4f}, mean={mean:.4f}, expected: ({lo}, {hi}))")
+      in_range = ok
+
+  def test_frame_skips(self):
+    for c in CAMERAS:
+      assert set(np.diff(self.logs[c]['frameId'])) == {1, }, f"{c} has frame skips"
+
+  def test_frame_sync(self):
+    SYNCED_CAMS = ('narrowRoadCameraState', 'wideRoadCameraState')
+    n = range(len(self.logs['narrowRoadCameraState']['t'][:-10]))
+
+    frame_ids = {i: [self.logs[cam]['frameId'][i] for cam in CAMERAS] for i in n}
+    assert all(len(set(v)) == 1 for v in frame_ids.values()), "frame IDs not aligned"
+
+    # road and wide cameras should be synced within 1.1ms
+    synced_times = {i: [self.logs[cam]['timestampSof'][i] for cam in SYNCED_CAMS] for i in n}
+    diffs = {i: (max(ts) - min(ts))/1e6 for i, ts in synced_times.items()}
+    laggy_frames = {k: v for k, v in diffs.items() if v > 1.1}
+    assert len(laggy_frames) == 0, f"Frames not synced properly: {laggy_frames=}"
+
+    # cabin camera should be staggered ~25ms from road camera
+    for i in n:
+      offset_ms = abs(self.logs['cabinCameraState']['timestampSof'][i] - self.logs['narrowRoadCameraState']['timestampSof'][i]) / 1e6
+      assert 20 < offset_ms < 30, f"cabin camera stagger out of range at frame {i}: {offset_ms:.1f}ms (expected ~25ms)"
+
+  def test_sanity_checks(self):
+    self._sanity_checks(self.logs)
+
+  def _sanity_checks(self, ts):
+    for c in CAMERAS:
+      assert c in ts
+      assert len(ts[c]['t']) > 20
+
+      # not a valid request id
+      assert 0 not in ts[c]['requestId']
+
+      # should monotonically increase
+      assert np.all(np.diff(ts[c]['frameId']) >= 1)
+      assert np.all(np.diff(ts[c]['requestId']) >= 1)
+
+      # EOF > SOF
+      assert np.all((ts[c]['timestampEof'] - ts[c]['timestampSof']) > 0)
+
+      # logMonoTime > SOF
+      assert np.all((ts[c]['t'] - ts[c]['timestampSof']/1e9) > 1e-7)
+
+      # logMonoTime > EOF, needs some tolerance since EOF is (SOF + readout time) but there is noise in the SOF timestamping (done via IRQ)
+      assert np.mean((ts[c]['t'] - ts[c]['timestampEof']/1e9) > 1e-7) > 0.7  # should be mostly logMonoTime > EOF
+      assert np.all((ts[c]['t'] - ts[c]['timestampEof']/1e9) > -0.10)        # when EOF > logMonoTime, it should never be more than two frames
+
+  def test_stress_test(self):
+    os.environ['SPECTRA_ERROR_PROB'] = '0.008'
+    try:
+      logs = run_and_log(["camerad", ], CAMERAS, 10)
+    finally:
+      del os.environ['SPECTRA_ERROR_PROB']
+    ts = msgs_to_time_series(logs)
+
+    # we should see some jumps from introduced errors
+    assert np.max([ np.max(np.diff(ts[c]['frameId'])) for c in CAMERAS ]) > 1
+    assert np.max([ np.max(np.diff(ts[c]['requestId'])) for c in CAMERAS ]) > 1
+
+    self._sanity_checks(ts)
+
+
+if __name__ == "__main__":
+  unittest.main()
@@ -0,0 +1,24 @@
+# Run openpilot with webcam on PC
+
+## Setup openpilot
+- Follow [this readme](/tools/README.md) to install and build the requirements
+
+## Connect the hardware
+- Connect the camera first
+- Connect your computer to panda
+
+## GO
+
+```
+USE_WEBCAM=1 openpilot/system/manager/manager.py
+```
+- Start the car, then the UI should show the road webcam's view
+- Adjust and secure the webcam
+- Finish calibration and engage!
+
+## Specify Cameras
+
+Use the `ROAD_CAM` (default 0) and optional `DRIVER_CAM`, `WIDE_CAM` environment variables to specify which camera is which (ie. `ROAD_CAM=1` uses `/dev/video1`, on Ubuntu, for the road camera):
+```
+USE_WEBCAM=1 ROAD_CAM=1 openpilot/system/manager/manager.py
+```
@@ -0,0 +1,39 @@
+import av
+import cv2 as cv
+
+class Camera:
+  def __init__(self, cam_type_state, stream_type, camera_id):
+    try:
+      camera_id = int(camera_id)
+    except ValueError: # allow strings, ex: /dev/video0
+      pass
+    self.cam_type_state = cam_type_state
+    self.stream_type = stream_type
+    self.cur_frame_id = 0
+
+    print(f"Opening {cam_type_state} at {camera_id}")
+
+    self.cap = cv.VideoCapture(camera_id)
+
+    self.cap.set(cv.CAP_PROP_FRAME_WIDTH, 1280.0)
+    self.cap.set(cv.CAP_PROP_FRAME_HEIGHT, 720.0)
+    self.cap.set(cv.CAP_PROP_FPS, 25.0)
+
+    self.W = self.cap.get(cv.CAP_PROP_FRAME_WIDTH)
+    self.H = self.cap.get(cv.CAP_PROP_FRAME_HEIGHT)
+
+  @classmethod
+  def bgr2nv12(self, bgr):
+    frame = av.VideoFrame.from_ndarray(bgr, format='bgr24')
+    return frame.reformat(format='nv12').to_ndarray()
+
+  def read_frames(self):
+    while True:
+      ret, frame = self.cap.read()
+      if not ret:
+        break
+      # Rotate the frame 180 degrees (flip both axes)
+      frame = cv.flip(frame, -1)
+      yuv = Camera.bgr2nv12(frame)
+      yield yuv.data.tobytes()
+    self.cap.release()
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+import threading
+import os
+import platform
+from collections import namedtuple
+
+from openpilot.cereal.visionipc import VisionStreamType
+from msgq.visionipc import VisionIpcServer
+from openpilot.cereal import messaging
+
+from openpilot.system.camerad.webcam.camera import Camera
+from openpilot.common.realtime import Ratekeeper
+
+NARROW_ROAD_CAM = os.getenv("NARROW_ROAD_CAM", os.getenv("ROAD_CAM", "0"))
+WIDE_CAM = os.getenv("WIDE_CAM")
+DRIVER_CAM = os.getenv("DRIVER_CAM")
+
+CameraType = namedtuple("CameraType", ["msg_name", "stream_type", "cam_id"])
+
+CAMERAS = [
+  CameraType("narrowRoadCameraState", VisionStreamType.VISION_STREAM_NARROW_ROAD, NARROW_ROAD_CAM)
+]
+if WIDE_CAM:
+  CAMERAS.append(CameraType("wideRoadCameraState", VisionStreamType.VISION_STREAM_WIDE_ROAD, WIDE_CAM))
+if DRIVER_CAM:
+  CAMERAS.append(CameraType("cabinCameraState", VisionStreamType.VISION_STREAM_CABIN, DRIVER_CAM))
+
+class Camerad:
+  def __init__(self):
+    self.pm = messaging.PubMaster([c.msg_name for c in CAMERAS])
+    self.vipc_server = VisionIpcServer("camerad")
+
+    self.cameras = []
+    for c in CAMERAS:
+      cam_device = f"/dev/video{c.cam_id}" if platform.system() != "Darwin" else c.cam_id
+      cam = Camera(c.msg_name, c.stream_type, cam_device)
+      self.cameras.append(cam)
+      self.vipc_server.create_buffers(c.stream_type, 20, cam.W, cam.H)
+
+    self.vipc_server.start_listener()
+
+  def _send_yuv(self, yuv, frame_id, pub_type, yuv_type):
+    eof = int(frame_id * 0.05 * 1e9)
+    self.vipc_server.send(yuv_type, yuv, frame_id, eof, eof)
+    dat = messaging.new_message(pub_type, valid=True)
+    msg = {
+      "frameId": frame_id,
+      "transform": [1.0, 0.0, 0.0,
+                    0.0, 1.0, 0.0,
+                    0.0, 0.0, 1.0]
+    }
+    setattr(dat, pub_type, msg)
+    self.pm.send(pub_type, dat)
+
+  def camera_runner(self, cam):
+    rk = Ratekeeper(20, None)
+    for yuv in cam.read_frames():
+      self._send_yuv(yuv, cam.cur_frame_id, cam.cam_type_state, cam.stream_type)
+      cam.cur_frame_id += 1
+      rk.keep_time()
+
+  def run(self):
+    threads = []
+    for cam in self.cameras:
+      cam_thread = threading.Thread(target=self.camera_runner, args=(cam,))
+      cam_thread.start()
+      threads.append(cam_thread)
+
+    for t in threads:
+      t.join()
+
+
+def main():
+  camerad = Camerad()
+  camerad.run()
+
+
+if __name__ == "__main__":
+  main()
@@ -0,0 +1,581 @@
+#!/usr/bin/env python3
+"""chestnut (ASM2464) SPI flasher using data-USB EP0 control transfers."""
+import argparse
+import ctypes
+import errno
+import fcntl
+import glob
+import hashlib
+import os
+import re
+import signal
+import struct
+import sys
+import time
+import zlib
+from pathlib import Path
+
+VID_PIDS = (("add1", "0001"), ("3801", "0001"))
+ROM_VID_PIDS = (("174c", "2464"), ("174c", "2463"))
+ROM_PRODUCT = "USB 3.2 PCIe TinyEnclosure"
+FIRMWARE_PATH = Path(__file__).with_name("firmware_wrapped.bin")
+CONFIG_DIR = "/data/chestnut_config"
+PM_PATHS = ("/sys/bus/platform/devices/a600000.ssusb", "/sys/bus/usb/devices/usb4")
+VBUS_PATH = "/sys/kernel/debug/regulator/smb2-vbus/enable"
+IMAGE_OFFSET = 0x100
+SECTOR, PAGE = 4096, 128
+MAX_CODE_SIZE = 0x10000
+FLASH_BUDGET = 600.0
+USBDEVFS_CONTROL = 0xC0185500
+USBDEVFS_BULK = 0xC0185502
+USBDEVFS_SETINTERFACE = 0x80085504
+USBDEVFS_SETCONFIGURATION = 0x80045505
+USBDEVFS_CLAIMINTERFACE = 0x8004550F
+USBDEVFS_RESET = 0x5514
+USBDEVFS_CLEAR_HALT = 0x80045515
+MAX_REGISTER_READ_SIZE = 255
+
+_deadline = float("inf")
+
+
+def check_budget():
+  if time.monotonic() > _deadline:
+    raise TimeoutError(f"flash did not converge within {FLASH_BUDGET:g}s")
+
+
+class Ctrl(ctypes.Structure):
+  _fields_ = [("request_type", ctypes.c_uint8), ("request", ctypes.c_uint8),
+              ("value", ctypes.c_uint16), ("index", ctypes.c_uint16),
+              ("length", ctypes.c_uint16), ("timeout", ctypes.c_uint32),
+              ("data", ctypes.c_void_p)]
+
+
+class Bulk(ctypes.Structure):
+  _fields_ = [("ep", ctypes.c_uint), ("len", ctypes.c_uint),
+              ("timeout", ctypes.c_uint), ("data", ctypes.c_void_p)]
+
+
+class RomFallback(Exception):
+  pass
+
+
+def find_chestnut():
+  found = []
+  for d in glob.glob("/sys/bus/usb/devices/*"):
+    try:
+      vid_pid = (open(d + "/idVendor").read().strip(), open(d + "/idProduct").read().strip())
+      if vid_pid in VID_PIDS + ROM_VID_PIDS:
+        found.append((d, vid_pid, open(d + "/product").read().strip()))
+    except OSError:
+      pass
+  if len(found) > 1:
+    raise RuntimeError(f"expected one chestnut, found {len(found)}")
+  return found[0] if found else (None, None, None)
+
+
+def in_rom_bootloader(vid_pid, product):
+  # the ROM bootloader reports the config page strings, or its own when the config page is lost
+  return vid_pid in ROM_VID_PIDS or product == ROM_PRODUCT or (product or "").startswith("AS2462")
+
+
+def disable_runtime_pm(path):
+  control = os.path.join(path, "power/control")
+  if not os.path.exists(control):
+    return
+  with open(control, "w") as f:
+    f.write("on\n")
+  if open(control).read().strip() != "on":
+    raise RuntimeError(f"could not disable USB runtime PM: {control}")
+  delay = os.path.join(path, "power/autosuspend_delay_ms")
+  if os.path.exists(delay):
+    with open(delay, "w") as f:
+      f.write("-1\n")
+
+
+def unbind_drivers(path):
+  for interface in glob.glob(path + ":*"):
+    driver = interface + "/driver"
+    if os.path.islink(driver):
+      with open(os.path.realpath(driver) + "/unbind", "w") as f:
+        f.write(os.path.basename(interface))
+
+
+def open_device(path):
+  bus, dev = int(open(path + "/busnum").read()), int(open(path + "/devnum").read())
+  return os.open(f"/dev/bus/usb/{bus:03d}/{dev:03d}", os.O_RDWR)
+
+
+def link_up() -> bool:
+  # asm enumerates on USB-C alone, gpu is only usable once pcie link is up
+  try:
+    path, _, _ = find_chestnut()
+    if path is None:
+      return False
+    fd = open_device(path)
+  except (OSError, RuntimeError):
+    return False
+  try:
+    fcntl.ioctl(fd, USBDEVFS_CONTROL, Ctrl(0x40, 0xF3, 1, 0, 0, 2000, None))
+    buf = (ctypes.c_ubyte * 1)()
+    fcntl.ioctl(fd, USBDEVFS_CONTROL, Ctrl(0xC0, 0xE4, 0xB450, 0, 1, 1000, ctypes.cast(buf, ctypes.c_void_p)))
+    return buf[0] == 0x78  # LTSSM L0
+  except OSError:
+    return False
+  finally:
+    os.close(fd)
+
+
+def claim_interface(path, setup=False):
+  # unbind usb-storage, which binds to the ROM bootloader
+  disable_runtime_pm(path)
+  unbind_drivers(path)
+  fd = open_device(path)
+  try:
+    if setup:
+      fcntl.ioctl(fd, USBDEVFS_SETCONFIGURATION, struct.pack("I", 1))
+    fcntl.ioctl(fd, USBDEVFS_CLAIMINTERFACE, struct.pack("I", 0))
+    if setup:
+      fcntl.ioctl(fd, USBDEVFS_SETINTERFACE, struct.pack("II", 0, 0))
+  except OSError as e:
+    os.close(fd)
+    if e.errno == errno.EBUSY:
+      raise RuntimeError("chestnut is in use, stop modeld/GPU processes before flashing") from e
+    raise
+  return fd
+
+
+class Flash:
+  def __init__(self):
+    self.fd = -1
+    self.max_register_read_size = MAX_REGISTER_READ_SIZE
+
+  def close(self):
+    if self.fd >= 0:
+      os.close(self.fd)
+      self.fd = -1
+
+  def connect(self, timeout=5.0):
+    self.close()
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+      path, vid_pid, product = find_chestnut()
+      if in_rom_bootloader(vid_pid, product):
+        raise RomFallback("chestnut fell back to the ROM bootloader")
+      if path is not None:
+        speed = int(open(path + "/speed").read())
+        # USB2 firmware truncates larger reads to one full packet without a terminating ZLP.
+        self.max_register_read_size = 64 if speed < 5000 else MAX_REGISTER_READ_SIZE
+        self.fd = claim_interface(path)
+        return
+      time.sleep(0.1)
+    raise RuntimeError(f"chestnut did not enumerate within {timeout:g}s")
+
+  def reg_write(self, addr, value):
+    fcntl.ioctl(self.fd, USBDEVFS_CONTROL,
+                Ctrl(0x40, 0xE5, addr & 0xFFFF, value & 0xFFFF, 0, 2000, None))
+
+  def reg_read(self, addr, length=1):
+    buf = (ctypes.c_ubyte * length)()
+    fcntl.ioctl(self.fd, USBDEVFS_CONTROL,
+                Ctrl(0xC0, 0xE4, addr & 0xFFFF, 0, length, 2000, ctypes.cast(buf, ctypes.c_void_p)))
+    return bytes(buf)
+
+  def write_buffer(self, data):
+    for i, value in enumerate(data):
+      self.reg_write(0x7000 + i, value)
+
+  def wait_controller(self, timeout=2.0):
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+      if not self.reg_read(0xC8A9)[0] & 1:
+        return
+    raise TimeoutError("flash controller timeout")
+
+  def transaction(self, command, addr=0, length=0, addr_len=0x07, mode=0):
+    for reg, value in ((0xC8AD, mode), (0xC8AE, 0), (0xC8AF, 0), (0xC8AA, command), (0xC8AC, addr_len),
+                       (0xC8A1, addr), (0xC8A2, addr >> 8), (0xC8AB, addr >> 16), (0xC8A3, length >> 8), (0xC8A4, length)):
+      self.reg_write(reg, value & 0xFF)
+    self.reg_write(0xC8A9, 1)
+    self.wait_controller()
+    for _ in range(4):
+      self.reg_write(0xC8AD, 0)
+
+  def write_enable(self):
+    for reg, value in ((0xC8AD, 0), (0xC8AA, 0x06), (0xC8AC, 0x04), (0xC8A3, 0), (0xC8A4, 0), (0xC8A9, 1)):
+      self.reg_write(reg, value)
+    self.wait_controller()
+
+  def status(self):
+    self.transaction(0x05, length=1, addr_len=0x04)
+    return self.reg_read(0x7000)[0]
+
+  def wait_write_done(self, timeout=10.0):
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+      if not self.status() & 1:
+        return
+      time.sleep(0.005)
+    raise TimeoutError("SPI flash WIP timeout")
+
+  def init(self):
+    self.reg_write(0xCC33, 0x04)
+    self.reg_write(0xCA81, self.reg_read(0xCA81)[0] | 1)
+    self.reg_write(0xC805, 0x02)
+    self.reg_write(0xC8A6, 0x04)
+    for _ in range(5):
+      self.write_enable()
+      self.write_buffer(bytes(4))
+      self.transaction(0x01, length=1, addr_len=0x04, mode=1)
+      time.sleep(0.01)
+      if not self.status() & 0x1C:
+        return
+    raise RuntimeError("could not clear SPI block protection")
+
+  def read(self, addr, length):
+    out = bytearray()
+    while len(out) < length:
+      n = min(4096, length - len(out))
+      self.transaction(0x03, addr + len(out), max(4096, n))
+      for off in range(0, n, self.max_register_read_size):
+        out += self.reg_read(0x7000 + off, min(self.max_register_read_size, n - off))
+    return bytes(out)
+
+  def erase_sector(self, addr):
+    self.write_enable()
+    self.transaction(0x20, addr)
+    self.wait_write_done()
+
+  def program(self, addr, data):
+    self.write_buffer(data + bytes((-len(data)) % 4))
+    self.write_enable()
+    self.transaction(0x02, addr, len(data), mode=1)
+    self.wait_write_done()
+
+
+def validate_image(data):
+  if len(data) < 10:
+    raise ValueError("wrapped firmware is too short")
+  body_len = int.from_bytes(data[:4], "little")
+  if body_len > MAX_CODE_SIZE:
+    raise ValueError(f"wrapped firmware body exceeds {MAX_CODE_SIZE} bytes")
+  if len(data) != body_len + 10 or data[4 + body_len] != 0xA5:
+    raise ValueError("invalid wrapped firmware length or magic")
+  body = data[4:4 + body_len]
+  if data[5 + body_len] != sum(body) & 0xFF:
+    raise ValueError("invalid wrapped firmware checksum")
+  if data[6 + body_len:] != zlib.crc32(body).to_bytes(4, "little"):
+    raise ValueError("invalid wrapped firmware CRC")
+
+
+def image_product(image):
+  match = re.search(rb"custom [0-9a-f]{8}-CLEAN", image)
+  if match is None:
+    raise ValueError("no product string in wrapped firmware")
+  return match.group().decode()
+
+
+def reconnect(flash):
+  attempt = 0
+  while True:
+    attempt += 1
+    check_budget()
+    try:
+      flash.connect()
+      flash.init()
+      return
+    except (OSError, TimeoutError, RuntimeError) as e:
+      print(f"waiting for chestnut (attempt {attempt}): {e}", flush=True)
+      time.sleep(1)
+
+
+def with_retries(flash, label, operation):
+  # on any transfer error, reconnect and restart the operation
+  attempt = 0
+  while True:
+    attempt += 1
+    try:
+      return operation()
+    except (OSError, TimeoutError, RuntimeError) as e:
+      check_budget()
+      print(f"{label} attempt {attempt}: {e}", flush=True)
+      reconnect(flash)
+
+
+def stable_read(flash, addr, length, count=2):
+  def read():
+    reads = [flash.read(addr, length) for _ in range(count)]
+    if any(x != reads[0] for x in reads[1:]):
+      raise RuntimeError(f"unstable flash read at 0x{addr:05x}")
+    return reads[0]
+  return with_retries(flash, f"read 0x{addr:05x}", read)
+
+
+def program_sector(flash, addr, target):
+  def program():
+    flash.erase_sector(addr)
+    if flash.read(addr, SECTOR) != bytes([0xFF]) * SECTOR:
+      raise RuntimeError("sector erase verification failed")
+    for off in range(0, SECTOR, PAGE):
+      chunk = target[off:off + PAGE]
+      if chunk != bytes([0xFF]) * len(chunk):
+        flash.program(addr + off, chunk)
+        if flash.read(addr + off, len(chunk)) != chunk:
+          raise RuntimeError(f"page verify failed at 0x{addr + off:05x}")
+    if flash.read(addr, SECTOR) != target:
+      raise RuntimeError("sector verification failed")
+  with_retries(flash, f"sector 0x{addr:05x}", program)
+
+
+def config_path():
+  return os.path.join(CONFIG_DIR, f"{os.uname().nodename}.bin")
+
+
+def saved_config(path, data):
+  os.makedirs(os.path.dirname(path), exist_ok=True)
+  try:
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+  except FileExistsError as e:
+    backup = open(path, "rb").read()
+    if len(backup) != 0x100:
+      raise RuntimeError(f"invalid config backup: {path}") from e
+    if backup != data:
+      print(f"restoring config from {path}", flush=True)
+    return backup
+  with os.fdopen(fd, "wb") as f:
+    f.write(data)
+    f.flush()
+    os.fsync(f.fileno())
+  return data
+
+
+def rom_write(image, config):
+  # the ROM bootloader implements only the BOT protocol, and requires a port reset before bulk transfers
+  path, _, _ = find_chestnut()
+  if path is None:
+    raise RuntimeError("chestnut disappeared before recovery")
+  unbind_drivers(path)
+  fd = open_device(path)
+  try:
+    fcntl.ioctl(fd, USBDEVFS_RESET)
+  finally:
+    os.close(fd)
+  time.sleep(3)
+  path, _, _ = find_chestnut()
+  if path is None:
+    raise RuntimeError("chestnut did not re-enumerate after reset")
+  fd = claim_interface(path, setup=True)
+  for ep in (0x02, 0x81):
+    fcntl.ioctl(fd, USBDEVFS_CLEAR_HALT, struct.pack("I", ep))
+  tag = 0
+
+  def bulk(ep, payload, timeout):
+    buf = ctypes.create_string_buffer(bytes(payload), len(payload))
+    fcntl.ioctl(fd, USBDEVFS_BULK, Bulk(ep, len(payload), timeout, ctypes.cast(buf, ctypes.c_void_p)))
+    return buf.raw
+
+  def cmd(cdb, data=b"", timeout=30000):
+    nonlocal tag
+    tag += 1
+    bulk(0x02, struct.pack("<IIIBBB16s", 0x43425355, tag, len(data), 0, 0, len(cdb), cdb), timeout)
+    if data:
+      bulk(0x02, data, timeout)
+    try:
+      csw = bulk(0x81, bytes(13), timeout)
+    except OSError as e:
+      if e.errno != errno.EPIPE:
+        raise
+      fcntl.ioctl(fd, USBDEVFS_CLEAR_HALT, struct.pack("I", 0x81))
+      csw = bulk(0x81, bytes(13), timeout)
+    if csw[:4] != b"USBS" or csw[12] != 0:
+      raise RuntimeError(f"ROM flash command {cdb[0]:02x} {cdb[1]:02x} failed")
+
+  print("recovering from the ROM bootloader", flush=True)
+  try:
+    cmd(struct.pack(">BBB12x", 0xE1, 0x50, 0), config[:0x80])
+    cmd(struct.pack(">BBB12x", 0xE1, 0x50, 1), config[0x80:])
+    cmd(struct.pack(">BBI", 0xE3, 0x50, min(len(image), 0xFF00)), image[:0xFF00])
+    if len(image) > 0xFF00:
+      cmd(struct.pack(">BBI", 0xE3, 0xD0, len(image) - 0xFF00), image[0xFF00:])
+    cmd(struct.pack(">BB13x", 0xE8, 0x51))
+  finally:
+    os.close(fd)
+  print("recovery flash done", flush=True)
+
+
+def vbus_write(value):
+  try:
+    with open(VBUS_PATH, "w") as f:
+      f.write(value + "\n")
+  except OSError:
+    pass
+
+
+def vbus_cycle():
+  if os.path.exists(VBUS_PATH):
+    vbus_write("0")
+    time.sleep(2)
+    vbus_write("1")
+    time.sleep(5)
+
+
+def activate(expected_product):
+  if not os.path.exists(VBUS_PATH):
+    print("no VBUS control, firmware activates on the next chestnut power cycle", flush=True)
+    return
+  print("power-cycling chestnut VBUS", flush=True)
+  vbus_write("0")
+  disconnected = False
+  deadline = time.monotonic() + 5.0
+  while time.monotonic() < deadline:
+    path, _, _ = find_chestnut()
+    if path is None:
+      disconnected = True
+      break
+    time.sleep(0.2)
+  time.sleep(1)
+  vbus_write("1")
+  if not disconnected:
+    print("chestnut stayed powered, firmware activates on its next power cycle", flush=True)
+    return
+  deadline = time.monotonic() + 15.0
+  while time.monotonic() < deadline:
+    _, _, product = find_chestnut()
+    if product is not None:
+      if product == expected_product:
+        print(f"activated {expected_product}", flush=True)
+      else:
+        print(f"chestnut re-enumerated with {product!r}, firmware activates on its next power cycle", flush=True)
+      return
+    time.sleep(0.2)
+  print("chestnut did not re-enumerate, firmware activates on its next power cycle", flush=True)
+
+
+def defer_signal(signum, _frame):
+  # writing from a handler must not reenter a print already in progress
+  os.write(1, f"signal {signum} deferred until the chestnut is powered back up\n".encode())
+
+
+def flash_chestnut(expected_version=None, force=False):
+  global _deadline
+
+  image = FIRMWARE_PATH.read_bytes()
+  validate_image(image)
+  expected_product = image_product(image)
+  if expected_version is not None and expected_product != f"custom {expected_version}-CLEAN":
+    raise RuntimeError(f"bundled firmware is {expected_product!r}, expected version {expected_version}")
+
+  path, vid_pid, product = find_chestnut()
+  if path is None:
+    print("no chestnut connected", flush=True)
+    return
+  if product == expected_product and not force:
+    print(f"chestnut firmware is up to date ({expected_product})", flush=True)
+    return
+
+  _deadline = time.monotonic() + FLASH_BUDGET
+  for pm_path in PM_PATHS:
+    disable_runtime_pm(pm_path)
+
+  previous = {sig: signal.signal(sig, defer_signal) for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)}
+  try:
+    if in_rom_bootloader(vid_pid, product):
+      if not recover_from_rom(image, expected_product):
+        return
+      # firmware is back, verify it against the bundled image
+      force, product = True, None
+    write_image(image, expected_product, product, force)
+  finally:
+    for sig, handler in previous.items():
+      signal.signal(sig, handler)
+
+
+def recover_from_rom(image, expected_product):
+  # returns whether the chestnut came back on custom firmware
+  backup = config_path()
+  if not os.path.isfile(backup):
+    raise RuntimeError(f"cannot recover from the ROM bootloader without a config backup at {backup}")
+  config = open(backup, "rb").read()
+  if len(config) != 0x100:
+    raise RuntimeError(f"invalid config backup: {backup}")
+
+  committed = False
+  while True:
+    check_budget()
+    path, vid_pid, product = find_chestnut()
+    if path is None:
+      if committed:
+        print("chestnut is offline, recovered firmware boots on its next power cycle", flush=True)
+        return False
+      vbus_cycle()
+      continue
+    if not in_rom_bootloader(vid_pid, product):
+      return True
+    if committed:
+      print("chestnut stayed powered, recovered firmware boots on its next power cycle", flush=True)
+      return False
+    try:
+      rom_write(image, config)
+      committed = True
+    except (OSError, TimeoutError, RuntimeError) as e:
+      print(f"ROM recovery failed, retrying: {e}", flush=True)
+      vbus_cycle()
+      continue
+    activate(expected_product)
+
+
+def write_image(image, expected_product, product, force):
+  if force:
+    print(f"forced reflash of {expected_product}", flush=True)
+  else:
+    print(f"chestnut firmware mismatch: {product!r}; expected {expected_product!r}", flush=True)
+
+  flash = Flash()
+  try:
+    reconnect(flash)
+    config = stable_read(flash, 0, 0x100, 3)
+    config = saved_config(config_path(), config)
+    image_end = IMAGE_OFFSET + len(image)
+    first_sector = IMAGE_OFFSET & ~(SECTOR - 1)
+    span = (image_end + SECTOR - 1) & ~(SECTOR - 1)
+    current = stable_read(flash, first_sector, span - first_sector)
+    target = bytearray(current)
+    target[:len(config)] = config
+    target[IMAGE_OFFSET - first_sector:image_end - first_sector] = image
+    target = bytes(target)
+    print(f"target {len(image)} bytes at 0x{IMAGE_OFFSET:05x}, sha256={hashlib.sha256(image).hexdigest()}", flush=True)
+
+    for addr in range(first_sector, span, SECTOR):
+      off = addr - first_sector
+      wanted = target[off:off + SECTOR]
+      if current[off:off + SECTOR] == wanted:
+        print(f"sector 0x{addr:05x}: unchanged", flush=True)
+      else:
+        print(f"sector 0x{addr:05x}: programming", flush=True)
+        program_sector(flash, addr, wanted)
+
+    verified = stable_read(flash, first_sector, span - first_sector, 3)
+    if verified != target:
+      raise RuntimeError("final full-image verification failed")
+    print(f"verified sha256={hashlib.sha256(verified).hexdigest()}", flush=True)
+  finally:
+    flash.close()
+
+  activate(expected_product)
+
+
+def main():
+  parser = argparse.ArgumentParser(description="check and flash the bundled chestnut firmware")
+  parser.add_argument("version", nargs="?", help="expected firmware version hash")
+  parser.add_argument("--force", action="store_true", help="reflash even when the version matches")
+  args = parser.parse_args()
+  if os.geteuid() != 0:
+    raise RuntimeError("flash.py must run as root")
+  flash_chestnut(expected_version=args.version, force=args.force)
+
+
+if __name__ == "__main__":
+  try:
+    main()
+  except Exception as e:
+    print(f"FAIL: {type(e).__name__}: {e}", file=sys.stderr)
+    sys.exit(1)
@@ -0,0 +1 @@
+../../../common/hardware/comma/agnos.json
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+import numpy as np
+
+from openpilot.common.pid import PIDController
+from openpilot.common.hardware import HARDWARE
+
+# raise fan setpoint on tici/tizi to reduce noise
+# after raising LMH threshold in AGNOS 18.1 to prevent CPU throttling
+OFFSET = 0 if HARDWARE.get_device_type() == "mici" else 5
+
+
+class FanController:
+  def __init__(self, rate: int) -> None:
+    self.last_ignition = False
+    self.controller = PIDController(k_p=0, k_i=4e-3, rate=rate)
+
+  def update(self, cur_temp: float, ignition: bool) -> int:
+    self.controller.pos_limit = 100 if ignition else 30
+    self.controller.neg_limit = 30 if ignition else 0
+
+    if ignition != self.last_ignition:
+      self.controller.reset()
+    self.last_ignition = ignition
+
+    return int(self.controller.update(
+                 error=(cur_temp - (75 + OFFSET)),  # temperature setpoint in C
+                 feedforward=np.interp(cur_temp, [60.0 + OFFSET, 100.0 + OFFSET], [0, 100])
+              ))
@@ -0,0 +1,538 @@
+#!/usr/bin/env python3
+import fcntl
+import os
+import queue
+import struct
+import subprocess
+import sys
+import threading
+import time
+from collections import OrderedDict, namedtuple
+
+import openpilot.cereal.messaging as messaging
+from openpilot.cereal import log
+from openpilot.cereal.services import SERVICE_LIST
+from openpilot.common.utils import strip_deprecated_keys
+from openpilot.common.filter_simple import FirstOrderFilter
+from openpilot.common.params import Params
+from openpilot.common.realtime import DT_HW
+from openpilot.selfdrive.modeld.helpers import MODELS_DIR, usbgpu_compiled
+from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+from openpilot.common.hardware import HARDWARE, COMMA_HARDWARE
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.hardware.usb import CHESTNUT_FW_VERSION, CHESTNUT_ROM_USB_IDS, CHESTNUT_USB_IDS, get_usb_state, get_usb_topology, set_usb_state
+from openpilot.common.linux import LinuxSystemStats
+from openpilot.system.loggerd.config import get_available_percent
+from openpilot.common.swaglog import cloudlog
+from openpilot.sunnypilot.system.statsd import statlog
+from openpilot.system.hardware.power_monitoring import PowerMonitoring
+from openpilot.system.hardware.fan_controller import FanController
+from openpilot.common.version import terms_version, training_version, get_build_metadata, terms_version_sp
+
+
+ThermalStatus = log.DeviceState.ThermalStatus
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+CURRENT_TAU = 15.   # 15s time constant
+TEMP_TAU = 5.   # 5s time constant
+DISCONNECT_TIMEOUT = 5.  # wait 5 seconds before going offroad after disconnect so you get an alert
+PANDA_STATES_TIMEOUT = round(1000 / SERVICE_LIST['pandaStates'].frequency * 1.5)  # 1.5x the expected pandaState frequency
+ONROAD_CYCLE_TIME = 1  # seconds to wait offroad after requesting an onroad cycle
+
+class Chestnut:
+  # flash offroad, modeld ignores chestnut until the product string matches
+  MAX_ATTEMPTS = 3
+  RETRY_INTERVAL = 20.
+
+  def __init__(self):
+    self.thread: threading.Thread | None = None
+    self.attempts = 0
+    self.last_attempt = 0.
+    self.flashed = False
+
+  def flash(self) -> None:
+    ret = subprocess.run(["sudo", sys.executable, os.path.join(BASEDIR, "openpilot/system/hardware/chestnut/flash.py"), CHESTNUT_FW_VERSION],
+                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=False)
+    cloudlog.event("chestnut flash done", returncode=ret.returncode, output=ret.stdout[-1000:], error=ret.returncode != 0)
+    self.flashed = ret.returncode == 0
+
+  def update(self, offroad: bool, usb_state: list[dict]) -> None:
+    mismatch = any((d["vendorId"], d["productId"]) in CHESTNUT_USB_IDS + CHESTNUT_ROM_USB_IDS and
+                   d["product"] != f"custom {CHESTNUT_FW_VERSION}-CLEAN" for d in usb_state)
+    if not mismatch:
+      self.flashed = False
+      return
+
+    if not offroad or self.flashed or self.attempts >= self.MAX_ATTEMPTS:
+      return
+    if self.thread is not None and self.thread.is_alive():
+      return
+    if time.monotonic() - self.last_attempt < self.RETRY_INTERVAL:
+      return
+
+    self.attempts += 1
+    self.last_attempt = time.monotonic()
+    cloudlog.warning(f"chestnut firmware out of date, flashing (attempt {self.attempts})")
+    self.thread = threading.Thread(target=self.flash, daemon=True)
+    self.thread.start()
+
+
+ThermalBand = namedtuple("ThermalBand", ['min_temp', 'max_temp'])
+HardwareState = namedtuple("HardwareState", ['network_type', 'network_info', 'network_strength', 'network_stats',
+                                             'network_metered', 'modem_temps', 'usb_state'])
+
+# List of thermal bands. We will stay within this region as long as we are within the bounds.
+# When exiting the bounds, we'll jump to the lower or higher band. Bands are ordered in the dict.
+if HARDWARE.get_device_type() == "mici":
+  THERMAL_BANDS = OrderedDict({
+    ThermalStatus.ok: ThermalBand(None, 100.0),
+    ThermalStatus.overheated: ThermalBand(92.0, 107.),
+    ThermalStatus.critical: ThermalBand(98.0, None),
+  })
+else:
+  THERMAL_BANDS = OrderedDict({
+    ThermalStatus.ok: ThermalBand(None, 96.0),
+    ThermalStatus.overheated: ThermalBand(88.0, 107.),
+    ThermalStatus.critical: ThermalBand(94.0, None),
+  })
+
+# Override to highest thermal band when offroad and above this temp
+OFFROAD_DANGER_TEMP = 85 if HARDWARE.get_device_type() == "mici" else 75
+
+prev_offroad_states: dict[str, tuple[bool, str | None]] = {}
+
+
+
+def set_offroad_alert_if_changed(offroad_alert: str, show_alert: bool, extra_text: str | None=None):
+  if prev_offroad_states.get(offroad_alert, None) == (show_alert, extra_text):
+    return
+  prev_offroad_states[offroad_alert] = (show_alert, extra_text)
+  set_offroad_alert(offroad_alert, show_alert, extra_text)
+
+def touch_thread(end_event):
+  count = 0
+
+  pm = messaging.PubMaster(["touch"])
+
+  event_format = "llHHi"
+  event_size = struct.calcsize(event_format)
+  event_frame = []
+
+  with open("/dev/input/by-path/platform-894000.i2c-event", "rb") as event_file:
+    fcntl.fcntl(event_file, fcntl.F_SETFL, os.O_NONBLOCK)
+    while not end_event.is_set():
+      if (count % int(1. / DT_HW)) == 0:
+        event = event_file.read(event_size)
+        if event:
+          (sec, usec, etype, code, value) = struct.unpack(event_format, event)
+          if etype != 0 or code != 0 or value != 0:
+            touch = log.Touch.new_message()
+            touch.sec = sec
+            touch.usec = usec
+            touch.type = etype
+            touch.code = code
+            touch.value = value
+            event_frame.append(touch)
+          else: # end of frame, push new log
+            msg = messaging.new_message('touch', len(event_frame), valid=True)
+            msg.touch = event_frame
+            pm.send('touch', msg)
+            event_frame = []
+          continue
+
+      count += 1
+      time.sleep(DT_HW)
+
+
+def hw_state_thread(end_event, hw_queue):
+  """Handles non critical hardware state, and sends over queue"""
+  count = 0
+  prev_hw_state = None
+  prev_usb_topology = set()
+
+  while not end_event.is_set():
+    usb_topology = get_usb_topology()
+    usb_changed = usb_topology != prev_usb_topology
+
+    # these are expensive calls. update every 10s or when USB devices change
+    if (count % int(10. / DT_HW)) == 0 or usb_changed:
+      prev_usb_topology = usb_topology
+      try:
+        network_type = HARDWARE.get_network_type()
+        modem_temps = HARDWARE.get_modem_temperatures()
+        if len(modem_temps) == 0 and prev_hw_state is not None:
+          modem_temps = prev_hw_state.modem_temps
+
+        tx, rx = HARDWARE.get_modem_data_usage()
+
+        hw_state = HardwareState(
+          network_type=network_type,
+          network_info=HARDWARE.get_network_info(),
+          network_strength=HARDWARE.get_network_strength(network_type),
+          network_stats={'wwanTx': tx, 'wwanRx': rx},
+          network_metered=HARDWARE.get_network_metered(network_type),
+          modem_temps=modem_temps,
+          usb_state=get_usb_state(),
+        )
+
+        try:
+          hw_queue.put_nowait(hw_state)
+        except queue.Full:
+          pass
+
+        prev_hw_state = hw_state
+      except Exception:
+        cloudlog.exception("Error getting hardware state")
+
+    count += 1
+    time.sleep(DT_HW)
+
+
+def hardware_thread(end_event, hw_queue) -> None:
+  system_stats = LinuxSystemStats()
+  pm = messaging.PubMaster(['deviceState'])
+  sm = messaging.SubMaster(["peripheralState", "gpsLocationExternal", "selfdriveState", "pandaStates"], poll="pandaStates")
+
+  count = 0
+
+  onroad_conditions: dict[str, bool] = {
+    "ignition": False,
+    "not_onroad_cycle": True,
+    "device_temp_good": True,
+  }
+  startup_conditions: dict[str, bool] = {}
+  startup_conditions_prev: dict[str, bool] = {}
+
+  off_ts: float | None = None
+  started_ts: float | None = None
+  started_seen = False
+  startup_blocked_ts: float | None = None
+  thermal_status = ThermalStatus.ok
+
+  last_hw_state = HardwareState(
+    network_type=NetworkType.none,
+    network_info=None,
+    network_metered=False,
+    network_strength=NetworkStrength.unknown,
+    network_stats={'wwanTx': -1, 'wwanRx': -1},
+    modem_temps=[],
+    usb_state=[],
+  )
+
+  all_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
+  offroad_temp_filter = FirstOrderFilter(0., TEMP_TAU, DT_HW, initialized=False)
+  should_start_prev = False
+  in_car = False
+  engaged_prev = False
+  pwrsave = False
+  offroad_cycle_count = 0
+
+  params = Params()
+  power_monitor = PowerMonitoring()
+
+  uptime_offroad: float = params.get("UptimeOffroad", return_default=True)
+  uptime_onroad: float = params.get("UptimeOnroad", return_default=True)
+  last_uptime_ts: float = time.monotonic()
+
+  HARDWARE.initialize_hardware()
+  thermal_config = HARDWARE.get_thermal_config()
+
+  fan_controller = FanController(int(1./DT_HW))
+  chestnut = Chestnut()
+  big_model_available = (MODELS_DIR / 'big_driving_supercombo.onnx').is_file() or usbgpu_compiled()
+
+  while not end_event.is_set():
+    sm.update(PANDA_STATES_TIMEOUT)
+
+    pandaStates = sm['pandaStates']
+    peripheralState = sm['peripheralState']
+
+    # handle requests to cycle system started state
+    if params.get_bool("OnroadCycleRequested"):
+      params.put_bool("OnroadCycleRequested", False, block=True)
+      offroad_cycle_count = sm.frame
+    onroad_conditions["not_onroad_cycle"] = (sm.frame - offroad_cycle_count) >= ONROAD_CYCLE_TIME * SERVICE_LIST['pandaStates'].frequency
+
+    if sm.updated['pandaStates'] and len(pandaStates) > 0:
+
+      # Set ignition based on any panda connected
+      onroad_conditions["ignition"] = any(ps.ignitionLine or ps.ignitionCan for ps in pandaStates if ps.pandaType != log.PandaState.PandaType.unknown)
+
+      pandaState = pandaStates[0]
+
+      in_car = pandaState.harnessStatus != log.PandaState.HarnessStatus.notConnected
+
+    elif (time.monotonic() - sm.recv_time['pandaStates']) > DISCONNECT_TIMEOUT:
+      if onroad_conditions["ignition"]:
+        onroad_conditions["ignition"] = False
+        cloudlog.error("panda timed out onroad")
+
+    # Run at 2Hz, plus either edge of ignition
+    ign_edge = (started_ts is not None) != all(onroad_conditions.values())
+    if (sm.frame % round(SERVICE_LIST['pandaStates'].frequency * DT_HW) != 0) and not ign_edge:
+      continue
+
+    msg = messaging.new_message('deviceState', valid=True)
+    msg.deviceState = thermal_config.get_msg()
+    msg.deviceState.deviceType = HARDWARE.get_device_type()
+
+    try:
+      last_hw_state = hw_queue.get_nowait()
+    except queue.Empty:
+      pass
+
+    msg.deviceState.freeSpacePercent = get_available_percent(default=100.0)
+    msg.deviceState.memoryUsagePercent = int(round(system_stats.memory_usage_percent()))
+    msg.deviceState.gpuUsagePercent = int(round(HARDWARE.get_gpu_usage_percent()))
+    online_cpu_usage = [int(round(n)) for n in system_stats.cpu_usage_percent()]
+    offline_cpu_usage = [0., ] * (len(msg.deviceState.cpuTempC) - len(online_cpu_usage))
+    msg.deviceState.cpuUsagePercent = online_cpu_usage + offline_cpu_usage
+
+    msg.deviceState.networkType = last_hw_state.network_type
+    msg.deviceState.networkMetered = last_hw_state.network_metered
+    msg.deviceState.networkStrength = last_hw_state.network_strength
+    msg.deviceState.networkStats = last_hw_state.network_stats
+    if last_hw_state.network_info is not None:
+      msg.deviceState.networkInfo = last_hw_state.network_info
+
+    msg.deviceState.modemTempC = last_hw_state.modem_temps
+
+    msg.deviceState.screenBrightnessPercent = HARDWARE.get_screen_brightness()
+
+    set_usb_state(msg.deviceState, last_hw_state.usb_state)
+    chestnut.update(started_ts is None, last_hw_state.usb_state)
+    set_offroad_alert_if_changed("Offroad_ChestnutBranch", msg.deviceState.chestnutPresent and not big_model_available)
+
+    # this subset is only used for offroad
+    temp_sources = [
+      msg.deviceState.memoryTempC,
+      max(msg.deviceState.cpuTempC, default=0.),
+      max(msg.deviceState.gpuTempC, default=0.),
+    ]
+    offroad_comp_temp = offroad_temp_filter.update(max(temp_sources))
+
+    # this drives the thermal status while onroad
+    temp_sources.append(max(msg.deviceState.pmicTempC, default=0.))
+    all_comp_temp = all_temp_filter.update(max(temp_sources))
+    msg.deviceState.maxTempC = all_comp_temp
+
+    msg.deviceState.fanSpeedPercentDesired = fan_controller.update(all_comp_temp, onroad_conditions["ignition"])
+
+    is_offroad_for_5_min = (started_ts is None) and ((not started_seen) or (off_ts is None) or (time.monotonic() - off_ts > 60 * 5))
+    if is_offroad_for_5_min and offroad_comp_temp > OFFROAD_DANGER_TEMP:
+      # if device is offroad and already hot without the extra onroad load,
+      # we want to cool down first before increasing load
+      thermal_status = ThermalStatus.critical
+    else:
+      current_band = THERMAL_BANDS[thermal_status]
+      band_idx = list(THERMAL_BANDS.keys()).index(thermal_status)
+      if current_band.min_temp is not None and all_comp_temp < current_band.min_temp:
+        thermal_status = list(THERMAL_BANDS.keys())[band_idx - 1]
+      elif current_band.max_temp is not None and all_comp_temp > current_band.max_temp:
+        thermal_status = list(THERMAL_BANDS.keys())[band_idx + 1]
+
+    # **** starting logic ****
+
+    startup_conditions["up_to_date"] = params.get("Offroad_ConnectivityNeeded") is None or params.get_bool("DisableUpdates") or params.get_bool("SnoozeUpdate")
+    startup_conditions["no_excessive_actuation"] = params.get("Offroad_ExcessiveActuation") is None
+    startup_conditions["not_uninstalling"] = not params.get_bool("DoUninstall")
+    startup_conditions["accepted_terms"] = params.get("HasAcceptedTerms") == terms_version
+    startup_conditions["accepted_terms_sp"] = params.get("HasAcceptedTermsSP") == terms_version_sp
+
+    # with 2% left, we killall, otherwise the phone will take a long time to boot
+    startup_conditions["free_space"] = msg.deviceState.freeSpacePercent > 2
+    startup_conditions["completed_training"] = params.get("CompletedTrainingVersion") == training_version
+    startup_conditions["not_driver_view"] = not params.get_bool("IsDriverViewEnabled")
+
+    # must be at an engageable thermal band to go onroad
+    startup_conditions["device_temp_engageable"] = thermal_status < ThermalStatus.overheated
+
+    # ensure device is fully booted
+    startup_conditions["device_booted"] = startup_conditions.get("device_booted", False) or HARDWARE.booted()
+
+    # user-forced status
+    offroad_mode = params.get_bool("OffroadMode")
+    startup_conditions["not_always_offroad"] = not offroad_mode
+    onroad_conditions["not_always_offroad"] = not offroad_mode
+
+    # if an unsupported device and branch is detected, going onroad is blocked
+    # only allow going onroad when:
+    # - TIZI, or
+    # - TICI and channel_type is "tici"
+    build_metadata = get_build_metadata()
+    is_unsupported_combo = COMMA_HARDWARE and HARDWARE.get_device_type() == "tici" and build_metadata.channel_type != "tici"
+    startup_conditions["not_tici"] = not is_unsupported_combo
+    onroad_conditions["not_tici"] = not is_unsupported_combo
+    set_offroad_alert("Offroad_TiciSupport", is_unsupported_combo, extra_text=build_metadata.channel)
+
+    # if the temperature enters the danger zone, go offroad to cool down
+    onroad_conditions["device_temp_good"] = thermal_status < ThermalStatus.critical
+    extra_text = f"{offroad_comp_temp:.1f}C"
+    show_alert = (not onroad_conditions["device_temp_good"] or not startup_conditions["device_temp_engageable"]) and onroad_conditions["ignition"]
+    set_offroad_alert_if_changed("Offroad_TemperatureTooHigh", show_alert, extra_text=extra_text)
+
+    if show_alert:
+      msg.deviceState.fanSpeedPercentDesired = 100
+
+    # Handle offroad/onroad transition
+    should_start = all(onroad_conditions.values())
+    if started_ts is None:
+      should_start = should_start and all(startup_conditions.values())
+
+    if should_start != should_start_prev or (count == 0):
+      params.put_bool("IsEngaged", False, block=True)
+      engaged_prev = False
+
+    if sm.updated['selfdriveState']:
+      engaged = sm['selfdriveState'].enabled
+      if engaged != engaged_prev:
+        params.put_bool("IsEngaged", engaged, block=True)
+        engaged_prev = engaged
+
+      try:
+        with open('/dev/kmsg', 'w') as kmsg:
+          kmsg.write(f"<3>[hardware] engaged: {engaged}\n")
+      except Exception:
+        pass
+
+    should_pwrsave = not onroad_conditions["ignition"] and msg.deviceState.screenBrightnessPercent < 1e-3
+    if should_pwrsave != pwrsave or (count == 0):
+      HARDWARE.set_power_save(should_pwrsave)
+    pwrsave = should_pwrsave
+
+    if should_start:
+      off_ts = None
+      if started_ts is None:
+        started_ts = time.monotonic()
+        started_seen = True
+        if startup_blocked_ts is not None:
+          cloudlog.event("Startup after block", block_duration=(time.monotonic() - startup_blocked_ts),
+                         startup_conditions=startup_conditions, onroad_conditions=onroad_conditions,
+                         startup_conditions_prev=startup_conditions_prev, error=True)
+      startup_blocked_ts = None
+    else:
+      if onroad_conditions["ignition"] and (startup_conditions != startup_conditions_prev):
+        cloudlog.event("Startup blocked", startup_conditions=startup_conditions, onroad_conditions=onroad_conditions, error=True)
+        startup_conditions_prev = startup_conditions.copy()
+        startup_blocked_ts = time.monotonic()
+
+      started_ts = None
+      if off_ts is None:
+        off_ts = time.monotonic()
+
+    # Offroad power monitoring
+    voltage = None if peripheralState.pandaType == log.PandaState.PandaType.unknown else peripheralState.voltage
+
+    # GitHub runner auto off: 9V is used as the threshold because most desktop runners
+    # will rarely exceed 5V so 9V is set as our buffer between desk use and car use.
+    params.put_bool("GithubRunnerSufficientVoltage", ((voltage or 0) and voltage > 9000))
+
+    power_monitor.calculate(voltage, onroad_conditions["ignition"])
+    msg.deviceState.offroadPowerUsageUwh = power_monitor.get_power_used()
+    msg.deviceState.carBatteryCapacityUwh = max(0, power_monitor.get_car_battery_capacity())
+    current_power_draw = HARDWARE.get_current_power_draw()
+    statlog.sample("power_draw", current_power_draw)
+    msg.deviceState.powerDrawW = current_power_draw
+
+    som_power_draw = HARDWARE.get_som_power_draw()
+    statlog.sample("som_power_draw", som_power_draw)
+    msg.deviceState.somPowerDrawW = som_power_draw
+
+    # Check if we need to shut down
+    if power_monitor.should_shutdown(onroad_conditions["ignition"], in_car, off_ts, started_seen):
+      cloudlog.warning(f"shutting device down, offroad since {off_ts}")
+      params.put_bool("DoShutdown", True, block=True)
+
+    msg.deviceState.started = started_ts is not None and not offroad_mode
+    msg.deviceState.startedMonoTime = int(1e9*(started_ts or 0))
+
+    last_ping = params.get("LastAthenaPingTime")
+    if last_ping is not None:
+      msg.deviceState.lastAthenaPingTime = last_ping
+
+    msg.deviceState.thermalStatus = thermal_status
+    pm.send("deviceState", msg)
+
+    statlog.gauge("free_space_percent", msg.deviceState.freeSpacePercent)
+    statlog.gauge("gpu_usage_percent", msg.deviceState.gpuUsagePercent)
+    statlog.gauge("memory_usage_percent", msg.deviceState.memoryUsagePercent)
+    for i, usage in enumerate(msg.deviceState.cpuUsagePercent):
+      statlog.gauge(f"cpu{i}_usage_percent", usage)
+    for i, temp in enumerate(msg.deviceState.cpuTempC):
+      statlog.gauge(f"cpu{i}_temperature", temp)
+    for i, temp in enumerate(msg.deviceState.gpuTempC):
+      statlog.gauge(f"gpu{i}_temperature", temp)
+    statlog.gauge("memory_temperature", msg.deviceState.memoryTempC)
+    for i, temp in enumerate(msg.deviceState.pmicTempC):
+      statlog.gauge(f"pmic{i}_temperature", temp)
+    for i, temp in enumerate(last_hw_state.modem_temps):
+      statlog.gauge(f"modem_temperature{i}", temp)
+    statlog.gauge("fan_speed_percent_desired", msg.deviceState.fanSpeedPercentDesired)
+    statlog.gauge("screen_brightness_percent", msg.deviceState.screenBrightnessPercent)
+
+    # report to server once every 10 minutes, or every 1s when thermally blocked
+    rising_edge_started = should_start and not should_start_prev
+    status_packet_interval = 1. if show_alert else 600.
+    if rising_edge_started or (count % int(status_packet_interval / DT_HW)) == 0:
+      dat = {
+        'count': count,
+        'pandaStates': [strip_deprecated_keys(p.to_dict()) for p in pandaStates],
+        'peripheralState': strip_deprecated_keys(peripheralState.to_dict()),
+        'location': (strip_deprecated_keys(sm["gpsLocationExternal"].to_dict()) if sm.alive["gpsLocationExternal"] else None),
+        'deviceState': strip_deprecated_keys(msg.to_dict())
+      }
+      cloudlog.event("STATUS_PACKET", **dat)
+
+      # save last one before going onroad
+      if rising_edge_started:
+        try:
+          params.put("LastOffroadStatusPacket", dat, block=True)
+        except Exception:
+          cloudlog.exception("failed to save offroad status")
+
+    params.put_bool("NetworkMetered", msg.deviceState.networkMetered)
+
+    now_ts = time.monotonic()
+    if off_ts:
+      uptime_offroad += now_ts - max(last_uptime_ts, off_ts)
+    elif started_ts:
+      uptime_onroad += now_ts - max(last_uptime_ts, started_ts)
+    last_uptime_ts = now_ts
+
+    if (count % int(60. / DT_HW)) == 0:
+      params.put("UptimeOffroad", uptime_offroad, block=True)
+      params.put("UptimeOnroad", uptime_onroad, block=True)
+
+    count += 1
+    should_start_prev = should_start
+
+
+def main():
+  hw_queue = queue.Queue(maxsize=1)
+  end_event = threading.Event()
+
+  threads = [
+    threading.Thread(target=hw_state_thread, args=(end_event, hw_queue)),
+    threading.Thread(target=hardware_thread, args=(end_event, hw_queue)),
+  ]
+
+  if COMMA_HARDWARE:
+    threads.append(threading.Thread(target=touch_thread, args=(end_event,)))
+
+  for t in threads:
+    t.start()
+
+  try:
+    while True:
+      time.sleep(1)
+      if not all(t.is_alive() for t in threads):
+        break
+  finally:
+    end_event.set()
+
+  for t in threads:
+    t.join()
+
+
+if __name__ == "__main__":
+  main()
@@ -0,0 +1,133 @@
+import time
+import threading
+
+from openpilot.common.params import Params
+from openpilot.common.hardware import HARDWARE
+from openpilot.common.swaglog import cloudlog
+from openpilot.sunnypilot.system.statsd import statlog
+
+CAR_VOLTAGE_LOW_PASS_K = 0.011 # LPF gain for 45s tau (dt/tau / (dt/tau + 1))
+
+# While driving, a battery charges completely in about 30-60 minutes
+CAR_BATTERY_CAPACITY_uWh = 30e6
+CAR_CHARGING_RATE_W = 45
+
+VBATT_PAUSE_CHARGING = 11.8           # Lower limit on the LPF car battery voltage
+MAX_TIME_OFFROAD_S = 30*3600
+MIN_ON_TIME_S = 3600
+DELAY_SHUTDOWN_TIME_S = 300 # Wait at least DELAY_SHUTDOWN_TIME_S seconds after offroad_time to shutdown.
+VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 60
+
+class PowerMonitoring:
+  def __init__(self):
+    self.params = Params()
+    self.last_measurement_time = None           # Used for integration delta
+    self.last_save_time = 0                     # Used for saving current value in a param
+    self.power_used_uWh = 0                     # Integrated power usage in uWh since going into offroad
+    self.next_pulsed_measurement_time = None
+    self.car_voltage_mV = 12e3                  # Low-passed version of peripheralState voltage
+    self.car_voltage_instant_mV = 12e3          # Last value of peripheralState voltage
+    self.integration_lock = threading.Lock()
+
+    car_battery_capacity_uWh = self.params.get("CarBatteryCapacity") or 0
+
+    # Reset capacity if it's low
+    self.car_battery_capacity_uWh = max((CAR_BATTERY_CAPACITY_uWh / 10), car_battery_capacity_uWh)
+
+  # Calculation tick
+  def calculate(self, voltage: float | None, ignition: bool):
+    try:
+      now = time.monotonic()
+
+      # If peripheralState is None, we're probably not in a car, so we don't care
+      if voltage is None:
+        with self.integration_lock:
+          self.last_measurement_time = None
+          self.next_pulsed_measurement_time = None
+          self.power_used_uWh = 0
+        return
+
+      # Low-pass battery voltage
+      self.car_voltage_instant_mV = voltage
+      self.car_voltage_mV = ((voltage * CAR_VOLTAGE_LOW_PASS_K) + (self.car_voltage_mV * (1 - CAR_VOLTAGE_LOW_PASS_K)))
+      statlog.gauge("car_voltage", self.car_voltage_mV / 1e3)
+
+      # Cap the car battery power and save it in a param every 10-ish seconds
+      self.car_battery_capacity_uWh = max(self.car_battery_capacity_uWh, 0)
+      self.car_battery_capacity_uWh = min(self.car_battery_capacity_uWh, CAR_BATTERY_CAPACITY_uWh)
+      if now - self.last_save_time >= 10:
+        self.params.put("CarBatteryCapacity", int(self.car_battery_capacity_uWh))
+        self.last_save_time = now
+
+      # First measurement, set integration time
+      with self.integration_lock:
+        if self.last_measurement_time is None:
+          self.last_measurement_time = now
+          return
+
+      if ignition:
+        # If there is ignition, we integrate the charging rate of the car
+        with self.integration_lock:
+          self.power_used_uWh = 0
+          integration_time_h = (now - self.last_measurement_time) / 3600
+          if integration_time_h < 0:
+            raise ValueError(f"Negative integration time: {integration_time_h}h")
+          self.car_battery_capacity_uWh += (CAR_CHARGING_RATE_W * 1e6 * integration_time_h)
+          self.last_measurement_time = now
+      else:
+        # Get current power draw somehow
+        current_power = HARDWARE.get_current_power_draw()
+
+        # Do the integration
+        self._perform_integration(now, current_power)
+    except Exception:
+      cloudlog.exception("Power monitoring calculation failed")
+
+  def _perform_integration(self, t: float, current_power: float) -> None:
+    with self.integration_lock:
+      try:
+        if self.last_measurement_time:
+          integration_time_h = (t - self.last_measurement_time) / 3600
+          power_used = (current_power * 1000000) * integration_time_h
+          if power_used < 0:
+            raise ValueError(f"Negative power used! Integration time: {integration_time_h} h Current Power: {power_used} uWh")
+          self.power_used_uWh += power_used
+          self.car_battery_capacity_uWh -= power_used
+          self.last_measurement_time = t
+      except Exception:
+        cloudlog.exception("Integration failed")
+
+  # Get the power usage
+  def get_power_used(self) -> int:
+    return int(self.power_used_uWh)
+
+  def get_car_battery_capacity(self) -> int:
+    return int(self.car_battery_capacity_uWh)
+
+  # Max Time Offroad
+  def max_time_offroad_exceeded(self, offroad_time):
+    param = self.params.get("MaxTimeOffroad")  # minutes, 0 = no limit
+    if param is not None and param >= 0:
+      return 0 < param * 60 <= offroad_time
+    return offroad_time > MAX_TIME_OFFROAD_S
+
+  # See if we need to shutdown
+  def should_shutdown(self, ignition: bool, in_car: bool, offroad_timestamp: float | None, started_seen: bool):
+    if offroad_timestamp is None:
+      return False
+
+    now = time.monotonic()
+    should_shutdown = False
+    offroad_time = (now - offroad_timestamp)
+    low_voltage_shutdown = (self.car_voltage_mV < (VBATT_PAUSE_CHARGING * 1e3) and
+                            offroad_time > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S)
+    should_shutdown |= self.max_time_offroad_exceeded(offroad_time)
+    should_shutdown |= low_voltage_shutdown
+    should_shutdown |= (self.car_battery_capacity_uWh <= 0)
+    should_shutdown &= not ignition
+    should_shutdown &= (not self.params.get_bool("DisablePowerDown"))
+    should_shutdown &= in_car
+    should_shutdown &= offroad_time > DELAY_SHUTDOWN_TIME_S
+    should_shutdown |= self.params.get_bool("ForcePowerDown")
+    should_shutdown &= started_seen or (now > MIN_ON_TIME_S)
+    return should_shutdown
@@ -0,0 +1,47 @@
+
+from openpilot.common.parameterized import parameterized
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.system.hardware.fan_controller import FanController
+
+ALL_CONTROLLERS = [FanController]
+
+class TestFanController(OpenpilotTestCase):
+  def wind_up(self, controller, ignition=True):
+    for _ in range(1000):
+      controller.update(100, ignition)
+
+  def wind_down(self, controller, ignition=False):
+    for _ in range(1000):
+      controller.update(10, ignition)
+
+  @parameterized.expand(ALL_CONTROLLERS)
+  def test_hot_onroad(self, controller_class):
+    controller = controller_class(2)
+    self.wind_up(controller)
+    assert controller.update(100, True) >= 70
+
+  @parameterized.expand(ALL_CONTROLLERS)
+  def test_offroad_limits(self, controller_class):
+    controller = controller_class(2)
+    self.wind_up(controller)
+    assert controller.update(100, False) <= 30
+
+  @parameterized.expand(ALL_CONTROLLERS)
+  def test_no_fan_wear(self, controller_class):
+    controller = controller_class(2)
+    self.wind_down(controller)
+    assert controller.update(10, False) == 0
+
+  @parameterized.expand(ALL_CONTROLLERS)
+  def test_limited(self, controller_class):
+    controller = controller_class(2)
+    self.wind_up(controller, True)
+    assert controller.update(100, True) == 100
+
+  @parameterized.expand(ALL_CONTROLLERS)
+  def test_windup_speed(self, controller_class):
+    controller = controller_class(2)
+    self.wind_down(controller, True)
+    for _ in range(10):
+      controller.update(90, True)
+    assert controller.update(90, True) >= 60
@@ -0,0 +1,231 @@
+from openpilot.common.parameterized import parameterized
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.common.params import Params
+from openpilot.system.hardware.power_monitoring import PowerMonitoring, CAR_BATTERY_CAPACITY_uWh, \
+  CAR_CHARGING_RATE_W, VBATT_PAUSE_CHARGING, DELAY_SHUTDOWN_TIME_S, MAX_TIME_OFFROAD_S
+
+# Create fake time
+ssb = 0.
+def mock_time_monotonic():
+  global ssb
+  ssb += 1.
+  return ssb
+
+def set_mock_time(value):
+  global ssb
+  ssb = value
+
+TEST_DURATION_S = 50
+GOOD_VOLTAGE = 12 * 1e3
+VOLTAGE_BELOW_PAUSE_CHARGING = (VBATT_PAUSE_CHARGING - 1) * 1e3
+
+def pm_patch(mocker, name, value, constant=False):
+  if constant:
+    mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", value)
+  else:
+    mocker.patch(f"openpilot.system.hardware.power_monitoring.{name}", return_value=value)
+
+
+class TestPowerMonitoring(OpenpilotTestCase):
+  def setup_method(self):
+    self._fixture("mocker").patch("time.monotonic", mock_time_monotonic)
+    self.params = Params()
+
+  # Test to see that it doesn't do anything when pandaState is None
+  def test_panda_state_present(self):
+    pm = PowerMonitoring()
+    for _ in range(10):
+      pm.calculate(None, False)
+    assert pm.get_power_used() == 0
+    assert pm.get_car_battery_capacity() == (CAR_BATTERY_CAPACITY_uWh / 10)
+
+  # Test to see that it doesn't integrate offroad when ignition is True
+  def test_offroad_ignition(self):
+    pm = PowerMonitoring()
+    for _ in range(10):
+      pm.calculate(GOOD_VOLTAGE, True)
+    assert pm.get_power_used() == 0
+
+  # Test to see that it integrates with discharging battery
+  def test_offroad_integration_discharging(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    expected_power_usage = ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_power_used() - expected_power_usage) < 10
+
+  # Test to check positive integration of car_battery_capacity
+  def test_car_battery_integration_onroad(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 0
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, True)
+    expected_capacity = ((TEST_DURATION_S/3600) * CAR_CHARGING_RATE_W * 1e6)
+    assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
+
+  # Test to check positive integration upper limit
+  def test_car_battery_integration_upper_limit(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh - 1000
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, True)
+    estimated_capacity = CAR_BATTERY_CAPACITY_uWh + (CAR_CHARGING_RATE_W / 3600 * 1e6)
+    assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
+
+  # Test to check negative integration of car_battery_capacity
+  def test_car_battery_integration_offroad(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    expected_capacity = CAR_BATTERY_CAPACITY_uWh - ((TEST_DURATION_S/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_car_battery_capacity() - expected_capacity) < 10
+
+  # Test to check negative integration lower limit
+  def test_car_battery_integration_lower_limit(self, mocker):
+    POWER_DRAW = 4
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 1000
+    for _ in range(TEST_DURATION_S + 1):
+      pm.calculate(GOOD_VOLTAGE, False)
+    estimated_capacity = 0 - ((1/3600) * POWER_DRAW * 1e6)
+    assert abs(pm.get_car_battery_capacity() - estimated_capacity) < 10
+
+  # Test to check policy of stopping charging after MAX_TIME_OFFROAD_S
+  def test_max_time_offroad(self, mocker):
+    MOCKED_MAX_OFFROAD_TIME = 3600
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    pm_patch(mocker, "MAX_TIME_OFFROAD_S", MOCKED_MAX_OFFROAD_TIME, constant=True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    start_time = ssb
+    ignition = False
+    set_mock_time(start_time + MOCKED_MAX_OFFROAD_TIME - 1)
+    assert not pm.should_shutdown(ignition, True, start_time, False)
+    set_mock_time(start_time + MOCKED_MAX_OFFROAD_TIME)
+    assert pm.should_shutdown(ignition, True, start_time, False)
+
+  def test_car_voltage(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 350
+    VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S = 50
+    pm_patch(mocker, "VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S", VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S, constant=True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = False
+    start_time = ssb
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert pm.should_shutdown(ignition, True, start_time, True) == \
+                          (pm.car_voltage_mV < VBATT_PAUSE_CHARGING * 1e3 and \
+                          (ssb - start_time) > VOLTAGE_SHUTDOWN_MIN_OFFROAD_TIME_S and \
+                            (ssb - start_time) > DELAY_SHUTDOWN_TIME_S)
+    assert pm.should_shutdown(ignition, True, start_time, True)
+
+  # Test to check policy of not stopping charging when DisablePowerDown is set
+  def test_disable_power_down(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    self.params.put_bool("DisablePowerDown", True, block=True)
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = False
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, True, ssb, False)
+    assert not pm.should_shutdown(ignition, True, ssb, False)
+
+  # Test to check policy of not stopping charging when ignition
+  def test_ignition(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+    ignition = True
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, True, ssb, False)
+    assert not pm.should_shutdown(ignition, True, ssb, False)
+
+  # Test to check policy of not stopping charging when harness is not connected
+  def test_harness_connection(self, mocker):
+    POWER_DRAW = 0 # To stop shutting down for other reasons
+    TEST_TIME = 100
+    pm_patch(mocker, "HARDWARE.get_current_power_draw", POWER_DRAW)
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = CAR_BATTERY_CAPACITY_uWh
+
+    ignition = False
+    for i in range(TEST_TIME):
+      pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+      if i % 10 == 0:
+        assert not pm.should_shutdown(ignition, False, ssb, False)
+    assert not pm.should_shutdown(ignition, False, ssb, False)
+
+  def test_delay_shutdown_time(self):
+    pm = PowerMonitoring()
+    pm.car_battery_capacity_uWh = 0
+    ignition = False
+    in_car = True
+    offroad_timestamp = ssb
+    started_seen = True
+    pm.calculate(VOLTAGE_BELOW_PAUSE_CHARGING, ignition)
+
+    set_mock_time(offroad_timestamp + DELAY_SHUTDOWN_TIME_S - 1)
+    assert not pm.should_shutdown(ignition, in_car, offroad_timestamp, started_seen), \
+                     f"Should not shutdown before {DELAY_SHUTDOWN_TIME_S} seconds offroad time"
+    set_mock_time(offroad_timestamp + DELAY_SHUTDOWN_TIME_S)
+    assert pm.should_shutdown(ignition, in_car,
+                                       offroad_timestamp,
+                                       started_seen), \
+                    f"Should shutdown after {DELAY_SHUTDOWN_TIME_S} seconds offroad time"
+
+  @parameterized.expand(
+    [
+      # No max time set – fallback to default (30 hours)
+      (None, 0, False),
+      (None, MAX_TIME_OFFROAD_S + 1, True),  # exceeds 30h (1800+ mins)
+
+      # Valid max time values (in minutes)
+      (60, 59, False),  # under limit
+      (60, 120, True),  # over limit
+      (10, 8, False),  # under limit
+      (10, 11, True),  # over limit
+
+      # Edge case: max time is zero → no limit enforced
+      (0, 0, False),
+      (0, 400, False),
+
+      # Invalid max time formats or negative values → fallback to 30 hours
+      (-100, 100, False),  # should fallback to 30h
+      (-1, MAX_TIME_OFFROAD_S + 1, True),  # should fallback to 30h, and exceed it
+    ]
+  )
+  def test_max_time_offroad_exceeded(self, max_time_offroad, offroad_time_min, expected_result):
+    # Set the parameter if provided
+    if max_time_offroad is not None:
+      self.params.put("MaxTimeOffroad", max_time_offroad, block=True)
+
+    # Convert offroad time from minutes to seconds
+    offroad_time_s = offroad_time_min * 60
+
+    pm = PowerMonitoring()
+    result = pm.max_time_offroad_exceeded(offroad_time_s)
+
+    assert result == expected_result
@@ -0,0 +1 @@
+../comma/agnos.json
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import json
+import subprocess
+
+import openpilot.cereal.messaging as messaging
+from openpilot.common.swaglog import cloudlog
+
+
+def main():
+  pm = messaging.PubMaster(['operatingSystemLog'])
+  cmd = ['journalctl', '-f', '-o', 'json']
+  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True)
+  assert proc.stdout is not None
+  try:
+    for line in proc.stdout:
+      line = line.strip()
+      if not line:
+        continue
+      try:
+        kv = json.loads(line)
+      except json.JSONDecodeError:
+        cloudlog.exception("failed to parse journalctl output")
+        continue
+
+      msg = messaging.new_message('operatingSystemLog')
+      entry = msg.operatingSystemLog
+      entry.ts = int(kv.get('__REALTIME_TIMESTAMP', 0))
+      entry.message = json.dumps(kv)
+      if '_PID' in kv:
+        entry.pid = int(kv['_PID'])
+      if 'PRIORITY' in kv:
+        entry.priority = int(kv['PRIORITY'])
+      if 'SYSLOG_IDENTIFIER' in kv:
+        entry.tag = kv['SYSLOG_IDENTIFIER']
+
+      pm.send('operatingSystemLog', msg)
+  finally:
+    proc.terminate()
+    proc.wait()
+
+
+if __name__ == '__main__':
+  main()
@@ -0,0 +1,4 @@
+loggerd
+encoderd
+bootlog
+tests/test_logger
@@ -0,0 +1,19 @@
+Import('env', 'arch', 'messaging', 'common', 'visionipc', 'ffmpeg_libs')
+
+libs = [common, messaging, visionipc] + ffmpeg_libs + ['pthread', 'm', 'zstd']
+frameworks = []
+
+src = ['logger.cc', 'zstd_writer.cc', 'video_writer.cc', 'encoder/encoder.cc', 'encoder/jpeg_encoder.cc']
+if arch == "comma_arm64":
+  src += ['clip_encoder.cc', 'encoder/v4l_encoder.cc', 'encoder/v4l_decoder.cc']
+else:
+  src += ['encoder/ffmpeg_encoder.cc']
+  if arch == "Darwin":
+    frameworks += ['VideoToolbox', 'CoreMedia', 'CoreFoundation', 'CoreVideo']
+
+logger_lib = env.Library('logger', src)
+libs.insert(0, logger_lib)
+
+env.Program('loggerd', ['loggerd.cc'], LIBS=libs, FRAMEWORKS=frameworks)
+env.Program('encoderd', ['encoderd.cc'], LIBS=libs, FRAMEWORKS=frameworks)
+env.Program('bootlog.cc', LIBS=libs, FRAMEWORKS=frameworks)
@@ -0,0 +1,68 @@
+#include <cassert>
+#include <string>
+
+#include "openpilot/cereal/messaging/messaging.h"
+#include "common/params.h"
+#include "common/swaglog.h"
+#include "system/loggerd/logger.h"
+#include "system/loggerd/zstd_writer.h"
+
+
+static kj::Array<capnp::word> build_boot_log() {
+  MessageBuilder msg;
+  auto boot = msg.initEvent().initBoot();
+
+  boot.setWallTimeNanos(nanos_since_epoch());
+
+  std::string pstore = "/sys/fs/pstore";
+  std::map<std::string, std::string> pstore_map = util::read_files_in_dir(pstore);
+
+  int i = 0;
+  auto lpstore = boot.initPstore().initEntries(pstore_map.size());
+  for (auto& kv : pstore_map) {
+    auto lentry = lpstore[i];
+    lentry.setKey(kv.first);
+    lentry.setValue(capnp::Data::Reader((const kj::byte*)kv.second.data(), kv.second.size()));
+    i++;
+  }
+
+  // Gather output of commands
+  std::vector<std::string> bootlog_commands = {
+    "[ -x \"$(command -v journalctl)\" ] && journalctl -b -n 2000 -o short-monotonic --no-pager",
+  };
+
+
+  auto commands = boot.initCommands().initEntries(bootlog_commands.size());
+  for (int j = 0; j < bootlog_commands.size(); j++) {
+    auto lentry = commands[j];
+
+    lentry.setKey(bootlog_commands[j]);
+
+    const std::string result = util::check_output(bootlog_commands[j]);
+    lentry.setValue(capnp::Data::Reader((const kj::byte*)result.data(), result.size()));
+  }
+
+  boot.setLaunchLog(util::read_file("/tmp/launch_log"));
+  return capnp::messageToFlatArray(msg);
+}
+
+int main(int argc, char** argv) {
+  const std::string id = logger_get_identifier("BootCount");
+  const std::string path = Path::log_root() + "/boot/" + id + ".zst";
+  LOGW("bootlog to %s", path.c_str());
+
+  // Open bootlog
+  bool r = util::create_directories(Path::log_root() + "/boot/", 0775);
+  assert(r);
+
+  ZstdFileWriter file(path, LOG_COMPRESSION_LEVEL);
+  // Write initdata
+  file.write(logger_build_init_data().asBytes());
+  // Write bootlog
+  file.write(build_boot_log().asBytes());
+
+  // Write out bootlog param to match routes with bootlog
+  Params().put("CurrentBootlog", id.c_str());
+
+  return 0;
+}
@@ -0,0 +1,290 @@
+#include "system/loggerd/clip_encoder.h"
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <exception>
+#include <filesystem>
+#include <memory>
+#include <thread>
+#include <unistd.h>
+#include <utility>
+#include <vector>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+}
+
+#include "common/swaglog.h"
+#include "system/loggerd/encoder/v4l_decoder.h"
+#include "system/loggerd/encoder/v4l_encoder.h"
+#include "system/loggerd/loggerd.h"
+#include "system/loggerd/video_writer.h"
+
+namespace {
+
+constexpr double SEGMENT_DURATION = 60.0;
+constexpr int CLIP_FPS = 20;
+constexpr double PARALLEL_CLIP_MIN_DURATION = 2 * SEGMENT_DURATION;
+
+const EncoderInfo clip_encoder_info = {
+  .publish_name = "livestreamNarrowRoadEncodeData",
+  .record = false,
+  .fps = CLIP_FPS,
+  .get_settings = [](int) { return EncoderSettings::StreamEncoderSettings(); },
+  INIT_ENCODE_FUNCTIONS(LivestreamNarrowRoadEncode),
+};
+
+bool open_input(const std::string &path, AVFormatContext **ctx, int *stream_index) {
+  if (avformat_open_input(ctx, path.c_str(), nullptr, nullptr) < 0 ||
+      avformat_find_stream_info(*ctx, nullptr) < 0 ||
+      (*stream_index = av_find_best_stream(*ctx, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0)) < 0) {
+    LOGE("failed to open clip input %s", path.c_str());
+    avformat_close_input(ctx);
+    return false;
+  }
+  return true;
+}
+
+void remove_file(const std::string &path) {
+  std::error_code error;
+  std::filesystem::remove(path, error);
+}
+
+int encode_clip_worker(const std::vector<std::string> &inputs, int width, int height,
+                       double start_time, double duration, int bitrate, int speedup,
+                       int64_t frame_offset, int64_t *encoded_frames,
+                       V4LEncoder::PacketCallback packet_callback) try {
+  EncoderInfo encoder_info = clip_encoder_info;
+  encoder_info.get_settings = [bitrate](int) {
+    return EncoderSettings{.encode_type = cereal::EncodeIndex::Type::QCAMERA_H264,
+                           .bitrate = bitrate, .gop_size = 5};
+  };
+  V4LDecoder decoder;
+  V4LEncoder::Options options = {
+    .packet_callback = std::move(packet_callback),
+    .input_format = V4L2_PIX_FMT_NV12_UBWC,
+    .input_done_callback = [&decoder](VisionBuf *buf) { decoder.releaseFrame(buf); },
+    .max_performance = true,
+  };
+  V4LEncoder encoder(encoder_info, width, height, std::move(options));
+  encoder.encoder_open();
+  if (!decoder.init(V4LDecoder::DEVICE, width, height, V4L2_PIX_FMT_HEVC, true, V4L2_PIX_FMT_NV12_UBWC)) return 1;
+
+  const int64_t first_frame = std::floor(start_time * CLIP_FPS);
+  const int64_t end_frame = std::ceil((start_time + duration) * CLIP_FPS);
+  int64_t input_frame = 0;
+  int64_t output_frame = 0;
+  int64_t received_frames = 0;
+  bool failed = false;
+  auto pump_decoder = [&](int timeout_ms) {
+    V4LDecodedFrame frame;
+    if (!decoder.pump(frame, timeout_ms)) return false;
+    if (!frame.buf) return true;
+    ++received_frames;
+    const int64_t source_frame = (int64_t)frame.token - 1;
+    if (source_frame < first_frame) {
+      decoder.releaseFrame(frame.buf);
+      return true;
+    }
+    if ((frame_offset + source_frame - first_frame) % speedup != 0) {
+      decoder.releaseFrame(frame.buf);
+      return true;
+    }
+
+    VisionIpcBufExtra extra = {};
+    extra.frame_id = output_frame;
+    extra.timestamp_sof = output_frame * 1000000000ULL / CLIP_FPS;
+    extra.timestamp_eof = extra.timestamp_sof;
+    if (encoder.encode_frame(frame.buf, &extra) < 0) {
+      decoder.releaseFrame(frame.buf);
+      return false;
+    }
+
+    ++output_frame;
+    return true;
+  };
+
+  for (size_t input_index = 0; input_index < inputs.size(); ++input_index) {
+    const std::string &input = inputs[input_index];
+    const int64_t segment_start_frame = input_frame;
+    AVFormatContext *ctx = nullptr;
+    int stream_index = -1;
+    if (!open_input(input, &ctx, &stream_index)) { failed = true; break; }
+    AVPacket packet = {};
+    while (input_frame < end_frame && av_read_frame(ctx, &packet) >= 0) {
+      if (packet.stream_index != stream_index) {
+        av_packet_unref(&packet);
+        continue;
+      }
+      if (packet.size <= 0 || (size_t)packet.size > decoder.maxPacketSize()) {
+        LOGE("decoder packet too large: %d > %zu", packet.size, decoder.maxPacketSize());
+        av_packet_unref(&packet);
+        failed = true;
+        break;
+      }
+
+      // Keep several compressed packets in flight so the firmware can sustain
+      // decode/encode overlap and does not downclock due to a shallow queue.
+      while (!decoder.queuePacket(&packet, input_frame + 1)) {
+        if (!pump_decoder(-1)) {
+          failed = true;
+          break;
+        }
+      }
+      av_packet_unref(&packet);
+      if (failed) break;
+
+      ++input_frame;
+    }
+    av_packet_unref(&packet);
+    avformat_close_input(&ctx);
+    // Only the final loggerd segment may be shorter than SEGMENT_DURATION. A
+    // short intermediate segment would silently close a gap in the source.
+    if (!failed && input_frame < end_frame && input_index + 1 < inputs.size() &&
+        input_frame - segment_start_frame < static_cast<int64_t>(SEGMENT_DURATION * CLIP_FPS)) {
+      failed = true;
+    }
+    if (failed || input_frame >= end_frame) break;
+  }
+
+  if (!failed) decoder.sendEOS();
+  for (int empty_polls = 0; !failed && received_frames < input_frame;) {
+    const int64_t before = received_frames;
+    failed = !pump_decoder(1000);
+    empty_polls = received_frames == before ? empty_polls + 1 : 0;
+    if (empty_polls == 5) failed = true;
+  }
+
+  encoder.encoder_close();
+  const int64_t source_frames = std::max<int64_t>(0, std::min(input_frame, end_frame) - first_frame);
+  const int64_t first_output_frame = (speedup - frame_offset % speedup) % speedup;
+  const int64_t expected_output_frames = first_output_frame < source_frames ?
+    1 + (source_frames - first_output_frame - 1) / speedup : 0;
+  if (failed || source_frames == 0 || output_frame != expected_output_frames) {
+    LOGE("clip failed: input=%lld/%lld decoded=%lld encoded=%lld/%lld",
+         (long long)input_frame, (long long)end_frame, (long long)received_frames,
+         (long long)output_frame, (long long)expected_output_frames);
+    return 1;
+  }
+  *encoded_frames = output_frame;
+  return 0;
+} catch (const std::exception &e) {
+  LOGE("clip worker failed: %s", e.what());
+  return 1;
+}
+
+struct SpoolPacket {
+  uint32_t size;
+  int64_t timestamp;
+  bool keyframe;
+};
+
+}  // namespace
+
+int encode_clip(const std::vector<std::string> &inputs, const std::string &output,
+                double start_time, double duration, int bitrate, int speedup,
+                const std::string &metadata) {
+  if (inputs.empty() || !std::isfinite(start_time) || !std::isfinite(duration) ||
+      start_time < 0 || duration <= 0 || bitrate <= 0 || speedup <= 0) {
+    return 1;
+  }
+
+  // Inputs are consecutive loggerd segments. Skip whole files before the clip
+  // so a late start does not spend hardware time decoding discarded minutes.
+  const double available_duration = inputs.size() * SEGMENT_DURATION;
+  if (start_time >= available_duration || duration > available_duration - start_time) return 1;
+  const size_t skipped_segments = start_time / SEGMENT_DURATION;
+  const std::vector<std::string> clip_inputs(inputs.begin() + skipped_segments, inputs.end());
+  const double local_start = start_time - skipped_segments * SEGMENT_DURATION;
+
+  AVFormatContext *ctx = nullptr;
+  int stream = -1;
+  if (!open_input(clip_inputs.front(), &ctx, &stream)) return 1;
+  AVCodecParameters *codec = ctx->streams[stream]->codecpar;
+  const int width = codec->width, height = codec->height;
+  const bool valid_codec = codec->codec_id == AV_CODEC_ID_HEVC && width > 0 && height > 0;
+  avformat_close_input(&ctx);
+  if (!valid_codec) return 1;
+
+  std::filesystem::path output_path(output);
+  const std::string output_dir = output_path.has_parent_path() ? output_path.parent_path() : ".";
+  auto writer = std::make_unique<VideoWriter>(output_dir.c_str(), output_path.filename().c_str(), true,
+                                              width, height, CLIP_FPS, cereal::EncodeIndex::Type::QCAMERA_H264);
+  if (!metadata.empty()) writer->set_metadata("ai.comma.clip.settings", metadata.c_str());
+  V4LEncoder::PacketCallback write_packet = [&writer](uint8_t *data, size_t size, int64_t timestamp,
+                                                      bool config, bool keyframe) {
+    writer->write(data, size, timestamp, config, keyframe);
+  };
+
+  if (clip_inputs.size() < 2 || duration < PARALLEL_CLIP_MIN_DURATION) {
+    int64_t encoded_frames = 0;
+    const bool success = encode_clip_worker(clip_inputs, width, height, local_start, duration,
+                                            bitrate, speedup, 0, &encoded_frames, write_packet) == 0;
+    if (!success) {
+      writer.reset();
+      remove_file(output);
+    }
+    return success ? 0 : 1;
+  }
+
+  const size_t split = std::clamp<size_t>(std::llround((local_start + duration / 2) / SEGMENT_DURATION),
+                                           1, clip_inputs.size() - 1);
+  const double split_time = split * SEGMENT_DURATION;
+  const std::array<std::vector<std::string>, 2> shard_inputs = {
+    std::vector<std::string>(clip_inputs.begin(), clip_inputs.begin() + split),
+    std::vector<std::string>(clip_inputs.begin() + split, clip_inputs.end()),
+  };
+  const std::array<double, 2> shard_starts = {local_start, 0};
+  const std::array<double, 2> shard_durations = {
+    split_time - local_start, local_start + duration - split_time,
+  };
+  const std::string spool_path = output + ".encoderd-" + std::to_string(getpid()) + ".tmp";
+  FILE *spool = fopen(spool_path.c_str(), "w+b");
+  if (!spool) {
+    writer.reset();
+    remove_file(output);
+    return 1;
+  }
+  remove_file(spool_path);
+  bool spool_ok = true;
+  V4LEncoder::PacketCallback spool_packet = [&](uint8_t *data, size_t size, int64_t timestamp,
+                                                bool config, bool keyframe) {
+    if (config) return;
+    const SpoolPacket packet = {(uint32_t)size, timestamp, keyframe};
+    spool_ok &= fwrite(&packet, sizeof(packet), 1, spool) == 1 && fwrite(data, 1, size, spool) == size;
+  };
+  std::array<int, 2> results = {1, 1};
+  std::array<int64_t, 2> encoded_frames = {};
+  const std::array<int64_t, 2> frame_offsets = {
+    0, (int64_t)std::llround(split_time * CLIP_FPS) - (int64_t)std::floor(local_start * CLIP_FPS),
+  };
+  std::array<std::thread, 2> workers;
+
+  for (size_t i = 0; i < workers.size(); ++i) {
+    workers[i] = std::thread([&, i]() {
+      results[i] = encode_clip_worker(shard_inputs[i], width, height, shard_starts[i], shard_durations[i],
+                                      bitrate, speedup, frame_offsets[i], &encoded_frames[i],
+                                      i == 0 ? write_packet : spool_packet);
+    });
+  }
+  for (std::thread &worker : workers) worker.join();
+
+  rewind(spool);
+  SpoolPacket packet;
+  std::vector<uint8_t> data;
+  const int64_t timestamp_offset = encoded_frames[0] * 1000000 / CLIP_FPS;
+  while (spool_ok && fread(&packet, sizeof(packet), 1, spool) == 1) {
+    data.resize(packet.size);
+    spool_ok = fread(data.data(), 1, data.size(), spool) == data.size();
+    if (spool_ok) writer->write(data.data(), data.size(), packet.timestamp + timestamp_offset, false, packet.keyframe);
+  }
+  fclose(spool);
+  bool success = results[0] == 0 && results[1] == 0 && spool_ok;
+  if (!success) {
+    writer.reset();
+    remove_file(output);
+  }
+  return success ? 0 : 1;
+}
@@ -0,0 +1,10 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+// inputs are consecutive 60-second loggerd HEVC segments; start_time is
+// relative to the beginning of the first input.
+int encode_clip(const std::vector<std::string> &inputs, const std::string &output,
+                double start_time, double duration, int bitrate = 5'000'000,
+                int speedup = 1, const std::string &metadata = {});
@@ -0,0 +1,34 @@
+import os
+from openpilot.common.hardware.hw import Paths
+
+
+CAMERA_FPS = 20
+SEGMENT_LENGTH = 60
+
+STATS_DIR_FILE_LIMIT = 10000
+STATS_SOCKET = "ipc:///tmp/stats"
+STATS_FLUSH_TIME_S = 60
+
+PATH_DICT = {
+  "internal": Paths.log_root(),
+  "external": Paths.log_root_external()
+}
+
+def get_available_percent(default: float, path_type="internal") -> float:
+  try:
+    statvfs = os.statvfs(PATH_DICT[path_type])
+    available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks
+  except (OSError, KeyError):
+    available_percent = default
+
+  return available_percent
+
+
+def get_available_bytes(default: int, path_type="internal") -> int:
+  try:
+    statvfs = os.statvfs(PATH_DICT[path_type])
+    available_bytes = statvfs.f_bavail * statvfs.f_frsize
+  except (OSError, KeyError):
+    available_bytes = default
+
+  return available_bytes
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+import os
+import time
+import shutil
+import threading
+from pathlib import Path
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.loggerd.config import get_available_bytes, get_available_percent
+from openpilot.system.loggerd.uploader import listdir_by_creation
+from openpilot.system.loggerd.xattr_cache import getxattr
+
+MIN_BYTES = 5 * 1024 * 1024 * 1024
+MIN_PERCENT = 10
+
+DELETE_LAST = ['boot', 'crash']
+
+PRESERVE_ATTR_NAME = 'user.preserve'
+PRESERVE_ATTR_VALUE = b'1'
+PRESERVE_COUNT = 5
+
+
+def has_preserve_xattr(d: str) -> bool:
+  return getxattr(os.path.join(Paths.log_root(), d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+
+def get_preserved_segments(dirs_by_creation: list[str]) -> set[str]:
+  # skip deleting most recent N preserved segments (and their prior segment)
+  preserved = set()
+  for n, d in enumerate(filter(has_preserve_xattr, reversed(dirs_by_creation))):
+    if n == PRESERVE_COUNT:
+      break
+    date_str, _, seg_str = d.rpartition("--")
+
+    # ignore non-segment directories
+    if not date_str:
+      continue
+    try:
+      seg_num = int(seg_str)
+    except ValueError:
+      continue
+
+    # preserve segment and two prior
+    for _seg_num in range(max(0, seg_num - 2), seg_num + 1):
+      preserved.add(f"{date_str}--{_seg_num}")
+
+  return preserved
+
+
+def deleter_step() -> tuple[bool, str | None]:
+  out_of_bytes = get_available_bytes(default=MIN_BYTES + 1) < MIN_BYTES
+  out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
+  out_of_space = out_of_percent or out_of_bytes
+  if not out_of_space:
+    return False, None
+
+  dirs = listdir_by_creation(Paths.log_root())
+  preserved_dirs = get_preserved_segments(dirs)
+
+  # remove the earliest directory we can
+  for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
+    delete_path = os.path.join(Paths.log_root(), delete_dir)
+
+    if any(name.endswith(".lock") for name in os.listdir(delete_path)):
+      continue
+
+    try:
+      cloudlog.info(f"deleting {delete_path}")
+      shutil.rmtree(delete_path)
+      return True, delete_path
+    except OSError:
+      cloudlog.exception(f"issue deleting {delete_path}")
+  return True, None
+
+
+def deleter_thread(exit_event: threading.Event):
+  while not exit_event.is_set():
+    out_of_bytes = get_available_bytes(default=MIN_BYTES + 1) < MIN_BYTES
+    out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
+
+    if out_of_percent or out_of_bytes:
+      dirs = listdir_by_creation(Paths.log_root())
+      preserved_dirs = get_preserved_segments(dirs)
+
+      # remove the earliest directory we can
+      for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
+        delete_path = os.path.join(Paths.log_root(), delete_dir)
+
+        if any(name.endswith(".lock") for name in os.listdir(delete_path)):
+          continue
+
+        if Path(Paths.log_root_external()).is_mount():
+          out_of_bytes_external = get_available_bytes(default=MIN_BYTES + 1, path_type="external") < MIN_BYTES
+          out_of_percent_external = get_available_percent(default=MIN_PERCENT + 1, path_type="external") < MIN_PERCENT
+
+          if out_of_percent_external or out_of_bytes_external:
+            dirs_external = listdir_by_creation(Paths.log_root_external())
+
+            # remove the earliest external directory we can
+            for delete_dir_external in sorted(dirs_external):
+              delete_path_external = os.path.join(Paths.log_root_external(), delete_dir_external)
+              try:
+                cloudlog.warning(f"deleting {delete_path_external}")
+                shutil.rmtree(delete_path_external)
+                break
+              except OSError:
+                cloudlog.exception(f"issue deleting {delete_path_external}")
+
+          # move directory from internal to external
+          path_external = os.path.join(Paths.log_root_external(), delete_dir)
+          try:
+            cloudlog.warning(f"moving {delete_path} to {path_external}")
+            start = time.monotonic()
+            shutil.move(delete_path, path_external)
+            cloudlog.warning(f"moved {delete_path} to {path_external} in {time.monotonic() - start:.2f}s")
+            break
+          except Exception:
+            cloudlog.error(f"issue moving {delete_path} to {path_external}")
+            try:
+              cloudlog.warning(f"deleting {delete_path}")
+              shutil.rmtree(delete_path)
+              break
+            except OSError:
+              cloudlog.exception(f"issue deleting {delete_path}")
+          continue
+
+        try:
+          cloudlog.info(f"deleting {delete_path}")
+          shutil.rmtree(delete_path)
+          break
+        except OSError:
+          cloudlog.exception(f"issue deleting {delete_path}")
+      exit_event.wait(.1)
+    else:
+      exit_event.wait(30)
+
+
+def main():
+  deleter_thread(threading.Event())
+
+
+if __name__ == "__main__":
+  main()
@@ -0,0 +1,42 @@
+#include "system/loggerd/encoder/encoder.h"
+
+VideoEncoder::VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height)
+    : encoder_info(encoder_info), in_width(in_width), in_height(in_height) {
+
+  out_width = encoder_info.frame_width > 0 ? encoder_info.frame_width : in_width;
+  out_height = encoder_info.frame_height > 0 ? encoder_info.frame_height : in_height;
+
+  pm.reset(new PubMaster(std::vector{encoder_info.publish_name}));
+}
+
+void VideoEncoder::publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra,
+                                     unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat) {
+  MessageBuilder msg;
+  auto event = msg.initEvent(true);
+  auto edat = (event.*(encoder_info.init_encode_data_func))();
+  auto edata = edat.initIdx();
+  struct timespec ts;
+  timespec_get(&ts, TIME_UTC);
+  edat.setUnixTimestampNanos((uint64_t)ts.tv_sec*1000000000 + ts.tv_nsec);
+  edata.setFrameId(extra.frame_id);
+  edata.setTimestampSof(extra.timestamp_sof);
+  edata.setTimestampEof(extra.timestamp_eof);
+  edata.setType(encoder_info.get_settings(in_width).encode_type);
+  edata.setEncodeId(cnt++);
+  edata.setSegmentNum(segment_num);
+  edata.setSegmentId(idx);
+  edata.setFlags(flags);
+  edata.setLen(dat.size());
+  edat.adoptData(msg.getOrphanage().referenceExternalData(dat));
+  edat.setWidth(out_width);
+  edat.setHeight(out_height);
+  if (flags & V4L2_BUF_FLAG_KEYFRAME) edat.setHeader(header);
+
+  uint32_t bytes_size = capnp::computeSerializedSizeInWords(msg) * sizeof(capnp::word);
+  if (msg_cache.size() < bytes_size) {
+    msg_cache.resize(bytes_size);
+  }
+  kj::ArrayOutputStream output_stream(kj::ArrayPtr<capnp::byte>(msg_cache.data(), bytes_size));
+  capnp::writeMessage(output_stream, msg);
+  pm->send(encoder_info.publish_name, msg_cache.data(), bytes_size);
+}
@@ -0,0 +1,44 @@
+#pragma once
+
+// has to be in this order
+#ifdef __linux__
+#include <linux/v4l2-controls.h>
+#include <linux/videodev2.h>
+#else
+#define V4L2_BUF_FLAG_KEYFRAME 8
+#endif
+
+#include <cassert>
+#include <cstdint>
+#include <memory>
+#include <thread>
+#include <vector>
+
+#include "openpilot/cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionipc.h"
+#include "common/queue.h"
+#include "system/loggerd/loggerd.h"
+
+class VideoEncoder {
+public:
+  VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  virtual ~VideoEncoder() {}
+  virtual int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) = 0;
+  virtual void encoder_open() = 0;
+  virtual void encoder_close() = 0;
+  virtual void set_bitrate(int bitrate) = 0;
+  virtual void request_keyframe() = 0;
+
+  void publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
+
+protected:
+  int in_width, in_height;
+  int out_width, out_height;
+  const EncoderInfo encoder_info;
+
+private:
+  // total frames encoded
+  int cnt = 0;
+  std::unique_ptr<PubMaster> pm;
+  std::vector<capnp::byte> msg_cache;
+};
@@ -0,0 +1,156 @@
+#include "system/loggerd/encoder/ffmpeg_encoder.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+
+#define __STDC_CONSTANT_MACROS
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+}
+
+#include "common/swaglog.h"
+#include "common/util.h"
+#include "common/yuv.h"
+
+const int env_debug_encoder = (getenv("DEBUG_ENCODER") != NULL) ? atoi(getenv("DEBUG_ENCODER")) : 0;
+
+FfmpegEncoder::FfmpegEncoder(const EncoderInfo &encoder_info, int in_width, int in_height)
+    : VideoEncoder(encoder_info, in_width, in_height) {
+  frame = av_frame_alloc();
+  assert(frame);
+  frame->format = AV_PIX_FMT_YUV420P;
+  frame->width = out_width;
+  frame->height = out_height;
+  frame->linesize[0] = out_width;
+  frame->linesize[1] = out_width/2;
+  frame->linesize[2] = out_width/2;
+
+  convert_buf.resize(in_width * in_height * 3 / 2);
+
+  if (in_width != out_width || in_height != out_height) {
+    downscale_buf.resize(out_width * out_height * 3 / 2);
+  }
+}
+
+FfmpegEncoder::~FfmpegEncoder() {
+  encoder_close();
+  av_frame_free(&frame);
+}
+
+void FfmpegEncoder::encoder_open() {
+  auto codec_id = encoder_info.get_settings(in_width).encode_type == cereal::EncodeIndex::Type::QCAMERA_H264
+                      ? AV_CODEC_ID_H264
+                      : AV_CODEC_ID_FFVHUFF;
+  const AVCodec *codec = avcodec_find_encoder(codec_id);
+
+  this->codec_ctx = avcodec_alloc_context3(codec);
+  assert(this->codec_ctx);
+  this->codec_ctx->width = frame->width;
+  this->codec_ctx->height = frame->height;
+  this->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
+  this->codec_ctx->time_base = (AVRational){ 1, encoder_info.fps };
+  int err = avcodec_open2(this->codec_ctx, codec, NULL);
+  assert(err >= 0);
+
+  is_open = true;
+  segment_num++;
+  counter = 0;
+}
+
+void FfmpegEncoder::encoder_close() {
+  if (!is_open) return;
+
+  avcodec_free_context(&codec_ctx);
+  is_open = false;
+}
+
+void FfmpegEncoder::set_bitrate(int bitrate) {
+  LOGE("adaptive bitrate is not supported for ffmpeg encoder %s", encoder_info.publish_name);
+}
+
+void FfmpegEncoder::request_keyframe() {
+  LOGE("keyframe request is not supported for ffmpeg encoder %s", encoder_info.publish_name);
+}
+
+int FfmpegEncoder::encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) {
+  assert(buf->width == this->in_width);
+  assert(buf->height == this->in_height);
+
+  uint8_t *cy = convert_buf.data();
+  uint8_t *cu = cy + in_width * in_height;
+  uint8_t *cv = cu + (in_width / 2) * (in_height / 2);
+  yuv::nv12_to_i420(buf->y, buf->stride,
+                    buf->uv, buf->stride,
+                    cy, in_width,
+                    cu, in_width/2,
+                    cv, in_width/2,
+                    in_width, in_height);
+
+  if (downscale_buf.size() > 0) {
+    uint8_t *out_y = downscale_buf.data();
+    uint8_t *out_u = out_y + frame->width * frame->height;
+    uint8_t *out_v = out_u + (frame->width / 2) * (frame->height / 2);
+    yuv::i420_scale(cy, in_width,
+                    cu, in_width/2,
+                    cv, in_width/2,
+                    in_width, in_height,
+                    out_y, frame->width,
+                    out_u, frame->width/2,
+                    out_v, frame->width/2,
+                    frame->width, frame->height);
+    frame->data[0] = out_y;
+    frame->data[1] = out_u;
+    frame->data[2] = out_v;
+  } else {
+    frame->data[0] = cy;
+    frame->data[1] = cu;
+    frame->data[2] = cv;
+  }
+  frame->pts = counter*50*1000; // 50ms per frame
+
+  int ret = counter;
+
+  int err = avcodec_send_frame(this->codec_ctx, frame);
+  if (err < 0) {
+    LOGE("avcodec_send_frame error %d", err);
+    ret = -1;
+  }
+
+  AVPacket pkt = {};
+  pkt.data = NULL;
+  pkt.size = 0;
+  while (ret >= 0) {
+    err = avcodec_receive_packet(this->codec_ctx, &pkt);
+    if (err == AVERROR_EOF) {
+      break;
+    } else if (err == AVERROR(EAGAIN)) {
+      // Encoder might need a few frames on startup to get started. Keep going
+      ret = 0;
+      break;
+    } else if (err < 0) {
+      LOGE("avcodec_receive_packet error %d", err);
+      ret = -1;
+      break;
+    }
+
+    if (env_debug_encoder) {
+      printf("%20s got %8d bytes flags %8x idx %4d id %8d\n", encoder_info.publish_name, pkt.size, pkt.flags, counter, extra->frame_id);
+    }
+
+    publisher_publish(segment_num, counter, *extra,
+      (pkt.flags & AV_PKT_FLAG_KEY) ? V4L2_BUF_FLAG_KEYFRAME : 0,
+      kj::arrayPtr<capnp::byte>(pkt.data, (size_t)0), // TODO: get the header
+      kj::arrayPtr<capnp::byte>(pkt.data, pkt.size));
+
+    counter++;
+  }
+  av_packet_unref(&pkt);
+  return ret;
+}
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+}
+
+#include "system/loggerd/encoder/encoder.h"
+#include "system/loggerd/loggerd.h"
+
+class FfmpegEncoder : public VideoEncoder {
+public:
+  FfmpegEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  ~FfmpegEncoder();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open();
+  void encoder_close();
+  void set_bitrate(int bitrate);
+  void request_keyframe();
+
+private:
+  int segment_num = -1;
+  int counter = 0;
+  bool is_open = false;
+
+  AVCodecContext *codec_ctx;
+  AVFrame *frame = NULL;
+  std::vector<uint8_t> convert_buf;
+  std::vector<uint8_t> downscale_buf;
+};
@@ -0,0 +1,115 @@
+#include "system/loggerd/encoder/jpeg_encoder.h"
+
+#include <cassert>
+#include <cstring>
+
+#include "common/swaglog.h"
+
+// Lower qscale = higher quality / bigger files for MJPEG.
+constexpr int MJPEG_QSCALE = 7;
+
+JpegEncoder::JpegEncoder(const std::string &publish_name, int width, int height)
+    : publish_name(publish_name), thumbnail_width(width), thumbnail_height(height) {
+  yuv_buffer.resize((thumbnail_width * thumbnail_height * 3) / 2);
+  pm = std::make_unique<PubMaster>(std::vector{publish_name.c_str()});
+
+  const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_MJPEG);
+  assert(codec);
+
+  codec_ctx = avcodec_alloc_context3(codec);
+  assert(codec_ctx);
+  codec_ctx->width = thumbnail_width;
+  codec_ctx->height = thumbnail_height;
+  codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
+  codec_ctx->time_base = (AVRational){1, 1};
+  codec_ctx->color_range = AVCOL_RANGE_JPEG;
+  codec_ctx->flags |= AV_CODEC_FLAG_QSCALE;
+  codec_ctx->global_quality = FF_QP2LAMBDA * MJPEG_QSCALE;
+
+  int err = avcodec_open2(codec_ctx, codec, NULL);
+  assert(err >= 0);
+
+  frame = av_frame_alloc();
+  assert(frame);
+  frame->format = codec_ctx->pix_fmt;
+  frame->width = thumbnail_width;
+  frame->height = thumbnail_height;
+  frame->linesize[0] = thumbnail_width;
+  frame->linesize[1] = thumbnail_width / 2;
+  frame->linesize[2] = thumbnail_width / 2;
+  frame->color_range = AVCOL_RANGE_JPEG;
+
+  pkt = av_packet_alloc();
+  assert(pkt);
+}
+
+JpegEncoder::~JpegEncoder() {
+  av_packet_free(&pkt);
+  av_frame_free(&frame);
+  avcodec_free_context(&codec_ctx);
+}
+
+void JpegEncoder::pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra) {
+  generateThumbnail(buf->y, buf->uv, buf->width, buf->height, buf->stride);
+
+  MessageBuilder msg;
+  auto thumbnaild = msg.initEvent().initThumbnail();
+  thumbnaild.setFrameId(extra.frame_id);
+  thumbnaild.setTimestampEof(extra.timestamp_eof);
+  thumbnaild.setThumbnail({out_buffer.data(), out_buffer.size()});
+
+  pm->send(publish_name.c_str(), msg);
+}
+
+void JpegEncoder::generateThumbnail(const uint8_t *y_addr, const uint8_t *uv_addr, int width, int height, int stride) {
+  int downscale = width / thumbnail_width;
+  assert(downscale * thumbnail_height == height);
+
+  uint8_t *y_plane = yuv_buffer.data();
+  uint8_t *u_plane = y_plane + thumbnail_width * thumbnail_height;
+  uint8_t *v_plane = u_plane + (thumbnail_width * thumbnail_height) / 4;
+  {
+    // subsampled conversion from nv12 to yuv420p
+    for (int hy = 0; hy < thumbnail_height / 2; hy++) {
+      for (int hx = 0; hx < thumbnail_width / 2; hx++) {
+        int ix = hx * downscale + (downscale - 1) / 2;
+        int iy = hy * downscale + (downscale - 1) / 2;
+        y_plane[(hy * 2 + 0) * thumbnail_width + (hx * 2 + 0)] = y_addr[(iy * 2 + 0) * stride + ix * 2 + 0];
+        y_plane[(hy * 2 + 0) * thumbnail_width + (hx * 2 + 1)] = y_addr[(iy * 2 + 0) * stride + ix * 2 + 1];
+        y_plane[(hy * 2 + 1) * thumbnail_width + (hx * 2 + 0)] = y_addr[(iy * 2 + 1) * stride + ix * 2 + 0];
+        y_plane[(hy * 2 + 1) * thumbnail_width + (hx * 2 + 1)] = y_addr[(iy * 2 + 1) * stride + ix * 2 + 1];
+        u_plane[hy * thumbnail_width / 2 + hx] = uv_addr[iy * stride + ix * 2 + 0];
+        v_plane[hy * thumbnail_width / 2 + hx] = uv_addr[iy * stride + ix * 2 + 1];
+      }
+    }
+  }
+
+  compressToJpeg(y_plane, u_plane, v_plane);
+}
+
+void JpegEncoder::compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane) {
+  frame->data[0] = y_plane;
+  frame->data[1] = u_plane;
+  frame->data[2] = v_plane;
+  // Required for MJPEG qscale to take effect (global_quality alone is not enough).
+  frame->quality = FF_QP2LAMBDA * MJPEG_QSCALE;
+  frame->pts = AV_NOPTS_VALUE;
+
+  int err = avcodec_send_frame(codec_ctx, frame);
+  if (err < 0) {
+    LOGE("thumbnail avcodec_send_frame error %d", err);
+    out_buffer.clear();
+    return;
+  }
+
+  av_packet_unref(pkt);
+  err = avcodec_receive_packet(codec_ctx, pkt);
+  if (err < 0) {
+    LOGE("thumbnail avcodec_receive_packet error %d", err);
+    out_buffer.clear();
+    return;
+  }
+
+  out_buffer.assign(pkt->data, pkt->data + pkt->size);
+  av_packet_unref(pkt);
+}
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "openpilot/cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionbuf.h"
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+}
+
+class JpegEncoder {
+public:
+  JpegEncoder(const std::string &publish_name, int width, int height);
+  ~JpegEncoder();
+  void pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra);
+
+private:
+  void generateThumbnail(const uint8_t *y, const uint8_t *uv, int width, int height, int stride);
+  void compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane);
+
+  int thumbnail_width;
+  int thumbnail_height;
+  std::string publish_name;
+  std::vector<uint8_t> yuv_buffer;
+  std::vector<uint8_t> out_buffer;
+  std::unique_ptr<PubMaster> pm;
+
+  AVCodecContext *codec_ctx = nullptr;
+  AVFrame *frame = nullptr;
+  AVPacket *pkt = nullptr;
+};
@@ -0,0 +1,393 @@
+#include "system/loggerd/encoder/v4l_decoder.h"
+
+#include <assert.h>
+#include <cerrno>
+#include <climits>
+#include <linux/v4l2-controls.h>
+#include <linux/videodev2.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+
+
+#include "common/swaglog.h"
+#include "common/util.h"
+
+constexpr int OFFLINE_CORE_PLACEMENT_RATE = 80 << 16;
+
+// echo "0xFFFF" > /sys/kernel/debug/msm_vidc/debug_level
+
+static void copyBuffer(VisionBuf *src_buf, VisionBuf *dst_buf) {
+  // Copy Y plane
+  memcpy(dst_buf->y, src_buf->y, src_buf->height * src_buf->stride);
+  // Copy UV plane
+  memcpy(dst_buf->uv, src_buf->uv, src_buf->height / 2 * src_buf->stride);
+}
+
+static void request_buffers(int fd, v4l2_buf_type buf_type, unsigned int count) {
+  struct v4l2_requestbuffers reqbuf = {
+    .count = count,
+    .type = buf_type,
+    .memory = V4L2_MEMORY_USERPTR
+  };
+  util::safe_ioctl(fd, VIDIOC_REQBUFS, &reqbuf, "VIDIOC_REQBUFS failed");
+}
+
+V4LDecoder::~V4LDecoder() {
+  if (fd > 0) {
+    close(fd);
+  }
+}
+
+bool V4LDecoder::init(const char* dev, size_t width, size_t height, uint64_t codec,
+                   bool direct_mode, uint32_t capture_fourcc) {
+  LOG("Initializing msm_vidc device %s", dev);
+  this->w = width;
+  this->h = height;
+  this->direct = direct_mode;
+  this->capture_format = capture_fourcc;
+  this->fd = open(dev, O_RDWR | O_NONBLOCK, 0);
+  if (fd < 0) {
+    LOGE("failed to open video device %s", dev);
+    return false;
+  }
+  subscribeEvents();
+  v4l2_buf_type out_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+  setPlaneFormat(out_type, codec); // Also allocates the output buffers
+  setFPS(FPS);
+  if (direct) {
+    struct v4l2_control ctrls[] = {
+      // A finite real-time load lets the driver place decode and encode on separate cores.
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE, .value = OFFLINE_CORE_PLACEMENT_RATE },
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY, .value = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_ENABLE },
+    };
+    for (auto ctrl : ctrls) {
+      util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL offline decode failed");
+    }
+  }
+  request_buffers(fd, out_type, OUTPUT_BUFFER_COUNT);
+  util::safe_ioctl(fd, VIDIOC_STREAMON, &out_type, "VIDIOC_STREAMON OUTPUT failed");
+  restartCapture();
+  pfd = {fd, POLLIN | POLLOUT | POLLWRNORM | POLLRDNORM | POLLPRI, 0};
+
+  this->initialized = true;
+  return true;
+}
+
+VisionBuf* V4LDecoder::decodeFrame(AVPacket *pkt, VisionBuf *buf) {
+  assert(initialized && !direct && pkt != nullptr && buf != nullptr);
+  bool queued = false;
+  while (true) {
+    if (!queued) queued = queuePacket(pkt, 0);
+    V4LDecodedFrame frame;
+    if (!pump(frame, -1)) return nullptr;
+    if (!frame.buf) continue;
+
+    VisionBuf *decoded = frame.buf;
+    copyBuffer(decoded, buf);
+    releaseFrame(decoded);
+    return buf;
+  }
+}
+
+void V4LDecoder::releaseFrame(VisionBuf *buf) {
+  assert(buf >= cap_bufs && buf < cap_bufs + CAPTURE_BUFFER_COUNT);
+  queueCaptureBuffer(buf - cap_bufs);
+}
+
+bool V4LDecoder::queuePacket(const AVPacket *pkt, uint64_t token) {
+  int buf_index = getBufferUnlocked();
+  return buf_index >= 0 && sendPacket(buf_index, pkt, token);
+}
+
+bool V4LDecoder::pump(V4LDecodedFrame &frame, int timeout_ms) {
+  frame = {};
+  int rc;
+  while (true) {
+    rc = poll(&pfd, 1, timeout_ms);
+    if (rc < 0) {
+      if (errno == EINTR) continue;
+      LOGE("poll() error: %d", errno);
+      return false;
+    }
+    break;
+  }
+
+  if (rc == 0) return true;
+
+  int result;
+
+  // Port changes must be handled before capture DQ so no old-format surface is
+  // handed to a client after the driver has requested a capture flush.
+  while ((result = handleEvent()) > 0) {}
+  if (result < 0) return false;
+
+  while ((result = handleOutput()) > 0) {}
+  if (result < 0) return false;
+
+  result = handleCapture(&frame);
+  return result >= 0;
+}
+
+int V4LDecoder::handleCapture(V4LDecodedFrame *frame) {
+  struct v4l2_buffer buf = {0};
+  struct v4l2_plane planes[1] = {0};
+  buf.type          = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+  buf.memory        = V4L2_MEMORY_USERPTR;
+  buf.m.planes      = planes;
+  buf.length        = 1;
+  int err = HANDLE_EINTR(ioctl(this->fd, VIDIOC_DQBUF, &buf));
+  if (err < 0 && errno == EAGAIN) return 0;
+  if (err < 0) {
+    LOGE("VIDIOC_DQBUF CAPTURE failed: %d", errno);
+    return -1;
+  }
+
+  const bool has_payload = buf.m.planes[0].bytesused != 0;
+  const bool eos = (buf.flags & V4L2_QCOM_BUF_FLAG_EOS) != 0;
+
+  frame->buf = nullptr;
+  if (!reconfigure_pending && has_payload) {
+    frame->buf = &cap_bufs[buf.index];
+    frame->token = (uint64_t)buf.timestamp.tv_sec * 1000000ULL + buf.timestamp.tv_usec;
+  } else if (!reconfigure_pending && !eos) {
+    queueCaptureBuffer(buf.index);
+  }
+
+  return 1;
+}
+
+bool V4LDecoder::subscribeEvents() {
+  for (uint32_t event : subscriptions) {
+    struct v4l2_event_subscription sub = { .type = event};
+    util::safe_ioctl(fd, VIDIOC_SUBSCRIBE_EVENT, &sub, "VIDIOC_SUBSCRIBE_EVENT failed");
+  }
+  return true;
+}
+
+bool V4LDecoder::setPlaneFormat(enum v4l2_buf_type type, uint32_t fourcc) {
+  struct v4l2_format fmt = {.type = type};
+  struct v4l2_pix_format_mplane *pix = &fmt.fmt.pix_mp;
+  *pix = {
+    .width = (__u32)this->w,
+    .height = (__u32)this->h,
+    .pixelformat = fourcc
+  };
+  util::safe_ioctl(fd, VIDIOC_S_FMT, &fmt, "VIDIOC_S_FMT failed");
+  if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
+    this->out_buf_size = pix->plane_fmt[0].sizeimage;
+    for (int i = 0; i < OUTPUT_BUFFER_COUNT; i++) {
+      this->out_bufs[i].allocate(this->out_buf_size);
+      this->out_buf_flag[i] = false;
+    }
+    LOGD("Set output buffer size to %d, count %d, addr %p", this->out_buf_size, OUTPUT_BUFFER_COUNT, this->out_bufs[0].addr);
+  } else if (type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
+    request_buffers(this->fd, type, CAPTURE_BUFFER_COUNT);
+    util::safe_ioctl(fd, VIDIOC_G_FMT, &fmt, "VIDIOC_G_FMT failed");
+    const __u32 y_size    = pix->plane_fmt[0].sizeimage;
+    const __u32 y_stride  = pix->plane_fmt[0].bytesperline;
+    for (size_t i = 0; i < CAPTURE_BUFFER_COUNT; i++) {
+      size_t uv_offset = (size_t)y_stride * pix->height;
+      size_t required = uv_offset + (y_stride * pix->height / 2); // enough for Y + UV. For linear NV12, UV plane starts at y_stride * height.
+      size_t alloc_size = std::max<size_t>(y_size, required);
+      this->cap_bufs[i].allocate(alloc_size);
+      this->cap_bufs[i].init_yuv(pix->width, pix->height, y_stride, uv_offset);
+    }
+    LOGD("Set capture buffer size to %d, count %d, addr %p, extradata size %d",
+      pix->plane_fmt[0].sizeimage, CAPTURE_BUFFER_COUNT, this->cap_bufs[0].addr, pix->plane_fmt[1].sizeimage);
+  }
+  return true;
+}
+
+bool V4LDecoder::setFPS(uint32_t fps) {
+  struct v4l2_streamparm streamparam = {
+    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+  };
+  streamparam.parm.output.timeperframe = {1, fps};
+  util::safe_ioctl(fd, VIDIOC_S_PARM, &streamparam, "VIDIOC_S_PARM failed");
+  return true;
+}
+
+bool V4LDecoder::restartCapture() {
+  // stop if already initialized
+  enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+  if (this->initialized) {
+    LOGD("Restarting capture, flushing buffers...");
+    util::safe_ioctl(this->fd, VIDIOC_STREAMOFF, &type, "VIDIOC_STREAMOFF CAPTURE failed");
+    struct v4l2_requestbuffers reqbuf = {.type = type, .memory = V4L2_MEMORY_USERPTR};
+    util::safe_ioctl(this->fd, VIDIOC_REQBUFS, &reqbuf, "VIDIOC_REQBUFS failed");
+    for (size_t i = 0; i < CAPTURE_BUFFER_COUNT; ++i) {
+      this->cap_bufs[i].free();
+      cap_bufs[i].~VisionBuf();
+      new (&cap_bufs[i]) VisionBuf();
+    }
+  }
+  // setup, start and queue capture buffers
+  setDBP();
+  setPlaneFormat(type, capture_format);
+  if (direct) {
+    struct v4l2_control ctrl = {
+      .id = V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE,
+      .value = OFFLINE_CORE_PLACEMENT_RATE,
+    };
+    util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL placement decode failed");
+  }
+  util::safe_ioctl(this->fd, VIDIOC_STREAMON, &type, "VIDIOC_STREAMON CAPTURE failed");
+  for (size_t i = 0; i < CAPTURE_BUFFER_COUNT; ++i) {
+    queueCaptureBuffer(i);
+  }
+  if (direct) {
+    struct v4l2_control ctrl = {
+      .id = V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE,
+      .value = INT_MAX,
+    };
+    util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL turbo decode failed");
+  }
+
+  return true;
+}
+
+bool V4LDecoder::queueCaptureBuffer(int i) {
+  struct v4l2_buffer buf = {0};
+  struct v4l2_plane planes[1] = {0};
+
+  buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+  buf.memory = V4L2_MEMORY_USERPTR;
+  buf.index = i;
+  buf.m.planes = planes;
+  buf.length = 1;
+  // decoded frame plane
+  planes[0].m.userptr     = (unsigned long)this->cap_bufs[i].addr; // no security
+  planes[0].length        = this->cap_bufs[i].len;
+  planes[0].reserved[0]   = this->cap_bufs[i].fd; // ION fd
+  planes[0].reserved[1]   = 0;
+  planes[0].bytesused     = this->cap_bufs[i].len;
+  planes[0].data_offset   = 0;
+  util::safe_ioctl(this->fd, VIDIOC_QBUF, &buf, "VIDIOC_QBUF failed");
+  return true;
+}
+
+bool V4LDecoder::queueOutputBuffer(int i, size_t size, uint64_t token) {
+  struct v4l2_buffer buf = {0};
+  struct v4l2_plane planes[1] = {0};
+
+  buf.type                = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+  buf.memory              = V4L2_MEMORY_USERPTR;
+  buf.index               = i;
+  buf.flags               = V4L2_BUF_FLAG_TIMESTAMP_COPY;
+  buf.timestamp.tv_sec    = token / 1000000ULL;
+  buf.timestamp.tv_usec   = token % 1000000ULL;
+  buf.m.planes            = planes;
+  buf.length              = 1;
+  // decoded frame plane
+  planes[0].m.userptr     = (unsigned long)this->out_bufs[i].addr;
+  planes[0].length        = this->out_buf_size;
+  planes[0].reserved[0]   = this->out_bufs[i].fd; // ION fd
+  planes[0].reserved[1]   = 0;
+  planes[0].bytesused     = size;
+  planes[0].data_offset   = 0;
+  assert(this->out_buf_size % 4096 == 0);               // ditto for size
+
+  util::safe_ioctl(this->fd, VIDIOC_QBUF, &buf, "VIDIOC_QBUF failed");
+  this->out_buf_flag[i] = true; // mark as queued
+  return true;
+}
+
+bool V4LDecoder::setDBP() {
+  struct v4l2_ext_control control[2] = {0};
+  struct v4l2_ext_controls controls = {0};
+  control[0].id           = V4L2_CID_MPEG_VIDC_VIDEO_STREAM_OUTPUT_MODE;
+  control[0].value        = 1; // V4L2_CID_MPEG_VIDC_VIDEO_STREAM_OUTPUT_SECONDARY
+  control[1].id           = V4L2_CID_MPEG_VIDC_VIDEO_DPB_COLOR_FORMAT;
+  control[1].value        = 0; // V4L2_MPEG_VIDC_VIDEO_DPB_COLOR_FMT_NONE
+  controls.count          = 2;
+  controls.ctrl_class     = V4L2_CTRL_CLASS_MPEG;
+  controls.controls       = control;
+  util::safe_ioctl(fd, VIDIOC_S_EXT_CTRLS, &controls, "VIDIOC_S_EXT_CTRLS failed");
+  return true;
+}
+
+bool V4LDecoder::sendPacket(int buf_index, const AVPacket *pkt, uint64_t token) {
+  assert(buf_index >= 0 && buf_index < OUTPUT_BUFFER_COUNT);
+  assert(pkt != nullptr && pkt->data != nullptr && pkt->size > 0);
+  assert((size_t)pkt->size <= (size_t)this->out_buf_size);
+  // Prepare output buffer
+  uint8_t * data = (uint8_t *)this->out_bufs[buf_index].addr;
+  memcpy(data, pkt->data, pkt->size);
+  queueOutputBuffer(buf_index, pkt->size, token);
+  return true;
+}
+
+int V4LDecoder::getBufferUnlocked() {
+  for (int i = 0; i < OUTPUT_BUFFER_COUNT; i++) {
+    if (!out_buf_flag[i]) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+
+int V4LDecoder::handleOutput() {
+  struct v4l2_buffer buf = {0};
+  struct v4l2_plane planes[1];
+  buf.type      = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+  buf.memory    = V4L2_MEMORY_USERPTR;
+  buf.m.planes  = planes;
+  buf.length    = 1;
+  int err = HANDLE_EINTR(ioctl(this->fd, VIDIOC_DQBUF, &buf));
+  if (err < 0 && errno == EAGAIN) return 0;
+  if (err < 0) {
+    LOGE("VIDIOC_DQBUF OUTPUT failed: %d", errno);
+    return -1;
+  }
+  this->out_buf_flag[buf.index] = false; // mark as not queued
+  return 1;
+}
+
+int V4LDecoder::handleEvent() {
+  // dequeue event
+  struct v4l2_event event = {0};
+  int err = HANDLE_EINTR(ioctl(this->fd, VIDIOC_DQEVENT, &event));
+  if (err < 0 && (errno == EAGAIN || errno == ENOENT)) return 0;
+  if (err < 0) {
+    LOGE("VIDIOC_DQEVENT failed: %d", errno);
+    return -1;
+  }
+  switch (event.type) {
+    case V4L2_EVENT_MSM_VIDC_PORT_SETTINGS_CHANGED_INSUFFICIENT: {
+      unsigned int *ptr     = (unsigned int *)event.u.data;
+      unsigned int height   = ptr[0];
+      unsigned int width    = ptr[1];
+      this->w               = width;
+      this->h               = height;
+      LOGD("Port Reconfig received insufficient, new size %ux%u, flushing capture bufs...", width, height); // This is normal
+      struct v4l2_decoder_cmd dec;
+      dec.flags = V4L2_QCOM_CMD_FLUSH_CAPTURE;
+      dec.cmd = V4L2_QCOM_CMD_FLUSH;
+      util::safe_ioctl(this->fd, VIDIOC_DECODER_CMD, &dec, "VIDIOC_DECODER_CMD FLUSH_CAPTURE failed");
+      this->reconfigure_pending = true;
+      LOGD("Waiting for flush done event to reconfigure capture queue");
+      break;
+    }
+
+    case V4L2_EVENT_MSM_VIDC_FLUSH_DONE: {
+      unsigned int *ptr   = (unsigned int *)event.u.data;
+      unsigned int flags  = ptr[0];
+      if (flags & V4L2_QCOM_CMD_FLUSH_CAPTURE) {
+        if (this->reconfigure_pending) {
+          this->restartCapture();
+          this->reconfigure_pending = false;
+        }
+      }
+      break;
+    }
+    default:
+      break;
+  }
+  return 1;
+}
+
+void V4LDecoder::sendEOS() {
+  struct v4l2_decoder_cmd command = { .cmd = V4L2_DEC_CMD_STOP };
+  util::safe_ioctl(fd, VIDIOC_DECODER_CMD, &command, "VIDIOC_DECODER_CMD STOP failed");
+}
@@ -0,0 +1,105 @@
+#pragma once
+
+#include <linux/videodev2.h>
+#include <poll.h>
+
+#include "msgq/visionipc/visionbuf.h"
+
+extern "C" {
+  #include <libavcodec/avcodec.h>
+  #include <libavformat/avformat.h>
+}
+
+#define V4L2_EVENT_MSM_VIDC_START (V4L2_EVENT_PRIVATE_START + 0x00001000)
+#define V4L2_EVENT_MSM_VIDC_FLUSH_DONE (V4L2_EVENT_MSM_VIDC_START + 1)
+#define V4L2_EVENT_MSM_VIDC_PORT_SETTINGS_CHANGED_INSUFFICIENT (V4L2_EVENT_MSM_VIDC_START + 3)
+#ifndef V4L2_CID_MPEG_MSM_VIDC_BASE
+#define V4L2_CID_MPEG_MSM_VIDC_BASE 0x00992000
+#endif
+#ifndef V4L2_CID_MPEG_VIDC_VIDEO_DPB_COLOR_FORMAT
+#define V4L2_CID_MPEG_VIDC_VIDEO_DPB_COLOR_FORMAT (V4L2_CID_MPEG_MSM_VIDC_BASE + 44)
+#endif
+#ifndef V4L2_CID_MPEG_VIDC_VIDEO_STREAM_OUTPUT_MODE
+#define V4L2_CID_MPEG_VIDC_VIDEO_STREAM_OUTPUT_MODE (V4L2_CID_MPEG_MSM_VIDC_BASE + 22)
+#endif
+#ifndef V4L2_PIX_FMT_NV12_UBWC
+#define V4L2_PIX_FMT_NV12_UBWC v4l2_fourcc('Q', '1', '2', '8')
+#endif
+#ifndef V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY
+#define V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY (V4L2_CID_MPEG_MSM_VIDC_BASE + 52)
+#define V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_ENABLE 0
+#define V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE 1
+#endif
+#ifndef V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE
+#define V4L2_CID_MPEG_VIDC_VIDEO_OPERATING_RATE (V4L2_CID_MPEG_MSM_VIDC_BASE + 53)
+#endif
+#define V4L2_QCOM_CMD_FLUSH_CAPTURE (1 << 1)
+#define V4L2_QCOM_CMD_FLUSH (4)
+#ifndef V4L2_QCOM_BUF_FLAG_EOS
+#define V4L2_QCOM_BUF_FLAG_EOS 0x02000000
+#endif
+
+#define OUTPUT_BUFFER_COUNT 	8
+#define CAPTURE_BUFFER_COUNT 	16
+#define FPS 									20
+
+struct V4LDecodedFrame {
+  VisionBuf *buf = nullptr;
+  uint64_t token = 0;
+};
+
+class V4LDecoder {
+public:
+  static constexpr const char *DEVICE = "/dev/video32";
+
+  V4LDecoder() = default;
+  ~V4LDecoder();
+
+  bool init(const char* dev, size_t width, size_t height, uint64_t codec,
+            bool direct_mode = false, uint32_t capture_fourcc = V4L2_PIX_FMT_NV12);
+  VisionBuf* decodeFrame(AVPacket* pkt, VisionBuf* buf);
+  // queuePacket() and pump() are single-threaded. releaseFrame() may be called
+  // from a consumer thread after a direct capture surface is no longer needed.
+  bool queuePacket(const AVPacket *pkt, uint64_t token);
+  bool pump(V4LDecodedFrame &frame, int timeout_ms);
+  void releaseFrame(VisionBuf *buf);
+  void sendEOS();
+  size_t maxPacketSize() const { return out_buf_size; }
+
+  AVFormatContext* avctx = nullptr;
+  int fd = 0;
+
+private:
+  bool initialized = false;
+  bool reconfigure_pending = false;
+  bool direct = false;
+  uint32_t capture_format = V4L2_PIX_FMT_NV12;
+
+  VisionBuf out_bufs[OUTPUT_BUFFER_COUNT];    // Distinct dma-buf per in-flight packet
+  VisionBuf cap_bufs[CAPTURE_BUFFER_COUNT];   // Capture (output) buffers
+
+  size_t w = 0, h = 0;
+  int out_buf_size = 0;
+
+  bool out_buf_flag[OUTPUT_BUFFER_COUNT] = {false};
+
+  const int subscriptions[2] = {
+    V4L2_EVENT_MSM_VIDC_FLUSH_DONE,
+    V4L2_EVENT_MSM_VIDC_PORT_SETTINGS_CHANGED_INSUFFICIENT
+  };
+
+  struct pollfd pfd = {};
+
+  bool subscribeEvents();
+  bool setPlaneFormat(v4l2_buf_type type, uint32_t fourcc);
+  bool setFPS(uint32_t fps);
+  bool restartCapture();
+  bool queueCaptureBuffer(int i);
+  bool queueOutputBuffer(int i, size_t size, uint64_t token);
+  bool setDBP();
+  bool sendPacket(int buf_index, const AVPacket* pkt, uint64_t token);
+  int getBufferUnlocked();
+  int handleCapture(V4LDecodedFrame *frame);
+  int handleOutput();
+  int handleEvent();
+};
@@ -0,0 +1,392 @@
+#include <cassert>
+#include <string>
+#include <sys/ioctl.h>
+#include <poll.h>
+#include <utility>
+
+#include "system/loggerd/encoder/v4l_encoder.h"
+#include "common/util.h"
+#include "common/timing.h"
+
+#include <media/msm_media_info.h>
+
+// has to be in this order
+#include <linux/v4l2-controls.h>
+#include <linux/videodev2.h>
+#define V4L2_QCOM_BUF_FLAG_CODECCONFIG 0x00020000
+#define V4L2_QCOM_BUF_FLAG_EOS 0x02000000
+
+/*
+  kernel debugging:
+  echo 0xff > /sys/module/videobuf2_core/parameters/debug
+  echo 0x7fffffff > /sys/kernel/debug/msm_vidc/debug_level
+  echo 0xff > /sys/devices/platform/soc/aa00000.qcom,vidc/video4linux/video33/dev_debug
+*/
+const int env_debug_encoder = (getenv("DEBUG_ENCODER") != NULL) ? atoi(getenv("DEBUG_ENCODER")) : 0;
+
+static void dequeue_buffer(int fd, v4l2_buf_type buf_type, unsigned int *index=NULL, unsigned int *bytesused=NULL, unsigned int *flags=NULL, struct timeval *timestamp=NULL) {
+  v4l2_plane plane = {0};
+  v4l2_buffer v4l_buf = {
+    .type = buf_type,
+    .memory = V4L2_MEMORY_USERPTR,
+    .m = { .planes = &plane, },
+    .length = 1,
+  };
+  util::safe_ioctl(fd, VIDIOC_DQBUF, &v4l_buf, "VIDIOC_DQBUF failed");
+
+  if (index) *index = v4l_buf.index;
+  if (bytesused) *bytesused = v4l_buf.m.planes[0].bytesused;
+  if (flags) *flags = v4l_buf.flags;
+  if (timestamp) *timestamp = v4l_buf.timestamp;
+  assert(v4l_buf.m.planes[0].data_offset == 0);
+}
+
+static void queue_buffer(int fd, v4l2_buf_type buf_type, unsigned int index, VisionBuf *buf, struct timeval timestamp={}) {
+  v4l2_plane plane = {
+    .bytesused = (uint32_t)buf->len,
+    .length = (unsigned int)buf->len,
+    .m = { .userptr = (unsigned long)buf->addr, },
+    .reserved = {(unsigned int)buf->fd}
+  };
+
+  v4l2_buffer v4l_buf = {
+    .index = index,
+    .type = buf_type,
+    .flags = V4L2_BUF_FLAG_TIMESTAMP_COPY,
+    .timestamp = timestamp,
+    .memory = V4L2_MEMORY_USERPTR,
+    .m = { .planes = &plane, },
+    .length = 1,
+  };
+  util::safe_ioctl(fd, VIDIOC_QBUF, &v4l_buf, "VIDIOC_QBUF failed");
+}
+
+static void request_buffers(int fd, v4l2_buf_type buf_type, unsigned int count) {
+  struct v4l2_requestbuffers reqbuf = {
+    .count = count,
+    .type = buf_type,
+    .memory = V4L2_MEMORY_USERPTR,
+  };
+  util::safe_ioctl(fd, VIDIOC_REQBUFS, &reqbuf, "VIDIOC_REQBUFS failed");
+}
+
+void V4LEncoder::dequeue_handler(V4LEncoder *e) {
+  std::string dequeue_thread_name = "dq-"+std::string(e->encoder_info.publish_name);
+  util::set_thread_name(dequeue_thread_name.c_str());
+
+  e->segment_num++;
+  uint32_t idx = -1;
+  bool exit = false;
+
+  // POLLIN is capture, POLLOUT is frame. Qualcomm's reference client also
+  // requests the corresponding normal-data bits.
+  struct pollfd pfd;
+  pfd.events = POLLIN | POLLRDNORM | POLLOUT | POLLWRNORM;
+  pfd.fd = e->fd;
+
+  // save the header
+  kj::Array<capnp::byte> header;
+
+  while (!exit) {
+    int rc = poll(&pfd, 1, 1000);
+    if (rc < 0) {
+      if (errno != EINTR) {
+        // TODO: exit encoder?
+        // ignore the error and keep going
+        LOGE("poll failed (%d - %d)", rc, errno);
+      }
+      continue;
+    } else if (rc == 0) {
+      LOGE("encoder dequeue poll timeout");
+      continue;
+    }
+
+    if (env_debug_encoder >= 2) {
+      printf("%20s poll %x at %.2f ms\n", e->encoder_info.publish_name, pfd.revents, millis_since_boot());
+    }
+
+    int frame_id = -1;
+    if (pfd.revents & (POLLIN | POLLRDNORM)) {
+      unsigned int bytesused, flags, index;
+      struct timeval timestamp;
+      dequeue_buffer(e->fd, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, &index, &bytesused, &flags, &timestamp);
+      e->buf_out[index].sync(VISIONBUF_SYNC_FROM_DEVICE);
+      uint8_t *buf = (uint8_t*)e->buf_out[index].addr;
+      int64_t ts = timestamp.tv_sec * 1000000 + timestamp.tv_usec;
+
+      // eof packet, we exit
+      if (flags & V4L2_QCOM_BUF_FLAG_EOS) {
+        exit = true;
+      } else if (flags & V4L2_QCOM_BUF_FLAG_CODECCONFIG) {
+        // save header
+        header = kj::heapArray<capnp::byte>(buf, bytesused);
+        if (e->packet_callback) e->packet_callback(header.begin(), header.size(), ts, true, false);
+      } else {
+        VisionIpcBufExtra extra = e->extras.pop();
+        assert(extra.timestamp_eof/1000 == ts); // stay in sync
+        frame_id = extra.frame_id;
+        ++idx;
+        if (e->packet_callback) {
+          e->packet_callback(buf, bytesused, ts, false, flags & V4L2_BUF_FLAG_KEYFRAME);
+        } else {
+          e->publisher_publish(e->segment_num, idx, extra, flags, header, kj::arrayPtr<capnp::byte>(buf, bytesused));
+        }
+      }
+
+      if (env_debug_encoder) {
+        printf("%20s got(%d) %6d bytes flags %8x idx %3d/%4d id %8d ts %ld lat %.2f ms (%lu frames free)\n",
+          e->encoder_info.publish_name, index, bytesused, flags, e->segment_num, idx, frame_id, ts, millis_since_boot()-(ts/1000.), e->free_buf_in.size());
+      }
+
+      // requeue the buffer
+      queue_buffer(e->fd, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, index, &e->buf_out[index]);
+    }
+
+    if (pfd.revents & (POLLOUT | POLLWRNORM)) {
+      unsigned int index;
+      dequeue_buffer(e->fd, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, &index);
+      VisionBuf *input_buf = e->input_bufs[index].exchange(nullptr);
+      if (input_buf && e->input_done_callback) e->input_done_callback(input_buf);
+      e->free_buf_in.push(index);
+    }
+  }
+}
+
+V4LEncoder::V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_height)
+    : V4LEncoder(encoder_info, in_width, in_height, Options{}) {}
+
+V4LEncoder::V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_height, Options options)
+    : VideoEncoder(encoder_info, in_width, in_height), packet_callback(std::move(options.packet_callback)),
+      input_done_callback(std::move(options.input_done_callback)) {
+  fd = HANDLE_EINTR(open("/dev/v4l/by-path/platform-aa00000.qcom_vidc-video-index1", O_RDWR|O_NONBLOCK));
+  assert(fd >= 0);
+
+  struct v4l2_capability cap;
+  util::safe_ioctl(fd, VIDIOC_QUERYCAP, &cap, "VIDIOC_QUERYCAP failed");
+  LOGD("opened encoder device %s %s = %d", cap.driver, cap.card, fd);
+  assert(strcmp((const char *)cap.driver, "msm_vidc_driver") == 0);
+  assert(strcmp((const char *)cap.card, "msm_vidc_venc") == 0);
+
+  EncoderSettings encoder_settings = encoder_info.get_settings(in_width);
+  current_bitrate = encoder_settings.bitrate;
+  bool is_h265 = encoder_settings.encode_type == cereal::EncodeIndex::Type::FULL_H_E_V_C;
+
+  struct v4l2_format fmt_out = {
+    .type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE,
+    .fmt = {
+      .pix_mp = {
+        // downscales are free with v4l
+        .width = (unsigned int)(out_width),
+        .height = (unsigned int)(out_height),
+        .pixelformat = is_h265 ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264,
+        .field = V4L2_FIELD_ANY,
+        .colorspace = V4L2_COLORSPACE_DEFAULT,
+      }
+    }
+  };
+  util::safe_ioctl(fd, VIDIOC_S_FMT, &fmt_out, "VIDIOC_S_FMT failed");
+
+  v4l2_streamparm streamparm = {
+    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+    .parm = {
+      .output = {
+        // TODO: more stuff here? we don't know
+        .timeperframe = {
+          .numerator = 1,
+          .denominator = (unsigned int)encoder_info.fps
+        }
+      }
+    }
+  };
+  util::safe_ioctl(fd, VIDIOC_S_PARM, &streamparm, "VIDIOC_S_PARM failed");
+
+  struct v4l2_format fmt_in = {
+    .type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+    .fmt = {
+      .pix_mp = {
+        .width = (unsigned int)in_width,
+        .height = (unsigned int)in_height,
+        .pixelformat = options.input_format,
+        .field = V4L2_FIELD_ANY,
+        .colorspace = V4L2_COLORSPACE_470_SYSTEM_BG,
+      }
+    }
+  };
+  util::safe_ioctl(fd, VIDIOC_S_FMT, &fmt_in, "VIDIOC_S_FMT failed");
+
+  LOGD("in buffer size %d, out buffer size %d",
+    fmt_in.fmt.pix_mp.plane_fmt[0].sizeimage,
+    fmt_out.fmt.pix_mp.plane_fmt[0].sizeimage);
+
+  // shared ctrls
+  {
+    struct v4l2_control ctrls[] = {
+      { .id = V4L2_CID_MPEG_VIDEO_BITRATE, .value = encoder_settings.bitrate},
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_NUM_P_FRAMES, .value = encoder_settings.gop_size - encoder_settings.b_frames - 1},
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_NUM_B_FRAMES, .value = encoder_settings.b_frames},
+      { .id = V4L2_CID_MPEG_VIDEO_HEADER_MODE, .value = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE},
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_RATE_CONTROL, .value = V4L2_CID_MPEG_VIDC_VIDEO_RATE_CONTROL_VBR_CFR},
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY, .value = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE},
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_IDR_PERIOD, .value = 1},
+    };
+    for (auto ctrl : ctrls) {
+      util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL failed");
+    }
+  }
+  if (options.max_performance) {
+    struct v4l2_control ctrl = {
+      .id = V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY,
+      .value = V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_ENABLE,
+    };
+    util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL offline encode failed");
+  }
+
+  if (is_h265) {
+    struct v4l2_control ctrls[] = {
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE, .value = V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN},
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL, .value = V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5},
+      { .id = V4L2_CID_MPEG_VIDC_VIDEO_VUI_TIMING_INFO, .value = V4L2_MPEG_VIDC_VIDEO_VUI_TIMING_INFO_ENABLED},
+    };
+    for (auto ctrl : ctrls) {
+      util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL failed");
+    }
+  } else {
+    if (encoder_info.is_live) {
+      struct v4l2_control ctrls[] = {
+        { .id = V4L2_CID_MPEG_VIDEO_H264_PROFILE, .value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH},
+        { .id = V4L2_CID_MPEG_VIDEO_H264_LEVEL, .value = V4L2_MPEG_VIDEO_H264_LEVEL_3_1},
+        { .id = V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE, .value = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC},
+        { .id = V4L2_CID_MPEG_VIDC_VIDEO_H264_CABAC_MODEL, .value = V4L2_CID_MPEG_VIDC_VIDEO_H264_CABAC_MODEL_0},
+      };
+      for (auto ctrl : ctrls) {
+        util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL failed");
+      }
+    } else {
+      struct v4l2_control ctrls[] = {
+        { .id = V4L2_CID_MPEG_VIDEO_H264_PROFILE, .value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH},
+        { .id = V4L2_CID_MPEG_VIDEO_H264_LEVEL, .value = V4L2_MPEG_VIDEO_H264_LEVEL_UNKNOWN},
+        { .id = V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE, .value = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC},
+        { .id = V4L2_CID_MPEG_VIDC_VIDEO_H264_CABAC_MODEL, .value = V4L2_CID_MPEG_VIDC_VIDEO_H264_CABAC_MODEL_0},
+      };
+      for (auto ctrl : ctrls) {
+        util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL failed");
+      }
+    }
+
+    struct v4l2_control ctrls[] = {
+      { .id = V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE, .value = V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED},
+      { .id = V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_ALPHA, .value = 0},
+      { .id = V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_BETA, .value = 0},
+      { .id = V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE, .value = 0},
+    };
+    for (auto ctrl : ctrls) {
+      util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl, "VIDIOC_S_CTRL failed");
+    }
+  }
+
+  // allocate buffers
+  request_buffers(fd, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, BUF_OUT_COUNT);
+  request_buffers(fd, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, BUF_IN_COUNT);
+
+  // start encoder
+  v4l2_buf_type buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+  util::safe_ioctl(fd, VIDIOC_STREAMON, &buf_type, "VIDIOC_STREAMON failed");
+  buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+  util::safe_ioctl(fd, VIDIOC_STREAMON, &buf_type, "VIDIOC_STREAMON failed");
+
+  // queue up output buffers
+  for (unsigned int i = 0; i < BUF_OUT_COUNT; i++) {
+    buf_out[i].allocate(fmt_out.fmt.pix_mp.plane_fmt[0].sizeimage);
+    queue_buffer(fd, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, i, &buf_out[i]);
+  }
+  // queue up input buffers
+  for (unsigned int i = 0; i < BUF_IN_COUNT; i++) {
+    free_buf_in.push(i);
+  }
+}
+
+void V4LEncoder::encoder_open() {
+  dequeue_handler_thread = std::thread(V4LEncoder::dequeue_handler, this);
+  this->is_open = true;
+  this->counter = 0;
+}
+
+int V4LEncoder::encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) {
+  struct timeval timestamp {
+    .tv_sec = (long)(extra->timestamp_eof/1000000000),
+    .tv_usec = (long)((extra->timestamp_eof/1000) % 1000000),
+  };
+
+  // reserve buffer
+  int buffer_in = free_buf_in.pop();
+  input_bufs[buffer_in].store(buf);
+
+  // push buffer
+  extras.push(*extra);
+  //buf->sync(VISIONBUF_SYNC_TO_DEVICE);
+  queue_buffer(fd, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, buffer_in, buf, timestamp);
+
+  return this->counter++;
+}
+
+void V4LEncoder::encoder_close() {
+  if (this->is_open) {
+    // pop all the frames before closing, then put the buffers back
+    for (int i = 0; i < BUF_IN_COUNT; i++) free_buf_in.pop();
+    for (int i = 0; i < BUF_IN_COUNT; i++) free_buf_in.push(i);
+    // no frames, stop the encoder
+    struct v4l2_encoder_cmd encoder_cmd = { .cmd = V4L2_ENC_CMD_STOP };
+    util::safe_ioctl(fd, VIDIOC_ENCODER_CMD, &encoder_cmd, "VIDIOC_ENCODER_CMD failed");
+    // join waits for V4L2_QCOM_BUF_FLAG_EOS
+    dequeue_handler_thread.join();
+    assert(extras.empty());
+  }
+  this->is_open = false;
+}
+
+void V4LEncoder::set_bitrate(int bitrate) {
+  if (bitrate == current_bitrate) return;
+  if (bitrate <= 0) {
+    LOGE("invalid livestream encoder bitrate %d", bitrate);
+    return;
+  }
+
+  struct v4l2_control ctrl = {
+    .id = V4L2_CID_MPEG_VIDEO_BITRATE,
+    .value = bitrate,
+  };
+
+  if (util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl) == -1) {
+    LOGE("failed to update %s bitrate to %d", encoder_info.publish_name, bitrate);
+    return;
+  }
+  current_bitrate = bitrate;
+}
+
+void V4LEncoder::request_keyframe() {
+  struct v4l2_control ctrl = {
+    .id = V4L2_CID_MPEG_VIDC_VIDEO_REQUEST_IFRAME,
+    .value = 1,
+  };
+
+  if (util::safe_ioctl(fd, VIDIOC_S_CTRL, &ctrl) == -1) {
+    LOGE("failed to request keyframe for %s", encoder_info.publish_name);
+  }
+}
+
+V4LEncoder::~V4LEncoder() {
+  encoder_close();
+  v4l2_buf_type buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
+  util::safe_ioctl(fd, VIDIOC_STREAMOFF, &buf_type, "VIDIOC_STREAMOFF failed");
+  request_buffers(fd, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, 0);
+  buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+  util::safe_ioctl(fd, VIDIOC_STREAMOFF, &buf_type, "VIDIOC_STREAMOFF failed");
+  request_buffers(fd, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, 0);
+  close(fd);
+
+  for (int i = 0; i < BUF_OUT_COUNT; i++) {
+    if (buf_out[i].free() != 0) {
+      LOGE("Failed to free buffer");
+    }
+  }
+}
@@ -0,0 +1,49 @@
+#pragma once
+
+#include <atomic>
+#include <functional>
+
+#include "common/queue.h"
+#include "system/loggerd/encoder/encoder.h"
+
+#define BUF_IN_COUNT 9
+#define BUF_OUT_COUNT 6
+
+class V4LEncoder : public VideoEncoder {
+public:
+  using PacketCallback = std::function<void(uint8_t *, size_t, int64_t, bool, bool)>;
+  using InputDoneCallback = std::function<void(VisionBuf *)>;
+  struct Options {
+    PacketCallback packet_callback;
+    uint32_t input_format = V4L2_PIX_FMT_NV12;
+    InputDoneCallback input_done_callback;
+    bool max_performance = false;
+  };
+
+  V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
+  V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_height, Options options);
+  ~V4LEncoder();
+  int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra);
+  void encoder_open();
+  void encoder_close();
+  void set_bitrate(int bitrate);
+  void request_keyframe();
+
+private:
+  int fd;
+
+  bool is_open = false;
+  int segment_num = -1;
+  int counter = 0;
+  int current_bitrate = -1;
+  SafeQueue<VisionIpcBufExtra> extras;
+  PacketCallback packet_callback;
+  InputDoneCallback input_done_callback;
+
+  static void dequeue_handler(V4LEncoder *e);
+  std::thread dequeue_handler_thread;
+
+  VisionBuf buf_out[BUF_OUT_COUNT];
+  std::atomic<VisionBuf *> input_bufs[BUF_IN_COUNT] = {};
+  SafeQueue<unsigned int> free_buf_in;
+};
@@ -0,0 +1,230 @@
+#include <cassert>
+#ifdef __COMMA_HARDWARE__
+#include <exception>
+#include <stdexcept>
+#endif
+
+#ifdef __COMMA_HARDWARE__
+#include "system/loggerd/clip_encoder.h"
+#endif
+#include "system/loggerd/loggerd.h"
+#include "system/loggerd/encoder/jpeg_encoder.h"
+
+#ifdef __COMMA_HARDWARE__
+#include "system/loggerd/encoder/v4l_encoder.h"
+#define Encoder V4LEncoder
+#else
+#include "system/loggerd/encoder/ffmpeg_encoder.h"
+#define Encoder FfmpegEncoder
+#endif
+
+ExitHandler do_exit;
+
+struct EncoderdState {
+  int max_waiting = 0;
+
+  // Sync logic for startup
+  std::atomic<int> encoders_ready = 0;
+  std::atomic<uint32_t> start_frame_id = 0;
+  bool camera_ready[VISION_STREAM_WIDE_ROAD + 1] = {};
+  bool camera_synced[VISION_STREAM_WIDE_ROAD + 1] = {};
+};
+
+// Handle initial encoder syncing by waiting for all encoders to reach the same frame id
+bool sync_encoders(EncoderdState *s, VisionStreamType cam_type, uint32_t frame_id) {
+  if (s->camera_synced[cam_type]) return true;
+
+  if (s->max_waiting > 1 && s->encoders_ready != s->max_waiting) {
+    // add a small margin to the start frame id in case one of the encoders already dropped the next frame
+    update_max_atomic(s->start_frame_id, frame_id + 2);
+    if (std::exchange(s->camera_ready[cam_type], true) == false) {
+      ++s->encoders_ready;
+      LOGD("camera %d encoder ready", cam_type);
+    }
+    return false;
+  } else {
+    if (s->max_waiting == 1) update_max_atomic(s->start_frame_id, frame_id);
+    bool synced = frame_id >= s->start_frame_id;
+    s->camera_synced[cam_type] = synced;
+    if (!synced) LOGD("camera %d waiting for frame %d, cur %d", cam_type, (int)s->start_frame_id, frame_id);
+    return synced;
+  }
+}
+
+void encoder_set_bitrate(std::unique_ptr<Encoder> &e) {
+  static Params params;
+  std::string val = params.get("LivestreamEncoderBitrate");
+  if (val.empty()) return;
+  int bitrate = std::stoi(val);
+  e->set_bitrate(bitrate);
+}
+
+void encoder_request_keyframe(std::unique_ptr<Encoder> &e) {
+  static Params params;
+  if (!params.getBool("LivestreamRequestKeyframe")) return;
+  e->request_keyframe();
+}
+
+void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
+  util::set_thread_name(cam_info.thread_name);
+
+  std::vector<std::unique_ptr<Encoder>> encoders;
+
+  VisionIpcClient vipc_client = VisionIpcClient("camerad", cam_info.stream_type, false);
+
+  std::unique_ptr<JpegEncoder> jpeg_encoder;
+
+  int cur_seg = 0;
+  while (!do_exit) {
+    if (!vipc_client.connect(false)) {
+      util::sleep_for(5);
+      continue;
+    }
+
+    // init encoders
+    if (encoders.empty()) {
+      const VisionBuf &buf_info = vipc_client.buffers[0];
+      LOGW("encoder %s init %zux%zu", cam_info.thread_name, buf_info.width, buf_info.height);
+      assert(buf_info.width > 0 && buf_info.height > 0);
+
+      for (const auto &encoder_info : cam_info.encoder_infos) {
+        auto &e = encoders.emplace_back(new Encoder(encoder_info, buf_info.width, buf_info.height));
+        e->encoder_open();
+      }
+
+      // Only one thumbnail can be generated per camera stream
+      if (auto thumbnail_name = cam_info.encoder_infos[0].thumbnail_name) {
+        jpeg_encoder = std::make_unique<JpegEncoder>(thumbnail_name, buf_info.width / 4, buf_info.height / 4);
+      }
+    }
+
+    bool lagging = false;
+    while (!do_exit) {
+      VisionIpcBufExtra extra;
+      VisionBuf* buf = vipc_client.recv(&extra);
+      if (buf == nullptr) continue;
+
+      // detect loop around and drop the frames
+      if (buf->get_frame_id() != extra.frame_id) {
+        if (!lagging) {
+          LOGE("encoder %s lag  buffer id: %" PRIu64 " extra id: %d", cam_info.thread_name, buf->get_frame_id(), extra.frame_id);
+          lagging = true;
+        }
+        continue;
+      }
+      lagging = false;
+
+      if (!sync_encoders(s, cam_info.stream_type, extra.frame_id)) {
+        continue;
+      }
+      if (do_exit) break;
+
+      // do rotation if required
+      const int frames_per_seg = SEGMENT_LENGTH * MAIN_FPS;
+      if (cur_seg >= 0 && extra.frame_id >= ((cur_seg + 1) * frames_per_seg) + s->start_frame_id) {
+        for (auto &e : encoders) {
+          e->encoder_close();
+          e->encoder_open();
+        }
+        ++cur_seg;
+      }
+
+      // encode a frame
+      for (int i = 0; i < encoders.size(); ++i) {
+        if (cam_info.encoder_infos[i].is_live) {
+          encoder_set_bitrate(encoders[i]);
+          encoder_request_keyframe(encoders[i]);
+        }
+
+        int out_id = encoders[i]->encode_frame(buf, &extra);
+
+        if (out_id == -1) {
+          LOGE("Failed to encode frame. frame_id: %d", extra.frame_id);
+        }
+      }
+
+      if (jpeg_encoder && (extra.frame_id % 1200 == 100)) {
+        jpeg_encoder->pushThumbnail(buf, extra);
+      }
+    }
+  }
+}
+
+template <size_t N>
+void encoderd_thread(const LogCameraInfo (&cameras)[N]) {
+  EncoderdState s;
+
+  std::set<VisionStreamType> streams;
+  while (!do_exit) {
+    streams = VisionIpcClient::getAvailableStreams("camerad", false);
+    if (!streams.empty()) {
+      break;
+    }
+    util::sleep_for(100);
+  }
+
+  if (!streams.empty()) {
+    std::vector<std::thread> encoder_threads;
+    for (auto stream : streams) {
+      auto it = std::find_if(std::begin(cameras), std::end(cameras),
+                             [stream](auto &cam) { return cam.stream_type == stream; });
+      assert(it != std::end(cameras));
+      ++s.max_waiting;
+      encoder_threads.push_back(std::thread(encoder_thread, &s, *it));
+    }
+
+    for (auto &t : encoder_threads) t.join();
+  }
+}
+
+int main(int argc, char* argv[]) {
+#ifdef __COMMA_HARDWARE__
+  if (argc > 1 && std::string(argv[1]) == "--clip") {
+    if (argc < 6) {
+      fprintf(stderr, "usage: encoderd --clip OUTPUT START DURATION [--bitrate BPS] [--speedup N] "
+                      "[--metadata JSON] SEGMENT [SEGMENT ...]\n");
+      return 2;
+    }
+    try {
+      int bitrate = 5'000'000;
+      int speedup = 1;
+      std::string metadata;
+      int input_arg = 5;
+      while (input_arg < argc && std::string(argv[input_arg]).rfind("--", 0) == 0) {
+        const std::string option = argv[input_arg++];
+        if (option == "--") break;
+        if (input_arg == argc) throw std::invalid_argument("missing clip option value");
+        if (option == "--bitrate") bitrate = std::stoi(argv[input_arg++]);
+        else if (option == "--speedup") speedup = std::stoi(argv[input_arg++]);
+        else if (option == "--metadata") metadata = argv[input_arg++];
+        else throw std::invalid_argument("unknown clip option: " + option);
+      }
+      if (input_arg == argc) throw std::invalid_argument("missing clip input");
+      std::vector<std::string> inputs(argv + input_arg, argv + argc);
+      return encode_clip(inputs, argv[2], std::stod(argv[3]), std::stod(argv[4]),
+                         bitrate, speedup, metadata);
+    } catch (const std::exception &e) {
+      fprintf(stderr, "clip encoding failed: %s\n", e.what());
+      return 1;
+    }
+  }
+#endif
+  if (!Hardware::PC()) {
+    int ret;
+    ret = util::set_realtime_priority(52);
+    assert(ret == 0);
+    ret = util::set_core_affinity({3});
+    assert(ret == 0);
+  }
+  if (argc > 1) {
+    std::string arg1(argv[1]);
+    if (arg1 == "--stream") {
+      encoderd_thread(stream_cameras_logged);
+    } else {
+      LOGE("Argument '%s' is not supported", arg1.c_str());
+    }
+  } else {
+    encoderd_thread(cameras_logged);
+  }
+  return 0;
+}
@@ -0,0 +1,204 @@
+#include "system/loggerd/logger.h"
+
+#include <fstream>
+#include <map>
+#include <vector>
+#include <iostream>
+#include <sstream>
+#include <random>
+
+#include "common/params.h"
+#include "common/swaglog.h"
+#include "common/version.h"
+
+#include "sunnypilot/common/version.h"
+
+// ***** log metadata *****
+kj::Array<capnp::word> logger_build_init_data(bool route_log) {
+  uint64_t wall_time = nanos_since_epoch();
+
+  MessageBuilder msg;
+  auto init = msg.initEvent().initInitData();
+
+  init.setWallTimeNanos(wall_time);
+  init.setVersion(SUNNYPILOT_VERSION);
+  init.setDirty(!getenv("CLEAN"));
+  init.setDeviceType(Hardware::get_device_type());
+
+  // log kernel args
+  std::ifstream cmdline_stream("/proc/cmdline");
+  std::vector<std::string> kernel_args;
+  std::string buf;
+  while (cmdline_stream >> buf) {
+    kernel_args.push_back(buf);
+  }
+
+  auto lkernel_args = init.initKernelArgs(kernel_args.size());
+  for (int i=0; i<kernel_args.size(); i++) {
+    lkernel_args.set(i, kernel_args[i]);
+  }
+
+  init.setKernelVersion(util::read_file("/proc/version"));
+  init.setOsVersion(util::read_file("/VERSION"));
+
+  // log params
+  Params params(util::getenv("PARAMS_COPY_PATH", ""));
+  std::map<std::string, std::string> params_map = params.readAll();
+
+  init.setGitCommit(params_map["GitCommit"]);
+  init.setGitCommitDate(params_map["GitCommitDate"]);
+  init.setGitBranch(params_map["GitBranch"]);
+  init.setGitRemote(params_map["GitRemote"]);
+  init.setPassive(false);
+  init.setDongleId(params_map["DongleId"]);
+
+  // for prebuilt branches
+  init.setGitSrcCommit(util::read_file("../../git_src_commit"));
+  init.setGitSrcCommitDate(util::read_file("../../git_src_commit_date"));
+
+  auto lparams = init.initParams().initEntries(params_map.size());
+  int j = 0;
+  for (auto& [key, value] : params_map) {
+    auto lentry = lparams[j];
+    lentry.setKey(key);
+    if ( !(params.getKeyFlag(key) & DONT_LOG) ) {
+      lentry.setValue(capnp::Data::Reader((const kj::byte*)value.data(), value.size()));
+    }
+    j++;
+  }
+
+  // log commands
+  std::vector<std::string> log_commands = {
+    "df -h",  // usage for all filesystems
+  };
+
+  auto hw_logs = Hardware::get_init_logs(route_log);
+
+  auto commands = init.initCommands().initEntries(log_commands.size() + hw_logs.size());
+  for (int i = 0; i < log_commands.size(); i++) {
+    auto lentry = commands[i];
+
+    lentry.setKey(log_commands[i]);
+
+    const std::string result = util::check_output(log_commands[i]);
+    lentry.setValue(capnp::Data::Reader((const kj::byte*)result.data(), result.size()));
+  }
+
+  int i = log_commands.size();
+  for (auto &[key, value] : hw_logs) {
+    auto lentry = commands[i];
+    lentry.setKey(key);
+    lentry.setValue(capnp::Data::Reader((const kj::byte*)value.data(), value.size()));
+    i++;
+  }
+
+  return capnp::messageToFlatArray(msg);
+}
+
+std::string logger_get_identifier(std::string key) {
+  // a log identifier is a 32 bit counter, plus a 10 character unique ID.
+  // e.g. 000001a3--c20ba54385
+
+  Params params;
+  uint32_t cnt;
+  try {
+    cnt = std::stoul(params.get(key));
+  } catch (std::exception &e) {
+    cnt = 0;
+  }
+  params.put(key, std::to_string(cnt + 1));
+
+  std::stringstream ss;
+  std::random_device rd;
+  std::mt19937 mt(rd());
+  std::uniform_int_distribution<int> dist(0, 15);
+  for (int i = 0; i < 10; ++i) {
+    ss << std::hex << dist(mt);
+  }
+
+  return util::string_format("%08x--%s", cnt, ss.str().c_str());
+}
+
+std::string zstd_decompress(const std::string &in) {
+  ZSTD_DCtx *dctx = ZSTD_createDCtx();
+  assert(dctx != nullptr);
+
+  // Initialize input and output buffers
+  ZSTD_inBuffer input = {in.data(), in.size(), 0};
+
+  // Estimate and reserve memory for decompressed data
+  size_t estimatedDecompressedSize = ZSTD_getFrameContentSize(in.data(), in.size());
+  if (estimatedDecompressedSize == ZSTD_CONTENTSIZE_ERROR || estimatedDecompressedSize == ZSTD_CONTENTSIZE_UNKNOWN) {
+    estimatedDecompressedSize = in.size() * 2;  // Use a fallback size
+  }
+
+  std::string decompressedData;
+  decompressedData.reserve(estimatedDecompressedSize);
+
+  const size_t bufferSize = ZSTD_DStreamOutSize();  // Recommended output buffer size
+  std::string outputBuffer(bufferSize, '\0');
+
+  while (input.pos < input.size) {
+    ZSTD_outBuffer output = {outputBuffer.data(), bufferSize, 0};
+
+    size_t result = ZSTD_decompressStream(dctx, &output, &input);
+    if (ZSTD_isError(result)) {
+      break;
+    }
+
+    decompressedData.append(outputBuffer.data(), output.pos);
+  }
+
+  ZSTD_freeDCtx(dctx);
+  decompressedData.shrink_to_fit();
+  return decompressedData;
+}
+
+
+static void log_sentinel(LoggerState *log, SentinelType type, int exit_signal = 0) {
+  MessageBuilder msg;
+  auto sen = msg.initEvent().initSentinel();
+  sen.setType(type);
+  sen.setSignal(exit_signal);
+  log->write(msg.toBytes(), true);
+}
+
+LoggerState::LoggerState(const std::string &log_root) {
+  route_name = logger_get_identifier("RouteCount");
+  route_path = log_root + "/" + route_name;
+  init_data = logger_build_init_data(true);
+}
+
+LoggerState::~LoggerState() {
+  if (rlog) {
+    log_sentinel(this, SentinelType::END_OF_ROUTE, exit_signal);
+    std::remove(lock_file.c_str());
+  }
+}
+
+bool LoggerState::next() {
+  if (rlog) {
+    log_sentinel(this, SentinelType::END_OF_SEGMENT);
+    std::remove(lock_file.c_str());
+  }
+
+  segment_path = route_path + "--" + std::to_string(++part);
+  bool ret = util::create_directories(segment_path, 0775);
+  assert(ret == true);
+
+  lock_file = segment_path + "/rlog.lock";
+  std::ofstream{lock_file};
+
+  rlog.reset(new ZstdFileWriter(segment_path + "/rlog.zst", LOG_COMPRESSION_LEVEL));
+  qlog.reset(new ZstdFileWriter(segment_path + "/qlog.zst", LOG_COMPRESSION_LEVEL));
+
+  // log init data & sentinel type.
+  write(init_data.asBytes(), true);
+  log_sentinel(this, part > 0 ? SentinelType::START_OF_SEGMENT : SentinelType::START_OF_ROUTE);
+  return true;
+}
+
+void LoggerState::write(uint8_t* data, size_t size, bool in_qlog) {
+  rlog->write(data, size);
+  if (in_qlog) qlog->write(data, size);
+}
@@ -0,0 +1,37 @@
+#pragma once
+
+#include <cassert>
+#include <memory>
+#include <string>
+
+#include "openpilot/cereal/messaging/messaging.h"
+#include "common/util.h"
+#include "common/hardware/hw.h"
+#include "system/loggerd/zstd_writer.h"
+
+constexpr int LOG_COMPRESSION_LEVEL = 10;
+
+typedef cereal::Sentinel::SentinelType SentinelType;
+
+class LoggerState {
+public:
+  LoggerState(const std::string& log_root = Path::log_root());
+  ~LoggerState();
+  bool next();
+  void write(uint8_t* data, size_t size, bool in_qlog);
+  inline int segment() const { return part; }
+  inline const std::string& segmentPath() const { return segment_path; }
+  inline const std::string& routeName() const { return route_name; }
+  inline void write(kj::ArrayPtr<kj::byte> bytes, bool in_qlog) { write(bytes.begin(), bytes.size(), in_qlog); }
+  inline void setExitSignal(int signal) { exit_signal = signal; }
+
+protected:
+  int part = -1, exit_signal = 0;
+  std::string route_path, route_name, segment_path, lock_file;
+  kj::Array<capnp::word> init_data;
+  std::unique_ptr<ZstdFileWriter> rlog, qlog;
+};
+
+kj::Array<capnp::word> logger_build_init_data(bool route_log = false);
+std::string logger_get_identifier(std::string key);
+std::string zstd_decompress(const std::string &in);
@@ -0,0 +1,358 @@
+#include <sys/xattr.h>
+
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "common/params.h"
+#include "system/loggerd/encoder/encoder.h"
+#include "system/loggerd/loggerd.h"
+#include "system/loggerd/video_writer.h"
+
+ExitHandler do_exit;
+
+struct LoggerdState {
+  LoggerState logger;
+  std::atomic<double> last_camera_seen_tms{0.0};
+  std::atomic<int> ready_to_rotate{0};  // count of encoders ready to rotate
+  int max_waiting = 0;
+  double last_rotate_tms = 0.;      // last rotate time in ms
+};
+
+void logger_rotate(LoggerdState *s) {
+  bool ret =s->logger.next();
+  assert(ret);
+  s->ready_to_rotate = 0;
+  s->last_rotate_tms = millis_since_boot();
+  LOGW((s->logger.segment() == 0) ? "logging to %s" : "rotated to %s", s->logger.segmentPath().c_str());
+}
+
+void rotate_if_needed(LoggerdState *s) {
+  // all encoders ready, trigger rotation
+  bool all_ready = s->max_waiting > 0 && s->ready_to_rotate == s->max_waiting;
+
+  // fallback logic to prevent extremely long segments in the case of camera, encoder, etc. malfunctions
+  bool timed_out = false;
+  double tms = millis_since_boot();
+  double seg_length_secs = (tms - s->last_rotate_tms) / 1000.;
+  if ((seg_length_secs > SEGMENT_LENGTH) && !LOGGERD_TEST) {
+    // TODO: might be nice to put these reasons in the sentinel
+    if ((tms - s->last_camera_seen_tms) > NO_CAMERA_PATIENCE) {
+      timed_out = true;
+      LOGE("no camera packets seen. auto rotating");
+    } else if (seg_length_secs > SEGMENT_LENGTH*1.2) {
+      timed_out = true;
+      LOGE("segment too long. auto rotating");
+    }
+  }
+
+  if (all_ready || timed_out) {
+    logger_rotate(s);
+  }
+}
+
+struct RemoteEncoder {
+  std::unique_ptr<VideoWriter> writer;
+  int encoderd_segment_offset;
+  int current_segment = -1;
+  std::vector<Message *> q;
+  int dropped_frames = 0;
+  bool recording = false;
+  bool marked_ready_to_rotate = false;
+  bool seen_first_packet = false;
+  bool audio_initialized = false;
+};
+
+size_t write_encode_data(LoggerdState *s, cereal::Event::Reader event, RemoteEncoder &re, const EncoderInfo &encoder_info) {
+  auto edata = (event.*(encoder_info.get_encode_data_func))();
+  auto idx = edata.getIdx();
+  auto flags = idx.getFlags();
+
+  // if we aren't recording yet, try to start, since we are in the correct segment
+  if (!re.recording) {
+    if (flags & V4L2_BUF_FLAG_KEYFRAME) {
+      // only create on iframe
+      if (re.dropped_frames) {
+        // this should only happen for the first segment, maybe
+        LOGW("%s: dropped %d non iframe packets before init", encoder_info.publish_name, re.dropped_frames);
+        re.dropped_frames = 0;
+      }
+      if (encoder_info.record) {
+        // write the header
+        auto header = edata.getHeader();
+        re.writer->write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof() / 1000, true, false);
+      }
+      re.recording = true;
+    } else {
+      // this is a sad case when we aren't recording, but don't have an iframe
+      // nothing we can do but drop the frame
+      ++re.dropped_frames;
+      return 0;
+    }
+  }
+
+  // we have to be recording if we are here
+  assert(re.recording);
+
+  // if we are actually writing the video file, do so
+  if (re.writer) {
+    auto data = edata.getData();
+    re.writer->write((uint8_t *)data.begin(), data.size(), idx.getTimestampEof() / 1000, false, flags & V4L2_BUF_FLAG_KEYFRAME);
+  }
+
+  // put it in log stream as the idx packet
+  MessageBuilder bmsg;
+  auto evt = bmsg.initEvent(event.getValid());
+  evt.setLogMonoTime(event.getLogMonoTime());
+  (evt.*(encoder_info.set_encode_idx_func))(idx);
+  auto new_msg = bmsg.toBytes();
+  s->logger.write((uint8_t *)new_msg.begin(), new_msg.size(), true);  // always in qlog?
+  return new_msg.size();
+}
+
+int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct RemoteEncoder &re, const EncoderInfo &encoder_info) {
+  int bytes_count = 0;
+
+  // extract the message
+  capnp::FlatArrayMessageReader cmsg(kj::ArrayPtr<capnp::word>((capnp::word *)msg->getData(), msg->getSize() / sizeof(capnp::word)));
+  auto event = cmsg.getRoot<cereal::Event>();
+  auto edata = (event.*(encoder_info.get_encode_data_func))();
+  auto idx = edata.getIdx();
+
+  // encoderd can have started long before loggerd
+  if (!re.seen_first_packet) {
+    re.seen_first_packet = true;
+    re.encoderd_segment_offset = idx.getSegmentNum();
+    ++s->max_waiting;  // only count encoders that publish so a disabled/missing camera doesn't stall rotation
+    LOGD("%s: has encoderd offset %d", name.c_str(), re.encoderd_segment_offset);
+  }
+  int offset_segment_num = idx.getSegmentNum() - re.encoderd_segment_offset;
+
+  if (offset_segment_num == s->logger.segment()) {
+    // loggerd is now on the segment that matches this packet
+
+    // if this is a new segment, we close any possible old segments, move to the new, and process any queued packets
+    if (re.current_segment != s->logger.segment()) {
+      // if we aren't actually recording, don't create the writer
+      if (encoder_info.record) {
+        assert(encoder_info.filename != NULL);
+        re.writer.reset(new VideoWriter(s->logger.segmentPath().c_str(),
+                                        encoder_info.filename, idx.getType() != cereal::EncodeIndex::Type::FULL_H_E_V_C,
+                                        edata.getWidth(), edata.getHeight(), encoder_info.fps, idx.getType()));
+        re.recording = false;
+        re.audio_initialized = false;
+      }
+      re.current_segment = s->logger.segment();
+      re.marked_ready_to_rotate = false;
+    }
+    if (re.audio_initialized || !encoder_info.include_audio) {
+      // we are in this segment now, process any queued messages before this one
+      if (!re.q.empty()) {
+        for (auto qmsg : re.q) {
+          capnp::FlatArrayMessageReader reader({(capnp::word *)qmsg->getData(), qmsg->getSize() / sizeof(capnp::word)});
+          bytes_count += write_encode_data(s, reader.getRoot<cereal::Event>(), re, encoder_info);
+          delete qmsg;
+        }
+        re.q.clear();
+      }
+      bytes_count += write_encode_data(s, event, re, encoder_info);
+      delete msg;
+    } else if (re.q.size() > MAIN_FPS*10) {
+      LOGE_100("%s: dropping frame waiting for audio initialization, queue is too large", name.c_str());
+      delete msg;
+    } else {
+      re.q.push_back(msg); // queue up all the new segment messages, they go in after audio is initialized
+    }
+  } else if (offset_segment_num > s->logger.segment()) {
+    // encoderd packet has a newer segment, this means encoderd has rolled over
+    if (!re.marked_ready_to_rotate) {
+      re.marked_ready_to_rotate = true;
+      ++s->ready_to_rotate;
+      LOGD("rotate %d -> %d ready %d/%d for %s",
+        s->logger.segment(), offset_segment_num,
+        s->ready_to_rotate.load(), s->max_waiting, name.c_str());
+    }
+
+    // TODO: define this behavior, but for now don't leak
+    if (re.q.size() > MAIN_FPS*10) {
+      LOGE_100("%s: dropping frame, queue is too large", name.c_str());
+      delete msg;
+    } else {
+      // queue up all the new segment messages, they go in after the rotate
+      re.q.push_back(msg);
+    }
+  } else {
+    LOGE("%s: encoderd packet has a older segment!!! idx.getSegmentNum():%d s->logger.segment():%d re.encoderd_segment_offset:%d",
+      name.c_str(), idx.getSegmentNum(), s->logger.segment(), re.encoderd_segment_offset);
+    // free the message, it's useless. this should never happen
+    // actually, this can happen if you restart encoderd
+    re.encoderd_segment_offset = -s->logger.segment();
+    delete msg;
+  }
+
+  return bytes_count;
+}
+
+void handle_preserve_segment(LoggerdState *s) {
+  static int prev_segment = -1;
+  if (s->logger.segment() == prev_segment) return;
+
+  LOGW("preserving %s", s->logger.segmentPath().c_str());
+
+#ifdef __APPLE__
+  int ret = setxattr(s->logger.segmentPath().c_str(), PRESERVE_ATTR_NAME, &PRESERVE_ATTR_VALUE, 1, 0, 0);
+#else
+  int ret = setxattr(s->logger.segmentPath().c_str(), PRESERVE_ATTR_NAME, &PRESERVE_ATTR_VALUE, 1, 0);
+#endif
+  if (ret) {
+    LOGE("setxattr %s failed for %s: %s", PRESERVE_ATTR_NAME, s->logger.segmentPath().c_str(), strerror(errno));
+  }
+
+  // mark route for uploading
+  Params params;
+  std::string routes = params.get("AthenadRecentlyViewedRoutes");
+  params.put("AthenadRecentlyViewedRoutes", routes + "," + s->logger.routeName());
+
+  prev_segment = s->logger.segment();
+}
+
+void loggerd_thread() {
+  // setup messaging
+  struct ServiceState {
+    std::string name;
+    int counter, freq;
+    bool encoder, preserve_segment, record_audio;
+  };
+  std::unordered_map<SubSocket*, ServiceState> service_state;
+  std::unordered_map<SubSocket*, struct RemoteEncoder> remote_encoders;
+
+  std::unique_ptr<Context> ctx(Context::create());
+  std::unique_ptr<Poller> poller(Poller::create());
+
+  // subscribe to all socks
+  for (const auto& [_, it] : services) {
+    const bool encoder = util::ends_with(it.name, "EncodeData");
+    const bool livestream_encoder = util::starts_with(it.name, "livestream");
+    const bool record_audio = (it.name == "rawAudioData") && Params().getBool("RecordAudio");
+    if (it.should_log || (encoder && !livestream_encoder) || record_audio) {
+      LOGD("logging %s", it.name.c_str());
+
+      SubSocket * sock = SubSocket::create(ctx.get(), it.name, "127.0.0.1", false, true, it.queue_size);
+      assert(sock != NULL);
+      poller->registerSocket(sock);
+      service_state[sock] = {
+        .name = it.name,
+        .counter = 0,
+        .freq = it.decimation,
+        .encoder = encoder,
+        .preserve_segment = it.name == "userBookmark",
+        .record_audio = record_audio,
+      };
+    }
+  }
+
+  LoggerdState s;
+  // init logger
+  logger_rotate(&s);
+  Params().put("CurrentRoute", s.logger.routeName());
+
+  std::map<std::string, EncoderInfo> encoder_infos_dict;
+  std::vector<RemoteEncoder*> encoders_with_audio;
+  for (const auto &cam : cameras_logged) {
+    for (const auto &encoder_info : cam.encoder_infos) {
+      encoder_infos_dict[encoder_info.publish_name] = encoder_info;
+    }
+  }
+
+  for (auto &[sock, service] : service_state) {
+    auto it = encoder_infos_dict.find(service.name);
+    if (it != encoder_infos_dict.end() && it->second.include_audio) {
+      encoders_with_audio.push_back(&remote_encoders[sock]);
+    }
+  }
+
+  uint64_t msg_count = 0, bytes_count = 0;
+  double start_ts = millis_since_boot();
+  while (!do_exit) {
+    // poll for new messages on all sockets
+    for (auto sock : poller->poll(1000)) {
+      if (do_exit) break;
+
+      ServiceState &service = service_state[sock];
+      if (service.preserve_segment) {
+        handle_preserve_segment(&s);
+      }
+
+      // drain socket
+      int count = 0;
+      Message *msg = nullptr;
+      while (!do_exit && (msg = sock->receive(true))) {
+        const bool in_qlog = service.freq != -1 && (service.counter++ % service.freq == 0);
+
+        if (service.record_audio) {
+          capnp::FlatArrayMessageReader cmsg(kj::ArrayPtr<capnp::word>((capnp::word *)msg->getData(), msg->getSize() / sizeof(capnp::word)));
+          auto event = cmsg.getRoot<cereal::Event>();
+          auto audio_data = event.getRawAudioData().getData();
+          auto sample_rate = event.getRawAudioData().getSampleRate();
+          for (auto* encoder : encoders_with_audio) {
+            if (encoder && encoder->writer) {
+              encoder->writer->write_audio((uint8_t*)audio_data.begin(), audio_data.size(), event.getLogMonoTime() / 1000, sample_rate);
+              encoder->audio_initialized = true;
+            }
+          }
+        }
+
+        if (service.encoder) {
+          s.last_camera_seen_tms = millis_since_boot();
+          bytes_count += handle_encoder_msg(&s, msg, service.name, remote_encoders[sock], encoder_infos_dict[service.name]);
+        } else {
+          s.logger.write((uint8_t *)msg->getData(), msg->getSize(), in_qlog);
+          bytes_count += msg->getSize();
+          delete msg;
+        }
+
+        rotate_if_needed(&s);
+
+        if ((++msg_count % 10000) == 0) {
+          double seconds = (millis_since_boot() - start_ts) / 1000.0;
+          LOGD("%" PRIu64 " messages, %.2f msg/sec, %.2f KB/sec", msg_count, msg_count / seconds, bytes_count * 0.001 / seconds);
+        }
+
+        count++;
+        if (count >= 200) {
+          LOGD("large volume of '%s' messages", service.name.c_str());
+          break;
+        }
+      }
+    }
+  }
+
+  LOGW("closing logger");
+  s.logger.setExitSignal(do_exit.signal);
+
+  if (do_exit.power_failure) {
+    LOGE("power failure");
+    sync();
+    LOGE("sync done");
+  }
+
+  // messaging cleanup
+  for (auto &[sock, service] : service_state) delete sock;
+}
+
+int main(int argc, char** argv) {
+  if (!Hardware::PC()) {
+    int ret;
+    ret = util::set_core_affinity({0, 1, 2, 3});
+    assert(ret == 0);
+    // TODO: why does this impact camerad timings?
+    //ret = util::set_realtime_priority(1);
+    //assert(ret == 0);
+  }
+
+  loggerd_thread();
+
+  return 0;
+}
@@ -0,0 +1,199 @@
+#pragma once
+
+#include <cstdlib>
+#include <vector>
+
+#include "openpilot/cereal/messaging/messaging.h"
+#include "openpilot/cereal/services.h"
+#include "openpilot/cereal/visionstream.h"
+#include "msgq/visionipc/visionipc_client.h"
+#include "common/hardware/hw.h"
+#include "common/params.h"
+#include "common/swaglog.h"
+#include "common/util.h"
+
+#include "system/loggerd/logger.h"
+
+constexpr int MAIN_FPS = 20;
+const auto MAIN_ENCODE_TYPE = Hardware::PC() ? cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS : cereal::EncodeIndex::Type::FULL_H_E_V_C;
+#define NO_CAMERA_PATIENCE 500  // fall back to time-based rotation if all cameras are dead
+
+#define INIT_ENCODE_FUNCTIONS(encode_type)                                \
+  .get_encode_data_func = &cereal::Event::Reader::get##encode_type##Data, \
+  .set_encode_idx_func = &cereal::Event::Builder::set##encode_type##Idx,  \
+  .init_encode_data_func = &cereal::Event::Builder::init##encode_type##Data
+
+const bool LOGGERD_TEST = getenv("LOGGERD_TEST");
+const int SEGMENT_LENGTH = LOGGERD_TEST ? atoi(getenv("LOGGERD_SEGMENT_LENGTH")) : 60;
+
+inline int livestream_width() {
+  switch (Hardware::get_device_type()) {
+    case cereal::InitData::DeviceType::TIZI: return 1152;
+    case cereal::InitData::DeviceType::MICI: return 1280;
+    default: return -1;
+  }
+}
+
+inline int livestream_height() {
+  switch (Hardware::get_device_type()) {
+    case cereal::InitData::DeviceType::TIZI:
+    case cereal::InitData::DeviceType::MICI: return 720;
+    default: return -1;
+  }
+}
+
+constexpr char PRESERVE_ATTR_NAME[] = "user.preserve";
+constexpr char PRESERVE_ATTR_VALUE = '1';
+
+struct EncoderSettings {
+  cereal::EncodeIndex::Type encode_type;
+  int bitrate;
+  int gop_size;
+  int b_frames = 0; // we don't use b frames
+
+  static EncoderSettings MainEncoderSettings(int in_width) {
+    if (in_width <= 1344) {
+      return EncoderSettings{.encode_type = MAIN_ENCODE_TYPE, .bitrate = 5'000'000, .gop_size = 20};
+    } else {
+      return EncoderSettings{.encode_type = MAIN_ENCODE_TYPE, .bitrate = 10'000'000, .gop_size = 30};
+    }
+  }
+
+  static EncoderSettings QcamEncoderSettings() {
+    return EncoderSettings{.encode_type = cereal::EncodeIndex::Type::QCAMERA_H264, .bitrate = 256'000, .gop_size = 15};
+  }
+
+  static EncoderSettings StreamEncoderSettings() {
+    int _stream_bitrate = getenv("STREAM_BITRATE") ? atoi(getenv("STREAM_BITRATE")) : 5'000'000;
+    return EncoderSettings{.encode_type = cereal::EncodeIndex::Type::QCAMERA_H264, .bitrate = _stream_bitrate , .gop_size = 5};
+  }
+};
+
+class EncoderInfo {
+public:
+  const char *publish_name;
+  const char *thumbnail_name = NULL;
+  const char *filename = NULL;
+  bool record = true;
+  bool include_audio = false;
+  bool is_live = false;
+  int frame_width = -1;
+  int frame_height = -1;
+  int fps = MAIN_FPS;
+  std::function<EncoderSettings(int)> get_settings;
+
+  ::cereal::EncodeData::Reader (cereal::Event::Reader::*get_encode_data_func)() const;
+  void (cereal::Event::Builder::*set_encode_idx_func)(::cereal::EncodeIndex::Reader);
+  cereal::EncodeData::Builder (cereal::Event::Builder::*init_encode_data_func)();
+};
+
+class LogCameraInfo {
+public:
+  const char *thread_name;
+  int fps = MAIN_FPS;
+  VisionStreamType stream_type;
+  std::vector<EncoderInfo> encoder_infos;
+};
+
+const EncoderInfo main_road_encoder_info = {
+  .publish_name = "narrowRoadEncodeData",
+  .thumbnail_name = "thumbnail",
+  .filename = "fcamera.hevc",
+  .get_settings = [](int in_width){return EncoderSettings::MainEncoderSettings(in_width);},
+  INIT_ENCODE_FUNCTIONS(NarrowRoadEncode),
+};
+
+const EncoderInfo main_wide_road_encoder_info = {
+  .publish_name = "wideRoadEncodeData",
+  .filename = "ecamera.hevc",
+  .get_settings = [](int in_width){return EncoderSettings::MainEncoderSettings(in_width);},
+  INIT_ENCODE_FUNCTIONS(WideRoadEncode),
+};
+
+const EncoderInfo main_cabin_encoder_info = {
+  .publish_name = "cabinEncodeData",
+  .filename = "dcamera.hevc",
+  .record = Params().getBool("RecordFront"),
+  .get_settings = [](int in_width){return EncoderSettings::MainEncoderSettings(in_width);},
+  INIT_ENCODE_FUNCTIONS(CabinEncode),
+};
+
+const EncoderInfo stream_road_encoder_info = {
+  .publish_name = "livestreamNarrowRoadEncodeData",
+  //.thumbnail_name = "thumbnail",
+  .record = false,
+  .is_live = true,
+  .frame_width = livestream_width(),
+  .frame_height = livestream_height(),
+  .get_settings = [](int){return EncoderSettings::StreamEncoderSettings();},
+  INIT_ENCODE_FUNCTIONS(LivestreamNarrowRoadEncode),
+};
+
+const EncoderInfo stream_wide_road_encoder_info = {
+  .publish_name = "livestreamWideRoadEncodeData",
+  .record = false,
+  .is_live = true,
+  .frame_width = livestream_width(),
+  .frame_height = livestream_height(),
+  .get_settings = [](int){return EncoderSettings::StreamEncoderSettings();},
+  INIT_ENCODE_FUNCTIONS(LivestreamWideRoadEncode),
+};
+
+const EncoderInfo stream_cabin_encoder_info = {
+  .publish_name = "livestreamCabinEncodeData",
+  .record = false,
+  .is_live = true,
+  .frame_width = livestream_width(),
+  .frame_height = livestream_height(),
+  .get_settings = [](int){return EncoderSettings::StreamEncoderSettings();},
+  INIT_ENCODE_FUNCTIONS(LivestreamCabinEncode),
+};
+
+const EncoderInfo qcam_encoder_info = {
+  .publish_name = "qNarrowRoadEncodeData",
+  .filename = "qcamera.ts",
+  .include_audio = Params().getBool("RecordAudio"),
+  .frame_width = 526,
+  .frame_height = 330,
+  .get_settings = [](int){return EncoderSettings::QcamEncoderSettings();},
+  INIT_ENCODE_FUNCTIONS(QNarrowRoadEncode),
+};
+
+const LogCameraInfo narrow_road_camera_info{
+  .thread_name = "narrow_road_cam_encoder",
+  .stream_type = VISION_STREAM_NARROW_ROAD,
+  .encoder_infos = {main_road_encoder_info, qcam_encoder_info}
+};
+
+const LogCameraInfo wide_road_camera_info{
+  .thread_name = "wide_road_cam_encoder",
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .encoder_infos = {main_wide_road_encoder_info}
+};
+
+const LogCameraInfo cabin_camera_info{
+  .thread_name = "cabin_cam_encoder",
+  .stream_type = VISION_STREAM_CABIN,
+  .encoder_infos = {main_cabin_encoder_info}
+};
+
+const LogCameraInfo stream_road_camera_info{
+  .thread_name = "narrow_road_cam_encoder",
+  .stream_type = VISION_STREAM_NARROW_ROAD,
+  .encoder_infos = {stream_road_encoder_info},
+};
+
+const LogCameraInfo stream_wide_road_camera_info{
+  .thread_name = "wide_road_cam_encoder",
+  .stream_type = VISION_STREAM_WIDE_ROAD,
+  .encoder_infos = {stream_wide_road_encoder_info},
+};
+
+const LogCameraInfo stream_cabin_camera_info{
+  .thread_name = "cabin_cam_encoder",
+  .stream_type = VISION_STREAM_CABIN,
+  .encoder_infos = {stream_cabin_encoder_info},
+};
+
+const LogCameraInfo cameras_logged[] = {narrow_road_camera_info, wide_road_camera_info, cabin_camera_info};
+const LogCameraInfo stream_cameras_logged[] = {stream_road_camera_info, stream_wide_road_camera_info, stream_cabin_camera_info};
@@ -0,0 +1,91 @@
+import os
+import random
+from pathlib import Path
+
+
+from openpilot.common.test import OpenpilotTestCase
+import openpilot.system.loggerd.deleter as deleter
+import openpilot.system.loggerd.uploader as uploader
+from openpilot.common.params import Params
+from openpilot.common.hardware.hw import Paths
+from openpilot.system.loggerd.xattr_cache import setxattr
+
+
+def create_random_file(file_path: Path, size_mb: float, lock: bool = False, upload_xattr: bytes | None = None) -> None:
+  file_path.parent.mkdir(parents=True, exist_ok=True)
+
+  if lock:
+    lock_path = str(file_path) + ".lock"
+    os.close(os.open(lock_path, os.O_CREAT | os.O_EXCL))
+
+  chunks = 128
+  chunk_bytes = int(size_mb * 1024 * 1024 / chunks)
+  data = os.urandom(chunk_bytes)
+
+  with open(file_path, "wb") as f:
+    for _ in range(chunks):
+      f.write(data)
+
+  if upload_xattr is not None:
+    setxattr(str(file_path), uploader.UPLOAD_ATTR_NAME, upload_xattr)
+
+class MockResponse:
+  def __init__(self, text, status_code):
+    self.text = text
+    self.status_code = status_code
+
+class MockApi:
+  def __init__(self, dongle_id):
+    pass
+
+  def get(self, *args, **kwargs):
+    return MockResponse('{"url": "http://localhost/does/not/exist", "headers": {}}', 200)
+
+  def get_token(self):
+    return "fake-token"
+
+class MockApiIgnore:
+  def __init__(self, dongle_id):
+    pass
+
+  def get(self, *args, **kwargs):
+    return MockResponse('', 412)
+
+  def get_token(self):
+    return "fake-token"
+
+class UploaderTestCase(OpenpilotTestCase):
+  f_type = "UNKNOWN"
+
+  root: Path
+  seg_num: int
+  seg_format: str
+  seg_format2: str
+  seg_dir: str
+
+  def set_ignore(self):
+    uploader.Api = MockApiIgnore  # ty: ignore[invalid-assignment]  # test double
+
+  def setup_method(self):
+    uploader.Api = MockApi  # ty: ignore[invalid-assignment]  # test double
+    uploader.fake_upload = True
+    uploader.force_wifi = True
+    uploader.allow_sleep = False
+    self.seg_num = random.randint(1, 300)
+    self.seg_format = "00000004--0ac3964c96--{}"
+    self.seg_format2 = "00000005--4c4e99b08b--{}"
+    self.seg_dir = self.seg_format.format(self.seg_num)
+
+    self.params = Params()
+    self.params.put("IsOffroad", True, block=True)
+    self.params.put("DongleId", "0000000000000000", block=True)
+
+  def make_file_with_data(self, f_dir: str, fn: str, size_mb: float = .1, lock: bool = False,
+                          upload_xattr: bytes | None = None, preserve_xattr: bytes | None = None) -> Path:
+    file_path = Path(Paths.log_root()) / f_dir / fn
+    create_random_file(file_path, size_mb, lock, upload_xattr)
+
+    if preserve_xattr is not None:
+      setxattr(str(file_path.parent), deleter.PRESERVE_ATTR_NAME, preserve_xattr)
+
+    return file_path
@@ -0,0 +1,80 @@
+from collections import namedtuple
+from pathlib import Path
+from collections.abc import Sequence
+
+import openpilot.system.loggerd.deleter as deleter
+from openpilot.system.loggerd.tests.loggerd_tests_common import UploaderTestCase
+
+Stats = namedtuple("Stats", ['f_bavail', 'f_blocks', 'f_frsize'])
+
+
+class TestDeleter(UploaderTestCase):
+  # Deletion behavior is independent of file size; use smaller files to keep these tests fast.
+  def make_file_with_data(self, f_dir: str, fn: str, size_mb: float = .001, lock: bool = False,
+                          upload_xattr: bytes | None = None, preserve_xattr: bytes | None = None) -> Path:
+    return super().make_file_with_data(f_dir, fn, size_mb, lock, upload_xattr, preserve_xattr)
+
+  def fake_statvfs(self, d):
+    return self.fake_stats
+
+  def setup_method(self):
+    self.f_type = "fcamera.hevc"
+    super().openpilot_setup_method()
+    self.fake_stats = Stats(f_bavail=0, f_blocks=10, f_frsize=4096)
+    deleter.os.statvfs = self.fake_statvfs  # ty: ignore[invalid-assignment]  # test double
+
+  def test_delete(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type)
+    assert deleter.deleter_step() == (True, str(f_path.parent))
+    assert not f_path.exists()
+
+  def assertDeleteOrder(self, f_paths: Sequence[Path]) -> None:
+    deleted_order = []
+    for _ in f_paths:
+      out_of_space, deleted_path = deleter.deleter_step()
+      assert out_of_space and deleted_path is not None
+      deleted_order.append(next(f for f in f_paths if f.parent == Path(deleted_path)))
+
+    assert deleted_order == f_paths, "Files not deleted in expected order"
+
+  def test_delete_order(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(1), self.f_type),
+      self.make_file_with_data(self.seg_format2.format(0), self.f_type),
+    ])
+
+  def test_delete_many_preserved(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(1), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE),
+      self.make_file_with_data(self.seg_format.format(2), self.f_type),
+    ] + [
+      self.make_file_with_data(self.seg_format2.format(i), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE)
+      for i in range(5)
+    ])
+
+  def test_delete_last(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(1), self.f_type),
+      self.make_file_with_data(self.seg_format2.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(0), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE),
+      self.make_file_with_data("boot", self.seg_format[:-4]),
+      self.make_file_with_data("crash", self.seg_format2[:-4]),
+    ])
+
+  def test_no_delete_when_available_space(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type)
+
+    block_size = 4096
+    available = (10 * 1024 * 1024 * 1024) / block_size  # 10GB free
+    self.fake_stats = Stats(f_bavail=available, f_blocks=10, f_frsize=block_size)
+
+    assert deleter.deleter_step() == (False, None)
+    assert f_path.exists(), "File deleted with available space"
+
+  def test_no_delete_with_lock_file(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True)
+
+    assert deleter.deleter_step() == (True, None)
+    assert f_path.exists(), "File deleted when locked"
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+
+import math
+import os
+import shutil
+import subprocess
+import time
+import unittest
+from pathlib import Path
+
+from tqdm import trange
+
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.common.hardware import COMMA_HARDWARE
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.common.hardware.hw import Paths
+
+SEGMENT_LENGTH = 2
+FULL_SIZE = 2507572
+def hevc_size(w): return FULL_SIZE // 2 if w <= 1344 else FULL_SIZE
+CAMERAS = [
+  ("fcamera.hevc", 20, hevc_size, "narrowRoadEncodeIdx"),
+  ("dcamera.hevc", 20, hevc_size, "cabinEncodeIdx"),
+  ("ecamera.hevc", 20, hevc_size, "wideRoadEncodeIdx"),
+  ("qcamera.ts", 20, lambda x: 130000, None),
+]
+
+# we check frame count, so we don't have to be too strict on size
+FILE_SIZE_TOLERANCE = 0.7
+
+
+class TestEncoder(OpenpilotTestCase):
+  COMMA_HARDWARE_TEST = True
+
+  def setup_method(self):
+    self._clear_logs()
+    os.environ["LOGGERD_TEST"] = "1"
+    os.environ["LOGGERD_SEGMENT_LENGTH"] = str(SEGMENT_LENGTH)
+
+  def teardown_method(self):
+    self._clear_logs()
+
+  def _clear_logs(self):
+    if os.path.exists(Paths.log_root()):
+      shutil.rmtree(Paths.log_root())
+
+  def _get_latest_segment_path(self):
+    last_route = sorted(Path(Paths.log_root()).iterdir())[-1]
+    return os.path.join(Paths.log_root(), last_route)
+
+  # TODO: this should run faster than real time
+  def test_log_rotation(self):
+    Params().put_bool("RecordFront", True, block=True)
+
+    managed_processes['sensord'].start()
+    managed_processes['loggerd'].start()
+    managed_processes['encoderd'].start()
+
+    time.sleep(1.0)
+    managed_processes['camerad'].start()
+
+    num_segments = 3
+
+    # wait for loggerd to make the dir for first segment
+    route_prefix_path = None
+    with Timeout(int(SEGMENT_LENGTH*3)):
+      while route_prefix_path is None:
+        try:
+          route_prefix_path = self._get_latest_segment_path().rsplit("--", 1)[0]
+        except Exception:
+          time.sleep(0.1)
+
+    def check_seg(i):
+      # check each camera file size
+      counts = []
+      first_frames = []
+      for camera, fps, size_lambda, encode_idx_name in CAMERAS:
+        file_path = f"{route_prefix_path}--{i}/{camera}"
+
+        # check file exists
+        assert os.path.exists(file_path), f"segment #{i}: '{file_path}' missing"
+
+        # TODO: this ffprobe call is really slow
+        # get width and check frame count
+        cmd = f"ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets,width -of csv=p=0 {file_path}"
+        if COMMA_HARDWARE:
+          cmd = "LD_LIBRARY_PATH=/usr/local/lib " + cmd
+
+        expected_frames = fps * SEGMENT_LENGTH
+        probe = subprocess.check_output(cmd, shell=True, encoding='utf8').split('\n')[0].strip().split(',')
+        frame_width, frame_count = int(probe[0]), int(probe[1])
+        counts.append(frame_count)
+
+        assert frame_count == expected_frames, \
+                         f"segment #{i}: {camera} failed frame count check: expected {expected_frames}, got {frame_count}"
+
+        # sanity check file size
+        file_size = os.path.getsize(file_path)
+        target_size = size_lambda(frame_width)
+        assert math.isclose(file_size, target_size, rel_tol=FILE_SIZE_TOLERANCE), \
+                        f"{file_path} size {file_size} isn't close to target size {target_size}"
+
+        # Check encodeIdx
+        if encode_idx_name is not None:
+          rlog_path = f"{route_prefix_path}--{i}/rlog.zst"
+          msgs = [m for m in LogReader(rlog_path) if m.which() == encode_idx_name]
+          encode_msgs = [getattr(m, encode_idx_name) for m in msgs]
+
+          valid = [m.valid for m in msgs]
+          segment_idxs = [m.segmentId for m in encode_msgs]
+          encode_idxs = [m.encodeId for m in encode_msgs]
+          frame_idxs = [m.frameId for m in encode_msgs]
+
+          # Check frame count
+          assert frame_count == len(segment_idxs)
+          assert frame_count == len(encode_idxs)
+
+          # Check for duplicates or skips
+          assert 0 == segment_idxs[0]
+          assert len(set(segment_idxs)) == len(segment_idxs)
+
+          assert all(valid)
+
+          assert expected_frames * i == encode_idxs[0]
+          first_frames.append(frame_idxs[0])
+          assert len(set(encode_idxs)) == len(encode_idxs)
+
+      assert 1 == len(set(first_frames))
+
+      if COMMA_HARDWARE:
+        expected_frames = fps * SEGMENT_LENGTH
+        assert min(counts) == expected_frames
+      shutil.rmtree(f"{route_prefix_path}--{i}")
+
+    try:
+      for i in trange(num_segments):
+        # poll for next segment
+        with Timeout(int(SEGMENT_LENGTH*10), error_msg=f"timed out waiting for segment {i}"):
+          while Path(f"{route_prefix_path}--{i+1}") not in Path(Paths.log_root()).iterdir():
+            time.sleep(0.1)
+        check_seg(i)
+    finally:
+      managed_processes['loggerd'].stop()
+      managed_processes['encoderd'].stop()
+      managed_processes['camerad'].stop()
+      managed_processes['sensord'].stop()
+
+
+if __name__ == "__main__":
+  unittest.main()
@@ -0,0 +1,341 @@
+import numpy as np
+import os
+import re
+import random
+import string
+import subprocess
+import time
+from collections.abc import Collection
+from collections import defaultdict
+from pathlib import Path
+
+from openpilot.common.parameterized import parameterized
+from openpilot.common.test import OpenpilotTestCase
+import openpilot.cereal.messaging as messaging
+from openpilot.cereal import log
+from openpilot.cereal.services import SERVICE_LIST
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.hardware import COMMA_HARDWARE
+from openpilot.system.loggerd.xattr_cache import getxattr
+from openpilot.system.loggerd.deleter import PRESERVE_ATTR_NAME, PRESERVE_ATTR_VALUE
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.common.version import get_version
+from openpilot.tools.lib.helpers import RE
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.cereal.visionipc import VisionStreamType
+from msgq.visionipc import VisionIpcServer
+
+SentinelType = log.Sentinel.SentinelType
+
+CEREAL_SERVICES = [f for f in log.Event.schema.union_fields if f in SERVICE_LIST
+                   and SERVICE_LIST[f].should_log and "encode" not in f.lower()]
+
+
+class TestLoggerd(OpenpilotTestCase):
+  def _get_latest_log_dir(self):
+    log_dirs = sorted(Path(Paths.log_root()).iterdir(), key=lambda f: f.stat().st_mtime)
+    return log_dirs[-1]
+
+  def _get_log_dir(self, x):
+    for l in x.splitlines():
+      for p in l.split(' '):
+        path = Path(p.strip())
+        if path.is_dir():
+          return path
+    return None
+
+  def _get_log_fn(self, x):
+    for l in x.splitlines():
+      for p in l.split(' '):
+        path = Path(p.strip())
+        if path.is_file():
+          return path
+    return None
+
+  def _gen_bootlog(self):
+    with Timeout(5):
+      out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "openpilot/system/loggerd"), encoding='utf-8')
+
+    log_fn = self._get_log_fn(out)
+
+    # check existence
+    assert log_fn is not None
+
+    return log_fn
+
+  def _check_init_data(self, msgs):
+    msg = msgs[0]
+    assert msg.which() == 'initData'
+
+  def _check_sentinel(self, msgs, route):
+    start_type = SentinelType.startOfRoute if route else SentinelType.startOfSegment
+    assert msgs[1].sentinel.type == start_type
+
+    end_type = SentinelType.endOfRoute if route else SentinelType.endOfSegment
+    assert msgs[-1].sentinel.type == end_type
+
+  def _publish_random_messages(self, services: Collection[str]) -> dict[str, list]:
+    pm = messaging.PubMaster(list(services))
+
+    managed_processes["loggerd"].start()
+    for s in services:
+      assert pm.wait_for_readers_to_update(s, timeout=5)
+
+    sent_msgs = defaultdict(list)
+    for i in range(random.randint(2, 10) * 100):
+      for s in services:
+        try:
+          m = messaging.new_message(s)
+        except Exception:
+          m = messaging.new_message(s, random.randint(2, 10))
+        pm.send(s, m)
+        sent_msgs[s].append(m)
+
+      # Keep msgq's finite per-service queues from wrapping; this test asserts
+      # that loggerd logged every message we sent.
+      if (i + 1) % 100 == 0:
+        for s in services:
+          assert pm.wait_for_readers_to_update(s, timeout=5)
+
+    for s in services:
+      assert pm.wait_for_readers_to_update(s, timeout=5)
+    managed_processes["loggerd"].stop()
+
+    return sent_msgs
+
+  def _publish_camera_and_audio_messages(self, num_segs=1, segment_length=5):
+    # Use small frame sizes for testing (width, height, size, stride, uv_offset)
+    # NV12 format: size = stride * height * 1.5, uv_offset = stride * height
+    w, h = 320, 240
+    frame_spec = (w, h, w * h * 3 // 2, w, w * h)
+    streams = [
+      (VisionStreamType.VISION_STREAM_NARROW_ROAD, frame_spec, "narrowRoadCameraState"),
+      (VisionStreamType.VISION_STREAM_CABIN, frame_spec, "cabinCameraState"),
+      (VisionStreamType.VISION_STREAM_WIDE_ROAD, frame_spec, "wideRoadCameraState"),
+    ]
+
+    sm = messaging.SubMaster(["narrowRoadEncodeData"])
+    pm = messaging.PubMaster([s for _, _, s in streams] + ["rawAudioData"])
+    vipc_server = VisionIpcServer("camerad")
+    for stream_type, frame_spec, _ in streams:
+      vipc_server.create_buffers_with_sizes(stream_type, 40, *(frame_spec))
+    vipc_server.start_listener()
+
+    os.environ["LOGGERD_TEST"] = "1"
+    os.environ["LOGGERD_SEGMENT_LENGTH"] = str(segment_length)
+    managed_processes["loggerd"].start()
+    managed_processes["encoderd"].start()
+    assert pm.wait_for_readers_to_update("narrowRoadCameraState", timeout=5)
+
+    fps = 20
+    for n in range(1, int(num_segs * segment_length * fps) + 1):
+      # send video
+      for stream_type, frame_spec, state in streams:
+        dat = np.empty(frame_spec[2], dtype=np.uint8)
+        vipc_server.send(stream_type, dat[:].flatten().tobytes(), n, n / fps, n / fps)
+
+        camera_state = messaging.new_message(state)
+        frame = getattr(camera_state, state)
+        frame.frameId = n
+        pm.send(state, camera_state)
+
+      # send audio
+      msg = messaging.new_message('rawAudioData')
+      msg.rawAudioData.data = bytes(800 * 2) # 800 samples of int16
+      msg.rawAudioData.sampleRate = 16000
+      pm.send('rawAudioData', msg)
+
+      for _, _, state in streams:
+        assert pm.wait_for_readers_to_update(state, timeout=5, dt=0.001)
+
+      sm.update(100)  # wait for encode data publish
+
+    managed_processes["loggerd"].stop()
+    managed_processes["encoderd"].stop()
+
+  def test_init_data_values(self):
+    os.environ["CLEAN"] = random.choice(["0", "1"])
+
+    dongle  = ''.join(random.choice(string.printable) for n in range(random.randint(1, 100)))
+    fake_params = [
+      # param, initData field, value
+      ("DongleId", "dongleId", dongle),
+      ("GitCommit", "gitCommit", "commit"),
+      ("GitCommitDate", "gitCommitDate", "date"),
+      ("GitBranch", "gitBranch", "branch"),
+      ("GitRemote", "gitRemote", "remote"),
+    ]
+    params = Params()
+    for k, _, v in fake_params:
+      params.put(k, v, block=True)
+    params.put("AccessToken", "abc", block=True)
+
+    lr = list(LogReader(str(self._gen_bootlog())))
+    initData = lr[0].initData
+
+    assert initData.dirty != bool(os.environ["CLEAN"])
+    assert initData.version == get_version()
+
+    if os.path.isfile("/proc/cmdline"):
+      with open("/proc/cmdline") as f:
+        assert list(initData.kernelArgs) == f.read().strip().split(" ")
+
+      with open("/proc/version") as f:
+        assert initData.kernelVersion == f.read()
+
+    # check params
+    logged_params = {entry.key: entry.value for entry in initData.params.entries}
+    expected_params = {k for k, _, __ in fake_params} | {'AccessToken', 'BootCount'}
+    assert set(logged_params.keys()) == expected_params, set(logged_params.keys()) ^ expected_params
+    assert logged_params['AccessToken'] == b'', f"DONT_LOG param value was logged: {repr(logged_params['AccessToken'])}"
+    for param_key, initData_key, v in fake_params:
+      assert getattr(initData, initData_key) == v
+      assert logged_params[param_key].decode() == v
+
+  def test_rotation(self):
+    Params().put("RecordFront", True, block=True)
+
+    expected_files = {"rlog.zst", "qlog.zst", "qcamera.ts", "fcamera.hevc", "dcamera.hevc", "ecamera.hevc"}
+
+    num_segs = random.randint(2, 3)
+    length = random.randint(4, 5) # H264 encoder uses 40 lookahead frames and does B-frame reordering, so minimum 3 seconds before qcam output
+
+    self._publish_camera_and_audio_messages(num_segs=num_segs, segment_length=length)
+
+    route_path = str(self._get_latest_log_dir()).rsplit("--", 1)[0]
+    for n in range(num_segs):
+      p = Path(f"{route_path}--{n}")
+      logged = {f.name for f in p.iterdir() if f.is_file()}
+      diff = logged ^ expected_files
+      assert len(diff) == 0, f"didn't get all expected files. seg={n} {route_path=}, {diff=}\n{logged=} {expected_files=}"
+
+  def test_bootlog(self):
+    # generate bootlog with fake launch log
+    launch_log = ''.join(str(random.choice(string.printable)) for _ in range(100))
+    with open("/tmp/launch_log", "w") as f:
+      f.write(launch_log)
+
+    bootlog_path = self._gen_bootlog()
+    lr = list(LogReader(str(bootlog_path)))
+
+    # check length
+    assert len(lr) == 2  # boot + initData
+
+    self._check_init_data(lr)
+
+    # check msgs
+    bootlog_msgs = [m for m in lr if m.which() == 'boot']
+    assert len(bootlog_msgs) == 1
+
+    # sanity check values
+    boot = bootlog_msgs.pop().boot
+    assert abs(boot.wallTimeNanos - time.time_ns()) < 5*1e9 # within 5s
+    assert boot.launchLog == launch_log
+
+    if COMMA_HARDWARE:
+      for fn in ["console-ramoops", "pmsg-ramoops-0"]:
+        path = Path(os.path.join("/sys/fs/pstore/", fn))
+        if path.is_file():
+          with open(path, "rb") as f:
+            expected_val = f.read()
+          bootlog_val = [e.value for e in boot.pstore.entries if e.key == fn][0]
+          assert expected_val == bootlog_val
+    else:
+      assert len(boot.pstore.entries) == 0
+
+    # next one should increment by one
+    bl1 = re.match(RE.LOG_ID_V2, bootlog_path.name)
+    bl2 = re.match(RE.LOG_ID_V2, self._gen_bootlog().name)
+    assert bl1.group('uid') != bl2.group('uid')
+    assert int(bl1.group('count')) == 0 and int(bl2.group('count')) == 1
+
+  def test_qlog(self):
+    qlog_services = [s for s in CEREAL_SERVICES if SERVICE_LIST[s].decimation is not None]
+    no_qlog_services = [s for s in CEREAL_SERVICES if SERVICE_LIST[s].decimation is None]
+
+    services = random.sample(qlog_services, random.randint(2, min(10, len(qlog_services)))) + \
+               random.sample(no_qlog_services, random.randint(2, min(10, len(no_qlog_services))))
+    sent_msgs = self._publish_random_messages(services)
+
+    qlog_path = os.path.join(self._get_latest_log_dir(), "qlog.zst")
+    lr = list(LogReader(qlog_path))
+
+    # check initData and sentinel
+    self._check_init_data(lr)
+    self._check_sentinel(lr, True)
+
+    recv_msgs = defaultdict(list)
+    for m in lr:
+      recv_msgs[m.which()].append(m)
+
+    for s, msgs in sent_msgs.items():
+      recv_cnt = len(recv_msgs[s])
+
+      if s in no_qlog_services:
+        # check services with no specific decimation aren't in qlog
+        assert recv_cnt == 0, f"got {recv_cnt} {s} msgs in qlog"
+      else:
+        # check logged message count matches decimation
+        decimation = SERVICE_LIST[s].decimation
+        assert decimation is not None
+        expected_cnt = (len(msgs) - 1) // decimation + 1
+        assert recv_cnt == expected_cnt, f"expected {expected_cnt} msgs for {s}, got {recv_cnt}"
+
+  def test_rlog(self):
+    services = random.sample(CEREAL_SERVICES, random.randint(5, 10))
+    sent_msgs = self._publish_random_messages(services)
+
+    lr = list(LogReader(os.path.join(self._get_latest_log_dir(), "rlog.zst")))
+
+    # check initData and sentinel
+    self._check_init_data(lr)
+    self._check_sentinel(lr, True)
+
+    # check all messages were logged and in order
+    lr = lr[2:-1] # slice off initData and both sentinels
+    for m in lr:
+      sent = sent_msgs[m.which()].pop(0)
+      sent.clear_write_flag()
+      assert sent.to_bytes() == m.as_builder().to_bytes()
+
+  def test_preserving_bookmarked_segments(self):
+    services = set(random.sample(CEREAL_SERVICES, random.randint(5, 10))) | {"userBookmark"}
+    self._publish_random_messages(services)
+
+    segment_dir = self._get_latest_log_dir()
+    assert getxattr(segment_dir, PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+  def test_not_preserving_nonbookmarked_segments(self):
+    services = set(random.sample(CEREAL_SERVICES, random.randint(5, 10))) - {"userBookmark"}
+    self._publish_random_messages(services)
+
+    segment_dir = self._get_latest_log_dir()
+    assert getxattr(segment_dir, PRESERVE_ATTR_NAME) is None
+
+  @parameterized.expand([True, False])
+  def test_record_front(self, record_front):
+    params = Params()
+    params.put_bool("RecordFront", record_front, block=True)
+
+    self._publish_camera_and_audio_messages()
+
+    cabin_hevc_exists = os.path.exists(os.path.join(self._get_latest_log_dir(), 'dcamera.hevc'))
+    assert cabin_hevc_exists == record_front
+
+  @parameterized.expand([True, False])
+  def test_record_audio(self, record_audio):
+    params = Params()
+    params.put_bool("RecordAudio", record_audio, block=True)
+
+    self._publish_camera_and_audio_messages()
+
+    qcamera_ts_path = os.path.join(self._get_latest_log_dir(), 'qcamera.ts')
+    ffprobe_cmd = f"ffprobe -i {qcamera_ts_path} -show_streams -select_streams a -loglevel error"
+    has_audio_stream = subprocess.run(ffprobe_cmd, shell=True, capture_output=True).stdout.strip() != b''
+    assert has_audio_stream == record_audio
+
+    raw_audio_in_rlog = any(m.which() == 'rawAudioData' for m in LogReader(os.path.join(self._get_latest_log_dir(), 'rlog.zst')))
+    assert raw_audio_in_rlog == record_audio
@@ -0,0 +1,181 @@
+import os
+import threading
+import logging
+import json
+from pathlib import Path
+from openpilot.common.hardware.hw import Paths
+
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.loggerd.uploader import clear_locks, main, Uploader, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE
+
+from openpilot.system.loggerd.tests.loggerd_tests_common import UploaderTestCase
+
+
+class FakeLogHandler(logging.Handler):
+  def __init__(self):
+    logging.Handler.__init__(self)
+    self.condition = threading.Condition()
+    self.reset()
+
+  def reset(self):
+    with self.condition:
+      self.upload_order = []
+      self.upload_ignored = []
+
+  def emit(self, record):
+    try:
+      j = json.loads(record.getMessage())
+      with self.condition:
+        if j["event"] == "upload_success":
+          self.upload_order.append(j["key"])
+        if j["event"] == "upload_ignored":
+          self.upload_ignored.append(j["key"])
+        self.condition.notify_all()
+    except Exception:
+      pass
+
+  def wait_for_uploads(self, count: int, ignored: bool = False):
+    uploads = self.upload_ignored if ignored else self.upload_order
+    with self.condition:
+      assert self.condition.wait_for(lambda: len(uploads) >= count, timeout=1), "Uploader did not process all files"
+
+log_handler = FakeLogHandler()
+cloudlog.addHandler(log_handler)
+
+
+class TestUploader(UploaderTestCase):
+  def setup_method(self):
+    super().openpilot_setup_method()
+    log_handler.reset()
+
+  def start_thread(self):
+    self.end_event = threading.Event()
+    self.up_thread = threading.Thread(target=main, args=[self.end_event])
+    self.up_thread.daemon = True
+    self.up_thread.start()
+
+  def join_thread(self):
+    self.end_event.set()
+    self.up_thread.join()
+
+  def gen_files(self, lock=False, xattr: bytes | None = None, boot=True) -> list[Path]:
+    f_paths = []
+    for t in ["qlog", "rlog", "dcamera.hevc", "fcamera.hevc"]:
+      f_paths.append(self.make_file_with_data(self.seg_dir, t, 1, lock=lock, upload_xattr=xattr))
+
+    if boot:
+      f_paths.append(self.make_file_with_data("boot", f"{self.seg_dir}", 1, lock=lock, upload_xattr=xattr))
+    return f_paths
+
+  def gen_order(self, seg1: list[int], seg2: list[int], boot=True) -> list[str]:
+    keys = []
+    if boot:
+      keys += [f"boot/{self.seg_format.format(i)}.zst" for i in seg1]
+      keys += [f"boot/{self.seg_format2.format(i)}.zst" for i in seg2]
+    keys += [f"{self.seg_format.format(i)}/qlog.zst" for i in seg1]
+    keys += [f"{self.seg_format2.format(i)}/qlog.zst" for i in seg2]
+    return keys
+
+  def test_upload(self):
+    self.gen_files(lock=False)
+    exp_order = self.gen_order([self.seg_num], [])
+
+    self.start_thread()
+    log_handler.wait_for_uploads(len(exp_order))
+    self.join_thread()
+
+    assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
+    assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
+    assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
+
+    assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
+
+  def test_upload_with_wrong_xattr(self):
+    self.gen_files(lock=False, xattr=b'0')
+    exp_order = self.gen_order([self.seg_num], [])
+
+    self.start_thread()
+    log_handler.wait_for_uploads(len(exp_order))
+    self.join_thread()
+
+    assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
+    assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
+    assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
+
+    assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
+
+  def test_upload_ignored(self):
+    self.set_ignore()
+    self.gen_files(lock=False)
+    exp_order = self.gen_order([self.seg_num], [])
+
+    self.start_thread()
+    log_handler.wait_for_uploads(len(exp_order), ignored=True)
+    self.join_thread()
+
+    assert len(log_handler.upload_order) == 0, "Some files were not ignored"
+    assert not len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore"
+    assert not len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not ignored"
+
+    assert log_handler.upload_ignored == exp_order, "Files ignored in wrong order"
+
+  def test_upload_files_in_create_order(self):
+    seg1_nums = [0, 1, 2, 10, 20]
+    for i in seg1_nums:
+      self.seg_dir = self.seg_format.format(i)
+      self.gen_files(boot=False)
+    seg2_nums = [5, 50, 51]
+    for i in seg2_nums:
+      self.seg_dir = self.seg_format2.format(i)
+      self.gen_files(boot=False)
+
+    exp_order = self.gen_order(seg1_nums, seg2_nums, boot=False)
+
+    self.start_thread()
+    log_handler.wait_for_uploads(len(exp_order))
+    self.join_thread()
+
+    assert len(log_handler.upload_ignored) == 0, "Some files were ignored"
+    assert not len(log_handler.upload_order) < len(exp_order), "Some files failed to upload"
+    assert not len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice"
+    for f_path in exp_order:
+      assert os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE, "All files not uploaded"
+
+    assert log_handler.upload_order == exp_order, "Files uploaded in wrong order"
+
+  def test_no_upload_with_lock_file(self):
+    f_paths = self.gen_files(lock=True, boot=False)
+    uploader = Uploader("0000000000000000", Paths.log_root())
+
+    for f_path in f_paths:
+      fn = f_path.with_suffix(f_path.suffix.replace(".zst", ""))
+      assert all(candidate[2] != str(fn) for candidate in uploader.list_upload_files(metered=False)), "Locked file selected for upload"
+
+  def test_no_upload_with_xattr(self):
+    f_paths = self.gen_files(lock=False, xattr=UPLOAD_ATTR_VALUE)
+    uploader = Uploader("0000000000000000", Paths.log_root())
+    upload_candidates = {candidate[2] for candidate in uploader.list_upload_files(metered=False)}
+    assert upload_candidates.isdisjoint(map(str, f_paths)), "Uploaded file selected again"
+
+  def test_clear_locks_on_startup(self, mocker):
+    f_paths = self.gen_files(lock=True, boot=False)
+    locks_cleared = threading.Event()
+
+    def clear_locks_and_signal(root):
+      clear_locks(root)
+      locks_cleared.set()
+
+    mocker.patch("openpilot.system.loggerd.uploader.clear_locks", side_effect=clear_locks_and_signal)
+    self.start_thread()
+    assert locks_cleared.wait(timeout=1), "Uploader did not clear locks on startup"
+    self.join_thread()
+
+    for f_path in f_paths:
+      lock_path = f_path.with_suffix(f_path.suffix + ".lock")
+      assert not lock_path.is_file(), "File lock not cleared on startup"
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+cd /sys/kernel/debug/tracing
+echo "" > trace
+echo 1 > tracing_on
+echo 1 > /sys/kernel/debug/tracing/events/msm_vidc/enable
+
+echo 0xff > /sys/module/videobuf2_core/parameters/debug
+echo 0x7fffffff > /sys/kernel/debug/msm_vidc/debug_level
+echo 0xff > /sys/devices/platform/soc/aa00000.qcom,vidc/video4linux/video33/dev_debug
+
+cat /sys/kernel/debug/tracing/trace_pipe
@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+import json
+import os
+import random
+import requests
+import threading
+import time
+import traceback
+import datetime
+from collections.abc import Iterator
+
+from openpilot.cereal import log
+import openpilot.cereal.messaging as messaging
+from openpilot.common.api import Api
+from openpilot.common.utils import get_upload_stream
+from openpilot.common.params import Params
+from openpilot.common.realtime import set_core_affinity
+from openpilot.common.hardware.hw import Paths
+from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
+from openpilot.common.swaglog import cloudlog
+
+NetworkType = log.DeviceState.NetworkType
+UPLOAD_ATTR_NAME = 'user.upload'
+UPLOAD_ATTR_VALUE = b'1'
+
+MAX_UPLOAD_SIZES = {
+  "qlog": 25*1e6,  # can't be too restrictive here since we use qlogs to find
+                   # bugs, including ones that can cause massive log sizes
+  "qcam": 5*1e6,
+}
+
+allow_sleep = bool(int(os.getenv("UPLOADER_SLEEP", "1")))
+force_wifi = os.getenv("FORCEWIFI") is not None
+fake_upload = os.getenv("FAKEUPLOAD") is not None
+
+
+class FakeRequest:
+  def __init__(self):
+    self.headers = {"Content-Length": "0"}
+
+
+class FakeResponse:
+  def __init__(self):
+    self.status_code = 200
+    self.request = FakeRequest()
+
+
+def get_directory_sort(d: str) -> list[str]:
+  return [s.rjust(10, '0') for s in d.rsplit('--', 1)]
+
+def listdir_by_creation(d: str) -> list[str]:
+  if not os.path.isdir(d):
+    return []
+
+  try:
+    paths = [f for f in os.listdir(d) if os.path.isdir(os.path.join(d, f))]
+    paths = sorted(paths, key=get_directory_sort)
+    return paths
+  except OSError:
+    cloudlog.exception("listdir_by_creation failed")
+    return []
+
+def clear_locks(root: str) -> None:
+  for logdir in os.listdir(root):
+    path = os.path.join(root, logdir)
+    try:
+      for fname in os.listdir(path):
+        if fname.endswith(".lock"):
+          os.unlink(os.path.join(path, fname))
+    except OSError:
+      cloudlog.exception("clear_locks failed")
+
+
+class Uploader:
+  def __init__(self, dongle_id: str, root: str):
+    self.dongle_id = dongle_id
+    self.api = Api(dongle_id)
+    self.root = root
+
+    self.params = Params()
+
+    # stats for last successfully uploaded file
+    self.last_filename = ""
+
+    self.immediate_folders = ["crash/", "boot/"]
+    self.immediate_priority = {"qlog": 0, "qlog.zst": 0, "qcamera.ts": 1}
+
+  def list_upload_files(self, metered: bool) -> Iterator[tuple[str, str, str]]:
+    r = self.params.get("AthenadRecentlyViewedRoutes")
+    requested_routes = [] if r is None else [route for route in r.split(",") if route]
+
+    for logdir in listdir_by_creation(self.root):
+      path = os.path.join(self.root, logdir)
+      try:
+        names = os.listdir(path)
+      except OSError:
+        continue
+
+      if any(name.endswith(".lock") for name in names):
+        continue
+
+      for name in sorted(names, key=lambda n: self.immediate_priority.get(n, 1000)):
+        key = os.path.join(logdir, name)
+        fn = os.path.join(path, name)
+        # skip files already uploaded
+        try:
+          ctime = os.path.getctime(fn)
+          is_uploaded = getxattr(fn, UPLOAD_ATTR_NAME) == UPLOAD_ATTR_VALUE
+        except OSError:
+          cloudlog.event("uploader_getxattr_failed", key=key, fn=fn)
+          # deleter could have deleted, so skip
+          continue
+        if is_uploaded:
+          continue
+
+        # limit uploading on metered connections
+        if metered:
+          dt = datetime.timedelta(hours=12)
+          if logdir in self.immediate_folders and (datetime.datetime.now() - datetime.datetime.fromtimestamp(ctime)) < dt:
+            continue
+
+          if name == "qcamera.ts" and not any(logdir.startswith(r.split('|')[-1]) for r in requested_routes):
+            continue
+
+        yield name, key, fn
+
+  def next_file_to_upload(self, metered: bool) -> tuple[str, str, str] | None:
+    upload_files = list(self.list_upload_files(metered))
+
+    for name, key, fn in upload_files:
+      if any(f in fn for f in self.immediate_folders):
+        return name, key, fn
+
+    for name, key, fn in upload_files:
+      if name in self.immediate_priority:
+        return name, key, fn
+
+    return None
+
+  def do_upload(self, key: str, fn: str):
+    url_resp = self.api.get("v1.4/" + self.dongle_id + "/upload_url/", timeout=10, path=key, access_token=self.api.get_token())
+    if url_resp.status_code == 412:
+      return url_resp
+
+    url_resp_json = json.loads(url_resp.text)
+    url = url_resp_json['url']
+    headers = url_resp_json['headers']
+    cloudlog.debug("upload_url v1.4 %s %s", url, str(headers))
+
+    if fake_upload:
+      return FakeResponse()
+
+    stream = None
+    try:
+      compress = key.endswith('.zst') and not fn.endswith('.zst')
+      stream, _ = get_upload_stream(fn, compress)
+      response = requests.put(url, data=stream, headers=headers, timeout=10)
+      return response
+    finally:
+      if stream:
+        stream.close()
+
+  def upload(self, name: str, key: str, fn: str, network_type: int, metered: bool) -> bool:
+    try:
+      sz = os.path.getsize(fn)
+    except OSError:
+      cloudlog.exception("upload: getsize failed")
+      return False
+
+    cloudlog.event("upload_start", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if sz == 0:
+      # tag files of 0 size as uploaded
+      success = True
+    elif name in MAX_UPLOAD_SIZES and sz > MAX_UPLOAD_SIZES[name]:
+      cloudlog.event("uploader_too_large", key=key, fn=fn, sz=sz)
+      success = True
+    else:
+      start_time = time.monotonic()
+
+      stat = None
+      last_exc = None
+      try:
+        stat = self.do_upload(key, fn)
+      except Exception as e:
+        last_exc = (e, traceback.format_exc())
+
+      if stat is not None and stat.status_code in (200, 201, 401, 403, 412):
+        self.last_filename = fn
+        dt = time.monotonic() - start_time
+        if stat.status_code == 412:
+          cloudlog.event("upload_ignored", key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+        else:
+          content_length = int(stat.request.headers.get("Content-Length", 0))
+          speed = (content_length / 1e6) / dt
+          cloudlog.event("upload_success", key=key, fn=fn, sz=sz, content_length=content_length,
+                         network_type=network_type, metered=metered, speed=speed)
+        success = True
+      else:
+        success = False
+        cloudlog.event("upload_failed", stat=stat, exc=last_exc, key=key, fn=fn, sz=sz, network_type=network_type, metered=metered)
+
+    if success:
+      # tag file as uploaded
+      try:
+        setxattr(fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)
+      except OSError:
+        cloudlog.event("uploader_setxattr_failed", exc=last_exc, key=key, fn=fn, sz=sz)
+
+    return success
+
+
+  def step(self, network_type: int, metered: bool) -> bool | None:
+    d = self.next_file_to_upload(metered)
+    if d is None:
+      return None
+
+    name, key, fn = d
+
+    # qlogs and bootlogs need to be compressed before uploading
+    if key.endswith(('qlog', 'rlog')) or (key.startswith('boot/') and not key.endswith('.zst')):
+      key += ".zst"
+
+    return self.upload(name, key, fn, network_type, metered)
+
+
+def main(exit_event: threading.Event | None = None) -> None:
+  if exit_event is None:
+    exit_event = threading.Event()
+
+  try:
+    set_core_affinity([0, 1, 2, 3])
+  except Exception:
+    cloudlog.exception("failed to set core affinity")
+
+  clear_locks(Paths.log_root())
+
+  params = Params()
+  dongle_id = params.get("DongleId")
+
+  if dongle_id is None:
+    cloudlog.info("uploader missing dongle_id")
+    raise Exception("uploader can't start without dongle id")
+
+  sm = messaging.SubMaster(['deviceState'])
+  uploader = Uploader(dongle_id, Paths.log_root())
+
+  backoff = 0.1
+  while not exit_event.is_set():
+    sm.update(0)
+    offroad = params.get_bool("IsOffroad")
+    network_type = sm['deviceState'].networkType if not force_wifi else NetworkType.wifi
+    if network_type == NetworkType.none:
+      if allow_sleep:
+        time.sleep(60 if offroad else 5)
+      continue
+
+    success = uploader.step(sm['deviceState'].networkType.raw, sm['deviceState'].networkMetered)
+    if success is None:
+      backoff = 60 if offroad else 5
+    elif success:
+      backoff = 0.1
+    else:
+      cloudlog.info("upload backoff %r", backoff)
+      backoff = min(backoff*2, 120)
+    if allow_sleep:
+      time.sleep(backoff + random.uniform(0, backoff))
+
+
+if __name__ == "__main__":
+  main()
@@ -0,0 +1,242 @@
+#include <cassert>
+
+#include "system/loggerd/video_writer.h"
+#include "common/swaglog.h"
+#include "common/util.h"
+
+VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec)
+  : remuxing(remuxing) {
+  vid_path = util::string_format("%s/%s", path, filename);
+  lock_path = util::string_format("%s/%s.lock", path, filename);
+
+  int lock_fd = HANDLE_EINTR(open(lock_path.c_str(), O_RDWR | O_CREAT, 0664));
+  assert(lock_fd >= 0);
+  close(lock_fd);
+
+  LOGD("encoder_open %s remuxing:%d", this->vid_path.c_str(), this->remuxing);
+  if (this->remuxing) {
+    bool raw = (codec == cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS);
+    avformat_alloc_output_context2(&this->ofmt_ctx, NULL, raw ? "matroska" : NULL, this->vid_path.c_str());
+    assert(this->ofmt_ctx);
+
+    // set codec correctly. needed?
+    assert(codec != cereal::EncodeIndex::Type::FULL_H_E_V_C);
+    const AVCodec *avcodec = avcodec_find_encoder(raw ? AV_CODEC_ID_FFVHUFF : AV_CODEC_ID_H264);
+    assert(avcodec);
+
+    this->codec_ctx = avcodec_alloc_context3(avcodec);
+    assert(this->codec_ctx);
+    this->codec_ctx->width = width;
+    this->codec_ctx->height = height;
+    this->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
+    this->codec_ctx->time_base = (AVRational){ 1, fps };
+
+    if (codec == cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS) {
+      // without this, there's just noise
+      int err = avcodec_open2(this->codec_ctx, avcodec, NULL);
+      assert(err >= 0);
+    }
+
+    this->out_stream = avformat_new_stream(this->ofmt_ctx, raw ? avcodec : NULL);
+    assert(this->out_stream);
+
+    int err = avio_open(&this->ofmt_ctx->pb, this->vid_path.c_str(), AVIO_FLAG_WRITE);
+    assert(err >= 0);
+
+  } else {
+    this->of = util::safe_fopen(this->vid_path.c_str(), "wb");
+    assert(this->of);
+  }
+}
+
+void VideoWriter::set_metadata(const char *key, const char *value) {
+  assert(remuxing && !header_written);
+  av_dict_set(&ofmt_ctx->metadata, key, value, 0);
+}
+
+void VideoWriter::initialize_audio(int sample_rate) {
+  assert(this->ofmt_ctx->oformat->audio_codec != AV_CODEC_ID_NONE); // check output format supports audio streams
+  const AVCodec *audio_avcodec = avcodec_find_encoder(AV_CODEC_ID_AAC);
+  assert(audio_avcodec);
+  this->audio_codec_ctx = avcodec_alloc_context3(audio_avcodec);
+  assert(this->audio_codec_ctx);
+  this->audio_codec_ctx->sample_fmt = AV_SAMPLE_FMT_FLTP;
+  this->audio_codec_ctx->sample_rate = sample_rate;
+  #if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 28, 100)  // FFmpeg 5.1+
+  av_channel_layout_default(&this->audio_codec_ctx->ch_layout, 1);
+  #else
+  this->audio_codec_ctx->channel_layout = AV_CH_LAYOUT_MONO;
+  #endif
+  this->audio_codec_ctx->bit_rate = 32000;
+  this->audio_codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+  this->audio_codec_ctx->time_base = (AVRational){1, audio_codec_ctx->sample_rate};
+  int err = avcodec_open2(this->audio_codec_ctx, audio_avcodec, NULL);
+  assert(err >= 0);
+  av_log_set_level(AV_LOG_WARNING); // hide "QAvg" info msgs at the end of every segment
+
+  this->audio_stream = avformat_new_stream(this->ofmt_ctx, NULL);
+  assert(this->audio_stream);
+  err = avcodec_parameters_from_context(this->audio_stream->codecpar, this->audio_codec_ctx);
+  assert(err >= 0);
+
+  this->audio_frame = av_frame_alloc();
+  assert(this->audio_frame);
+  this->audio_frame->format = this->audio_codec_ctx->sample_fmt;
+  #if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 28, 100)  // FFmpeg 5.1+
+  av_channel_layout_copy(&this->audio_frame->ch_layout, &this->audio_codec_ctx->ch_layout);
+  #else
+  this->audio_frame->channel_layout = this->audio_codec_ctx->channel_layout;
+  #endif
+  this->audio_frame->sample_rate = this->audio_codec_ctx->sample_rate;
+  this->audio_frame->nb_samples = this->audio_codec_ctx->frame_size;
+  err = av_frame_get_buffer(this->audio_frame, 0);
+  assert(err >= 0);
+}
+
+void VideoWriter::write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe) {
+  if (of && data) {
+    size_t written = util::safe_fwrite(data, 1, len, of);
+    if (written != len) {
+      LOGE("failed to write file.errno=%d", errno);
+    }
+  }
+
+  if (remuxing) {
+    if (codecconfig) {
+      if (len > 0) {
+        codec_ctx->extradata = (uint8_t*)av_mallocz(len + AV_INPUT_BUFFER_PADDING_SIZE);
+        codec_ctx->extradata_size = len;
+        memcpy(codec_ctx->extradata, data, len);
+      }
+      int err = avcodec_parameters_from_context(out_stream->codecpar, codec_ctx);
+      assert(err >= 0);
+      // if there is an audio stream, it must be initialized before this point
+      AVDictionary *options = nullptr;
+      if (ofmt_ctx->metadata) av_dict_set(&options, "movflags", "+faststart+use_metadata_tags", 0);
+      err = avformat_write_header(ofmt_ctx, &options);
+      av_dict_free(&options);
+      assert(err >= 0);
+      header_written = true;
+    } else {
+      // input timestamps are in microseconds
+      AVRational in_timebase = {1, 1000000};
+
+      AVPacket pkt = {};
+      pkt.data = data;
+      pkt.size = len;
+      pkt.stream_index = this->out_stream->index;
+
+      enum AVRounding rnd = static_cast<enum AVRounding>(AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX);
+      pkt.pts = pkt.dts = av_rescale_q_rnd(timestamp, in_timebase, ofmt_ctx->streams[0]->time_base, rnd);
+      pkt.duration = av_rescale_q(50*1000, in_timebase, ofmt_ctx->streams[0]->time_base);
+
+      if (keyframe) {
+        pkt.flags |= AV_PKT_FLAG_KEY;
+      }
+
+      // TODO: can use av_write_frame for non raw?
+      int err = av_interleaved_write_frame(ofmt_ctx, &pkt);
+      if (err < 0) { LOGW("ts encoder write issue len: %d ts: %lld", len, timestamp); }
+
+      av_packet_unref(&pkt);
+    }
+  }
+}
+
+void VideoWriter::write_audio(uint8_t *data, int len, long long timestamp, int sample_rate) {
+  if (!remuxing) return;
+  if (!audio_initialized) {
+    initialize_audio(sample_rate);
+    audio_initialized = true;
+  }
+  if (!audio_codec_ctx) return;
+  // sync logMonoTime of first audio packet with the timestampEof of first video packet
+  if (audio_pts == 0) {
+    audio_pts = (timestamp * audio_codec_ctx->sample_rate) / 1000000ULL;
+  }
+
+  // convert s16le samples to fltp and add to buffer
+  const int16_t *raw_samples = reinterpret_cast<const int16_t*>(data);
+  int sample_count = len / sizeof(int16_t);
+  constexpr float normalizer = 1.0f / 32768.0f;
+
+  const size_t max_buffer_size = sample_rate * 10; // 10 seconds
+  if (audio_buffer.size() + sample_count > max_buffer_size) {
+    size_t samples_to_drop = (audio_buffer.size() + sample_count) - max_buffer_size;
+    LOGE("Audio buffer overflow, dropping %zu oldest samples", samples_to_drop);
+    audio_buffer.erase(audio_buffer.begin(), audio_buffer.begin() + samples_to_drop);
+    audio_pts += samples_to_drop;
+  }
+
+  // Add new samples to the buffer
+  const size_t original_size = audio_buffer.size();
+  audio_buffer.resize(original_size + sample_count);
+  std::transform(raw_samples, raw_samples + sample_count, audio_buffer.begin() + original_size,
+                [](int16_t sample) { return sample * normalizer; });
+
+  if (!header_written) return; // header not written yet, process audio frame after header is written
+  while (audio_buffer.size() >= audio_codec_ctx->frame_size) {
+    audio_frame->pts = audio_pts;
+    float *f_samples = reinterpret_cast<float*>(audio_frame->data[0]);
+    std::copy(audio_buffer.begin(), audio_buffer.begin() + audio_codec_ctx->frame_size, f_samples);
+    audio_buffer.erase(audio_buffer.begin(), audio_buffer.begin() + audio_codec_ctx->frame_size);
+    encode_and_write_audio_frame(audio_frame);
+  }
+}
+
+void VideoWriter::encode_and_write_audio_frame(AVFrame* frame) {
+  if (!remuxing || !audio_codec_ctx) return;
+  int send_result = avcodec_send_frame(audio_codec_ctx, frame); // encode frame
+  if (send_result >= 0) {
+    AVPacket *pkt = av_packet_alloc();
+    while (avcodec_receive_packet(audio_codec_ctx, pkt) == 0) {
+      av_packet_rescale_ts(pkt, audio_codec_ctx->time_base, audio_stream->time_base);
+      pkt->stream_index = audio_stream->index;
+
+      int err = av_interleaved_write_frame(ofmt_ctx, pkt); // write encoded frame
+      if (err < 0) {
+        LOGW("AUDIO: Write frame failed - error: %d", err);
+      }
+      av_packet_unref(pkt);
+    }
+    av_packet_free(&pkt);
+  } else {
+    LOGW("AUDIO: Failed to send audio frame to encoder: %d", send_result);
+  }
+  audio_pts += audio_codec_ctx->frame_size;
+}
+
+void VideoWriter::process_remaining_audio() {
+  // Process remaining audio samples by padding with silence
+  if (audio_buffer.size() > 0 && audio_buffer.size() < audio_codec_ctx->frame_size) {
+    audio_buffer.resize(audio_codec_ctx->frame_size, 0.0f);
+
+    // Encode final frame
+    audio_frame->pts = audio_pts;
+    float *f_samples = reinterpret_cast<float *>(audio_frame->data[0]);
+    std::copy(audio_buffer.begin(), audio_buffer.end(), f_samples);
+    encode_and_write_audio_frame(audio_frame);
+  }
+}
+
+VideoWriter::~VideoWriter() {
+  if (this->remuxing) {
+    if (this->audio_codec_ctx) {
+      process_remaining_audio();
+      encode_and_write_audio_frame(NULL); // flush encoder
+      avcodec_free_context(&this->audio_codec_ctx);
+    }
+    int err = av_write_trailer(this->ofmt_ctx);
+    if (err != 0) LOGE("av_write_trailer failed %d", err);
+    avcodec_free_context(&this->codec_ctx);
+    if (this->audio_frame) av_frame_free(&this->audio_frame);
+    err = avio_closep(&this->ofmt_ctx->pb);
+    if (err != 0) LOGE("avio_closep failed %d", err);
+    avformat_free_context(this->ofmt_ctx);
+  } else {
+    util::safe_fflush(this->of);
+    fclose(this->of);
+    this->of = nullptr;
+  }
+  unlink(this->lock_path.c_str());
+}
@@ -0,0 +1,43 @@
+#pragma once
+
+#include <string>
+#include <deque>
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+}
+
+#include "openpilot/cereal/messaging/messaging.h"
+
+class VideoWriter {
+public:
+  VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec);
+  void set_metadata(const char *key, const char *value);
+  void write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe);
+  void write_audio(uint8_t *data, int len, long long timestamp, int sample_rate);
+
+  ~VideoWriter();
+
+private:
+  void initialize_audio(int sample_rate);
+  void encode_and_write_audio_frame(AVFrame* frame);
+  void process_remaining_audio();
+
+  std::string vid_path, lock_path;
+  FILE *of = nullptr;
+
+  AVCodecContext *codec_ctx;
+  AVFormatContext *ofmt_ctx;
+  AVStream *out_stream;
+
+  bool audio_initialized = false;
+  bool header_written = false;
+  AVStream *audio_stream = nullptr;
+  AVCodecContext *audio_codec_ctx = nullptr;
+  AVFrame *audio_frame = nullptr;
+  uint64_t audio_pts = 0;
+  std::deque<float> audio_buffer;
+
+  bool remuxing;
+};
@@ -0,0 +1,70 @@
+import ctypes
+import errno
+import os
+import sys
+
+
+if sys.platform == "darwin":
+  _libc = ctypes.CDLL(None, use_errno=True)
+  _libc.getxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_int]
+  _libc.getxattr.restype = ctypes.c_ssize_t
+  _libc.setxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint32, ctypes.c_int]
+  _libc.setxattr.restype = ctypes.c_int
+
+
+def _raise_os_error(path: str) -> None:
+  error = ctypes.get_errno()
+  raise OSError(error, os.strerror(error), path)
+
+
+def _getxattr(path: str, attr_name: str) -> bytes:
+  if sys.platform != "darwin":
+    return os.getxattr(path, attr_name)
+
+  encoded_path = os.fsencode(path)
+  encoded_attr_name = os.fsencode(attr_name)
+  while True:
+    size = _libc.getxattr(encoded_path, encoded_attr_name, None, 0, 0, 0)
+    if size == -1:
+      _raise_os_error(path)
+    if size == 0:
+      return b""
+
+    value = ctypes.create_string_buffer(size)
+    result = _libc.getxattr(encoded_path, encoded_attr_name, value, size, 0, 0)
+    if result != -1:
+      return value.raw[:result]
+    if ctypes.get_errno() != errno.ERANGE:
+      _raise_os_error(path)
+
+
+def _setxattr(path: str, attr_name: str, attr_value: bytes) -> None:
+  if sys.platform != "darwin":
+    os.setxattr(path, attr_name, attr_value)
+    return
+
+  encoded_path = os.fsencode(path)
+  encoded_attr_name = os.fsencode(attr_name)
+  value = ctypes.create_string_buffer(attr_value)
+  if _libc.setxattr(encoded_path, encoded_attr_name, value, len(attr_value), 0, 0) == -1:
+    _raise_os_error(path)
+
+_cached_attributes: dict[tuple, bytes | None] = {}
+
+def getxattr(path: str, attr_name: str) -> bytes | None:
+  key = (path, attr_name)
+  if key not in _cached_attributes:
+    try:
+      response = _getxattr(path, attr_name)
+    except OSError as e:
+      # ENODATA (Linux) or ENOATTR (macOS) means attribute hasn't been set
+      if e.errno == errno.ENODATA or (hasattr(errno, 'ENOATTR') and e.errno == errno.ENOATTR):
+        response = None
+      else:
+        raise
+    _cached_attributes[key] = response
+  return _cached_attributes[key]
+
+def setxattr(path: str, attr_name: str, attr_value: bytes) -> None:
+  _cached_attributes.pop((path, attr_name), None)
+  _setxattr(path, attr_name, attr_value)
@@ -0,0 +1,65 @@
+
+#include "system/loggerd/zstd_writer.h"
+
+#include <cassert>
+
+#include "common/util.h"
+
+// Constructor: Initializes compression stream and opens file
+ZstdFileWriter::ZstdFileWriter(const std::string& filename, int compression_level) {
+  // Create the compression stream
+  cstream_ = ZSTD_createCStream();
+  assert(cstream_);
+
+  size_t initResult = ZSTD_initCStream(cstream_, compression_level);
+  assert(!ZSTD_isError(initResult));
+
+  input_cache_capacity_ = ZSTD_CStreamInSize();
+  input_cache_.reserve(input_cache_capacity_);
+  output_buffer_.resize(ZSTD_CStreamOutSize());
+
+  file_ = util::safe_fopen(filename.c_str(), "wb");
+  assert(file_ != nullptr);
+}
+
+// Destructor: Finalizes compression and closes file
+ZstdFileWriter::~ZstdFileWriter() {
+  flushCache(true);
+  util::safe_fflush(file_);
+
+  int err = fclose(file_);
+  assert(err == 0);
+
+  ZSTD_freeCStream(cstream_);
+}
+
+// Compresses and writes data to file
+void ZstdFileWriter::write(void* data, size_t size) {
+  // Add data to the input cache
+  input_cache_.insert(input_cache_.end(), (uint8_t*)data, (uint8_t*)data + size);
+
+  // If the cache is full, compress and write to the file
+  if (input_cache_.size() >= input_cache_capacity_) {
+    flushCache(false);
+  }
+}
+
+// Compress and flush the input cache to the file
+void ZstdFileWriter::flushCache(bool last_chunk) {
+  ZSTD_inBuffer input = {input_cache_.data(), input_cache_.size(), 0};
+  ZSTD_EndDirective mode = !last_chunk ? ZSTD_e_continue : ZSTD_e_end;
+  int finished = 0;
+
+  do {
+    ZSTD_outBuffer output = {output_buffer_.data(), output_buffer_.size(), 0};
+    size_t remaining = ZSTD_compressStream2(cstream_, &output, &input, mode);
+    assert(!ZSTD_isError(remaining));
+
+    size_t written = util::safe_fwrite(output_buffer_.data(), 1, output.pos, file_);
+    assert(written == output.pos);
+
+    finished = last_chunk ? (remaining == 0) : (input.pos == input.size);
+  } while (!finished);
+
+  input_cache_.clear();  // Clear cache after compression
+}
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <zstd.h>
+
+#include <string>
+#include <vector>
+#include <capnp/common.h>
+
+class ZstdFileWriter {
+public:
+  ZstdFileWriter(const std::string &filename, int compression_level);
+  ~ZstdFileWriter();
+  void write(void* data, size_t size);
+  inline void write(kj::ArrayPtr<capnp::byte> array) { write(array.begin(), array.size()); }
+
+private:
+  void flushCache(bool last_chunk);
+
+  size_t input_cache_capacity_ = 0;
+  std::vector<char> input_cache_;
+  std::vector<char> output_buffer_;
+  ZSTD_CStream *cstream_;
+  FILE* file_ = nullptr;
+};
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+import zmq
+from typing import NoReturn
+
+import openpilot.cereal.messaging as messaging
+from openpilot.common.logging_extra import SwagLogFileFormatter
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.swaglog import get_file_handler
+
+
+def main() -> NoReturn:
+  log_handler = get_file_handler()
+  log_handler.setFormatter(SwagLogFileFormatter(None))
+  log_level = 20  # logging.INFO
+
+  ctx = zmq.Context.instance()
+  sock = ctx.socket(zmq.PULL)
+  sock.bind(Paths.swaglog_ipc())
+
+  # and we publish them
+  log_message_sock = messaging.pub_sock('logMessage')
+  error_log_message_sock = messaging.pub_sock('errorLogMessage')
+
+  try:
+    while True:
+      dat = b''.join(sock.recv_multipart())
+      level = dat[0]
+      record = dat[1:].decode("utf-8")
+      if level >= log_level:
+        log_handler.emit(record)
+
+      if len(record) > 2*1024*1024:
+        print("WARNING: log too big to publish", len(record))
+        print(record[:100])
+        continue
+
+      # then we publish them
+      msg = messaging.new_message(None, valid=True, logMessage=record)
+      log_message_sock.send(msg.to_bytes())
+
+      if level >= 40:  # logging.ERROR
+        msg = messaging.new_message(None, valid=True, errorLogMessage=record)
+        error_log_message_sock.send(msg.to_bytes())
+  finally:
+    sock.close()
+    ctx.term()
+
+    # can hit this if interrupted during a rollover
+    try:
+      log_handler.close()
+    except ValueError:
+      pass
+
+if __name__ == "__main__":
+  main()
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+
+# NOTE: Do NOT import anything here that needs be built (e.g. params)
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.spinner import Spinner
+from openpilot.common.text_window import TextWindow
+from openpilot.common.hardware import HARDWARE, AGNOS
+
+def build() -> None:
+  spinner = Spinner()
+  spinner.update_progress(0, 100)
+
+  HARDWARE.set_power_save(False)
+  if AGNOS:
+    os.sched_setaffinity(0, range(8))  # ensure we can use the isolcpus cores
+
+  # building with all cores can result in using too much memory, so retry serially
+  compile_output: list[bytes] = []
+  for parallelism in ([], ["-j4"], ["-j1"]):
+    compile_output.clear()
+    with subprocess.Popen(["scons", *parallelism], cwd=BASEDIR, env={**os.environ, "PWD": BASEDIR}, stderr=subprocess.PIPE) as scons:
+      assert scons.stderr is not None
+
+      # Read progress from stderr and update spinner
+      while scons.poll() is None:
+        try:
+          line = scons.stderr.readline()
+          if line is None:
+            continue
+          line = line.rstrip()
+
+          prefix = b'progress: '
+          if line.startswith(prefix):
+            progress = float(line[len(prefix):])
+            spinner.update_progress(100 * min(1., progress / 100.), 100.)
+          elif len(line):
+            compile_output.append(line)
+            print(line.decode('utf8', 'replace'))
+        except Exception:
+          pass
+
+      # Drain and close the pipe before retrying or returning.
+      for line in scons.stderr.read().split(b'\n'):
+        line = line.rstrip()
+        if len(line):
+          compile_output.append(line)
+
+    if scons.returncode == 0:
+      break
+
+  os.sync()
+
+  if scons.returncode != 0:
+    # Build failed log errors
+    error_s = b"\n".join(compile_output).decode('utf8', 'replace')
+
+    # Show TextWindow
+    spinner.close()
+    if not os.getenv("CI"):
+      with TextWindow("openpilot failed to build\n \n" + error_s) as t:
+        t.wait_for_exit()
+    exit(1)
+
+if __name__ == "__main__":
+  build()
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Define the service name
+SERVICE_NAME="actions.runner.sunnypilot.$(uname -n)"
+
+# Function to control the service
+control_service() {
+    local action=$1  # Store the function argument in a local variable
+    sudo systemctl $action ${SERVICE_NAME}
+}
+
+service_exists_and_is_loaded() {
+    sudo systemctl status ${SERVICE_NAME} &>/dev/null
+    if [[ $? -ne 4 ]]; then
+        return 0  # Service is known to systemd (i.e., loaded)
+    else
+        return 1  # Service is unknown to systemd (i.e., not loaded)
+    fi
+}
+
+# Check for required argument
+if [[ -z $1 ]] || { [[ $1 != "start" ]] && [[ $1 != "stop" ]]; }; then
+    echo "Usage: $0 {start|stop}"
+    exit 1
+fi
+
+# Store the script argument in a descriptive variable
+ACTION=$1
+
+# Trap EXIT signal (Ctrl+C) and stop the service
+trap 'control_service stop ; exit' SIGINT SIGKILL EXIT
+
+# Enter the main loop
+while true; do
+    # Check if the service is actually present on the system
+    if service_exists_and_is_loaded; then
+        control_service $ACTION  # Call the function with the specified action
+    fi
+    sleep 1  # Pause before the next iteration
+done
@@ -0,0 +1,61 @@
+import errno
+import fcntl
+import os
+import sys
+import pathlib
+import shutil
+import signal
+import subprocess
+import tempfile
+import threading
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
+
+def unblock_stdout() -> None:
+  # get a non-blocking stdout
+  child_pid, child_pty = os.forkpty()
+  if child_pid != 0:  # parent
+
+    # child is in its own process group, manually pass kill signals
+    signal.signal(signal.SIGINT, lambda signum, frame: os.kill(child_pid, signal.SIGINT))
+    signal.signal(signal.SIGTERM, lambda signum, frame: os.kill(child_pid, signal.SIGTERM))
+
+    fcntl.fcntl(sys.stdout, fcntl.F_SETFL, fcntl.fcntl(sys.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+    while True:
+      try:
+        dat = os.read(child_pty, 4096)
+      except OSError as e:
+        if e.errno == errno.EIO:
+          break
+        continue
+
+      if not dat:
+        break
+
+      try:
+        sys.stdout.write(dat.decode('utf8'))
+      except (OSError, UnicodeDecodeError):
+        pass
+
+    # os.wait() returns a tuple with the pid and a 16 bit value
+    # whose low byte is the signal number and whose high byte is the exit status
+    exit_status = os.wait()[1] >> 8
+    os._exit(exit_status)
+
+def save_bootlog():
+  # copy current params
+  tmp = tempfile.mkdtemp()
+  params_dirname = pathlib.Path(Params().get_param_path()).name
+  params_dir = os.path.join(tmp, params_dirname)
+  shutil.copytree(Params().get_param_path(), params_dir, dirs_exist_ok=True)
+
+  def fn(tmpdir):
+    env = os.environ.copy()
+    env['PARAMS_COPY_PATH'] = tmpdir
+    subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "openpilot/system/loggerd"), env=env)
+    shutil.rmtree(tmpdir)
+  t = threading.Thread(target=fn, args=(tmp, ))
+  t.daemon = True
+  t.start()
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+import datetime
+import os
+import signal
+import sys
+import time
+import traceback
+
+from openpilot.cereal import log
+import openpilot.cereal.messaging as messaging
+import openpilot.system.sentry as sentry
+from openpilot.common.utils import atomic_write
+from openpilot.common.params import Params, ParamKeyFlag
+from openpilot.common.text_window import TextWindow
+from openpilot.common.hardware import HARDWARE, PC
+from openpilot.system.manager.helpers import unblock_stdout, save_bootlog
+from openpilot.system.manager.process import ensure_running
+from openpilot.system.manager.process_config import managed_processes
+from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID
+from openpilot.common.swaglog import cloudlog, add_file_handler
+from openpilot.common.version import get_build_metadata
+from openpilot.common.hardware.hw import Paths
+
+from openpilot.sunnypilot.system.params_migration import run_migration
+
+
+def manager_init() -> None:
+  save_bootlog()
+
+  build_metadata = get_build_metadata()
+
+  params = Params()
+  params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
+  params.clear_all(ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION)
+  params.clear_all(ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION)
+  params.clear_all(ParamKeyFlag.CLEAR_ON_IGNITION_ON)
+  # if build_metadata.release_channel:
+  #   params.clear_all(ParamKeyFlag.DEVELOPMENT_ONLY)
+
+  # device boot mode
+  if params.get("DeviceBootMode") == 1:  # start in Always Offroad mode
+    params.put_bool("OffroadMode", True, block=True)
+
+  # quick boot
+  if params.get_bool("QuickBootToggle") and not PC:
+    prebuilt_path = "/data/openpilot/prebuilt"
+    if not os.path.exists(prebuilt_path):
+      open(prebuilt_path, 'x').close()
+
+  if params.get_bool("RecordFrontLock"):
+    params.put_bool("RecordFront", True, block=True)
+
+  if not PC:
+    run_migration(params)
+
+  # set unset params to their default value
+  for k in params.all_keys():
+    default_value = params.get_default_value(k)
+    if default_value is not None and params.get(k) is None:
+      params.put(k, default_value, block=True)
+
+  # Create folders needed for msgq
+  try:
+    os.mkdir(Paths.shm_path())
+  except FileExistsError:
+    pass
+  except PermissionError:
+    print(f"WARNING: failed to make {Paths.shm_path()}")
+
+  # set params
+  serial = HARDWARE.get_serial()
+  params.put("Version", build_metadata.openpilot.version, block=True)
+  params.put("GitCommit", build_metadata.openpilot.git_commit, block=True)
+  params.put("GitCommitDate", build_metadata.openpilot.git_commit_date, block=True)
+  params.put("GitBranch", build_metadata.channel, block=True)
+  params.put("GitRemote", build_metadata.openpilot.git_origin, block=True)
+  params.put_bool("IsDevelopmentBranch", build_metadata.development_channel, block=True)
+  params.put_bool("IsTestedBranch", build_metadata.tested_channel, block=True)
+  params.put_bool("IsReleaseBranch", build_metadata.release_channel, block=True)
+  params.put_bool("IsReleaseSpBranch", build_metadata.release_sp_channel, block=True)
+  params.put("HardwareSerial", serial, block=True)
+
+  # set dongle id
+  reg_res = register(show_spinner=True)
+  if reg_res:
+    dongle_id = reg_res
+  else:
+    raise Exception(f"Registration failed for device {serial}")
+  os.environ['DONGLE_ID'] = dongle_id  # Needed for swaglog
+  os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog
+  os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog
+  os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog
+
+  if not build_metadata.openpilot.is_dirty:
+    os.environ['CLEAN'] = '1'
+
+  # init logging
+  sentry.init(sentry.SentryProject.SELFDRIVE)
+  cloudlog.bind_global(dongle_id=dongle_id,
+                       version=build_metadata.openpilot.version,
+                       origin=build_metadata.openpilot.git_normalized_origin,
+                       branch=build_metadata.channel,
+                       commit=build_metadata.openpilot.git_commit,
+                       dirty=build_metadata.openpilot.is_dirty,
+                       device=HARDWARE.get_device_type())
+
+def manager_cleanup() -> None:
+  # send signals to kill all procs
+  for p in managed_processes.values():
+    p.stop(block=False)
+
+  # ensure all are killed
+  for p in managed_processes.values():
+    p.stop(block=True)
+
+  cloudlog.info("everything is dead")
+
+
+def manager_thread() -> None:
+  cloudlog.bind(daemon="manager")
+  cloudlog.info("manager start")
+  cloudlog.info({"environ": os.environ})
+
+  params = Params()
+
+  ignore: list[str] = []
+  if params.get("DongleId") in (None, UNREGISTERED_DONGLE_ID):
+    ignore += ["manage_athenad", "uploader"]
+  if os.getenv("NOBOARD") is not None:
+    ignore.append("pandad")
+  ignore += [x for x in os.getenv("BLOCK", "").split(",") if len(x) > 0]
+
+  sm = messaging.SubMaster(['deviceState', 'carParams', 'pandaStates'], poll='deviceState')
+  pm = messaging.PubMaster(['managerState'])
+
+  params.put_bool("IsOffroad", True, block=True)
+  ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore)
+
+  started_prev = False
+  ignition_prev = False
+
+  while True:
+    sm.update(1000)
+
+    started = sm['deviceState'].started
+
+    if started and not started_prev:
+      params.clear_all(ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION)
+    elif not started and started_prev:
+      params.clear_all(ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION)
+
+    ignition = any(ps.ignitionLine or ps.ignitionCan for ps in sm['pandaStates'] if ps.pandaType != log.PandaState.PandaType.unknown)
+    if ignition and not ignition_prev:
+      params.clear_all(ParamKeyFlag.CLEAR_ON_IGNITION_ON)
+
+    # update offroad state for services that don't subscribe to deviceState
+    if started != started_prev:
+      params.put_bool("IsOffroad", not started, block=True)
+
+    started_prev = started
+    ignition_prev = ignition
+
+    ensure_running(managed_processes.values(), started, params=params, CP=sm['carParams'], not_run=ignore)
+
+    running = ' '.join("{}{}\u001b[0m".format("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name)
+                       for p in managed_processes.values() if p.proc)
+    print(running)
+    cloudlog.debug(running)
+
+    # send managerState
+    msg = messaging.new_message('managerState', valid=True)
+    msg.managerState.processes = [p.get_process_state_msg() for p in managed_processes.values()]
+    pm.send('managerState', msg)
+
+    # kick AGNOS power monitoring watchdog
+    try:
+      if sm.all_checks(['deviceState']):
+        with atomic_write("/var/tmp/power_watchdog", "w", overwrite=True) as f:
+          f.write(str(time.monotonic()))
+    except Exception:
+      pass
+
+    # Exit main loop when uninstall/shutdown/reboot is needed
+    shutdown = False
+    for param in ("DoUninstall", "DoShutdown", "DoReboot"):
+      if params.get_bool(param):
+        shutdown = True
+        params.put("LastManagerExitReason", f"{param} {datetime.datetime.now()}", block=True)
+        cloudlog.warning(f"Shutting down manager - {param} set")
+
+    if shutdown:
+      break
+
+
+def main() -> None:
+  manager_init()
+  if os.getenv("PREPAREONLY") is not None:
+    return
+
+  # SystemExit on sigterm
+  signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1))
+
+  try:
+    manager_thread()
+  except Exception:
+    traceback.print_exc()
+    sentry.capture_exception()
+  finally:
+    manager_cleanup()
+
+  params = Params()
+  if params.get_bool("DoUninstall"):
+    cloudlog.warning("uninstalling")
+    HARDWARE.uninstall()
+  elif params.get_bool("DoReboot"):
+    cloudlog.warning("reboot")
+    HARDWARE.reboot()
+  elif params.get_bool("DoShutdown"):
+    cloudlog.warning("shutdown")
+    HARDWARE.shutdown()
+
+
+if __name__ == "__main__":
+  unblock_stdout()
+
+  try:
+    main()
+  except KeyboardInterrupt:
+    print("got CTRL-C, exiting")
+  except Exception:
+    add_file_handler(cloudlog)
+    cloudlog.exception("Manager failed to start")
+
+    try:
+      managed_processes['ui'].stop()
+    except Exception:
+      pass
+
+    # Show last 3 lines of traceback
+    error = traceback.format_exc(-3)
+    error = "Manager failed to start\n\n" + error
+    with TextWindow(error) as t:
+      t.wait_for_exit()
+
+    raise
+
+  # manual exit because we are forked
+  sys.exit(0)
@@ -0,0 +1,240 @@
+import importlib
+import os
+import signal
+import time
+import subprocess
+from collections.abc import Callable, ValuesView
+from abc import ABC, abstractmethod
+from multiprocessing import Process
+
+from setproctitle import setproctitle
+
+from openpilot.cereal import log
+from opendbc.car.structs import car
+import openpilot.cereal.messaging as messaging
+import openpilot.system.sentry as sentry
+from openpilot.common.basedir import BASEDIR
+from openpilot.common.params import Params
+from openpilot.common.swaglog import cloudlog
+
+
+def launcher(proc: str, name: str) -> None:
+  try:
+    # import the process
+    mod = importlib.import_module(proc)
+
+    # rename the process
+    setproctitle(proc)
+
+    # create new context since we forked
+    messaging.reset_context()
+
+    # add daemon name tag to logs
+    cloudlog.bind(daemon=name)
+    sentry.set_tag("daemon", name)
+
+    # exec the process
+    mod.main()
+  except KeyboardInterrupt:
+    cloudlog.warning(f"child {proc} got SIGINT")
+  except Exception:
+    # can't install the crash handler because sys.excepthook doesn't play nice
+    # with threads, so catch it here.
+    sentry.capture_exception()
+    raise
+
+
+def nativelauncher(pargs: list[str], cwd: str, name: str) -> None:
+  os.environ['MANAGER_DAEMON'] = name
+
+  # exec the process
+  os.chdir(cwd)
+  os.execvp(pargs[0], pargs)
+
+
+def join_process(process: Process, timeout: float) -> None:
+  # Process().join(timeout) will hang due to a python 3 bug: https://bugs.python.org/issue28382
+  # We have to poll the exitcode instead
+  t = time.monotonic()
+  while time.monotonic() - t < timeout and process.exitcode is None:
+    time.sleep(0.001)
+
+
+class ManagerProcess(ABC):
+  daemon = False
+  sigkill = False
+  should_run: Callable[[bool, Params, car.CarParams], bool]
+  proc: Process | None = None
+  enabled = True
+  name = ""
+  shutting_down = False
+
+  @abstractmethod
+  def start(self) -> None:
+    pass
+
+  def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals | None = None) -> int | None:
+    if self.proc is None:
+      return None
+
+    if self.proc.exitcode is None:
+      if not self.shutting_down:
+        cloudlog.info(f"killing {self.name}")
+        if sig is None:
+          sig = signal.SIGKILL if self.sigkill else signal.SIGINT
+        self.signal(sig)
+        self.shutting_down = True
+
+        if not block:
+          return None
+
+      join_process(self.proc, 5)
+
+      # If process failed to die send SIGKILL
+      if self.proc.exitcode is None and retry:
+        cloudlog.info(f"killing {self.name} with SIGKILL")
+        self.signal(signal.SIGKILL)
+        self.proc.join()
+
+    ret = self.proc.exitcode
+    cloudlog.info(f"{self.name} is dead with {ret}")
+
+    if self.proc.exitcode is not None:
+      self.shutting_down = False
+      self.proc = None
+
+    return ret
+
+  def signal(self, sig: int) -> None:
+    if self.proc is None:
+      return
+
+    # Don't signal if already exited
+    if self.proc.exitcode is not None and self.proc.pid is not None:
+      return
+
+    # Can't signal if we don't have a pid
+    if self.proc.pid is None:
+      return
+
+    cloudlog.info(f"sending signal {sig} to {self.name}")
+    os.kill(self.proc.pid, sig)
+
+  def get_process_state_msg(self):
+    state = log.ManagerState.ProcessState.new_message()
+    state.name = self.name
+    if self.proc:
+      state.running = self.proc.is_alive()
+      state.shouldBeRunning = self.proc is not None and not self.shutting_down
+      state.pid = self.proc.pid or 0
+      state.exitCode = self.proc.exitcode or 0
+    return state
+
+
+class NativeProcess(ManagerProcess):
+  def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False):
+    self.name = name
+    self.cwd = cwd
+    self.cmdline = cmdline
+    self.should_run = should_run
+    self.enabled = enabled
+    self.sigkill = sigkill
+    self.launcher = nativelauncher
+
+  def start(self) -> None:
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+      self.stop()
+
+    if self.proc is not None:
+      return
+
+    cwd = os.path.join(BASEDIR, self.cwd)
+    cloudlog.info(f"starting process {self.name}")
+    self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name))
+    self.proc.start()
+    self.shutting_down = False
+
+
+class PythonProcess(ManagerProcess):
+  def __init__(self, name, module, should_run, enabled=True, sigkill=False):
+    self.name = name
+    self.module = module
+    self.should_run = should_run
+    self.enabled = enabled
+    self.sigkill = sigkill
+    self.launcher = launcher
+
+  def start(self) -> None:
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+      self.stop()
+
+    if self.proc is not None:
+      return
+
+    cloudlog.info(f"starting python {self.module}")
+    self.proc = Process(name=self.name, target=self.launcher, args=(self.module, self.name))
+    self.proc.start()
+    self.shutting_down = False
+
+
+class DaemonProcess(ManagerProcess):
+  """Python process that has to stay running across manager restart.
+  This is used for athena so you don't lose SSH access when restarting manager."""
+  def __init__(self, name, module, param_name, enabled=True):
+    self.name = name
+    self.module = module
+    self.param_name = param_name
+    self.enabled = enabled
+    self.params = None
+
+  @staticmethod
+  def should_run(started, params, CP):
+    return True
+
+  def start(self) -> None:
+    if self.params is None:
+      self.params = Params()
+
+    pid = self.params.get(self.param_name)
+    if pid is not None:
+      try:
+        os.kill(int(pid), 0)
+        with open(f'/proc/{pid}/cmdline') as f:
+          if self.module in f.read():
+            # daemon is running
+            return
+      except (OSError, FileNotFoundError):
+        # process is dead
+        pass
+
+    cloudlog.info(f"starting daemon {self.name}")
+    proc = subprocess.Popen(['python', '-m', self.module],
+                               stdin=open('/dev/null'),
+                               stdout=open('/dev/null', 'w'),
+                               stderr=open('/dev/null', 'w'),
+                               preexec_fn=os.setpgrp)
+
+    self.params.put(self.param_name, proc.pid, block=True)
+
+  def stop(self, retry=True, block=True, sig=None) -> None:
+    pass
+
+
+def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params: Params, CP: car.CarParams,
+                   not_run: list[str] | None=None) -> list[ManagerProcess]:
+  if not_run is None:
+    not_run = []
+
+  running = []
+  for p in procs:
+    if p.enabled and p.name not in not_run and p.should_run(started, params, CP):
+      running.append(p)
+    else:
+      p.stop(block=False)
+
+  for p in running:
+    p.start()
+
+  return running
@@ -0,0 +1,210 @@
+import os
+import operator
+import platform
+
+from opendbc.car.structs import car
+from openpilot.cereal import custom
+from openpilot.common.params import Params
+from openpilot.common.hardware import PC, COMMA_HARDWARE
+from openpilot.system.manager.process import PythonProcess, NativeProcess, DaemonProcess
+from openpilot.common.hardware.hw import Paths
+
+from openpilot.sunnypilot.mapd.mapd_manager import MAPD_PATH
+
+from openpilot.sunnypilot.models.helpers import get_active_model_runner
+from openpilot.sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready, use_sunnylink_uploader
+
+WEBCAM = os.getenv("USE_WEBCAM") is not None
+
+def driverview(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started or params.get_bool("IsDriverViewEnabled")
+
+def notcar(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and CP.notCar
+
+def iscar(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not CP.notCar
+
+def logging(started: bool, params: Params, CP: car.CarParams) -> bool:
+  run = (not CP.notCar) or not params.get_bool("DisableLogging")
+  return started and run
+
+def ublox_available() -> bool:
+  return os.path.exists('/dev/ttyHS0') and not os.path.exists('/persist/comma/use-quectel-gps')
+
+def ublox(started: bool, params: Params, CP: car.CarParams) -> bool:
+  use_ublox = ublox_available()
+  if use_ublox != params.get_bool("UbloxAvailable"):
+    params.put_bool("UbloxAvailable", use_ublox, block=True)
+  return started and use_ublox
+
+def joystick(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("JoystickDebugMode")
+
+def not_joystick(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not params.get_bool("JoystickDebugMode")
+
+def long_maneuver(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("LongitudinalManeuverMode")
+
+def lat_maneuver(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("LateralManeuverMode")
+
+def not_long_maneuver(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not params.get_bool("LongitudinalManeuverMode")
+
+def qcomgps(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not ublox_available()
+
+def always_run(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return True
+
+def only_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started
+
+def only_offroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return not started
+
+def livestream(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return params.get_bool("IsLiveStreaming")
+
+def use_github_runner(started, params, CP: car.CarParams) -> bool:
+  return not PC and params.get_bool("EnableGithubRunner") and (
+    not params.get_bool("NetworkMetered") and not params.get_bool("GithubRunnerSufficientVoltage"))
+
+def use_copyparty(started, params, CP: car.CarParams) -> bool:
+  return bool(params.get_bool("EnableCopyparty"))
+
+def sunnylink_ready_shim(started, params, CP: car.CarParams) -> bool:
+  """Shim for sunnylink_ready to match the process manager signature."""
+  return sunnylink_ready(params)
+
+def sunnylink_need_register_shim(started, params, CP: car.CarParams) -> bool:
+  """Shim for sunnylink_need_register to match the process manager signature."""
+  return sunnylink_need_register(params)
+
+def use_sunnylink_uploader_shim(started, params, CP: car.CarParams) -> bool:
+  """Shim for use_sunnylink_uploader to match the process manager signature."""
+  return use_sunnylink_uploader(params)
+
+def is_tinygrad_model(started, params, CP: car.CarParams) -> bool:
+  """Check if the active model runner is tinygrad."""
+  return bool(get_active_model_runner(params, not started) == custom.ModelManagerSP.Runner.tinygrad)
+
+def is_stock_model(started, params, CP: car.CarParams) -> bool:
+  """Check if the active model runner is stock."""
+  return bool(get_active_model_runner(params, not started) == custom.ModelManagerSP.Runner.stock)
+
+def mapd_ready(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return bool(os.path.exists(Paths.mapd_root()))
+
+def uploader_ready(started: bool, params: Params, CP: car.CarParams) -> bool:
+  if not params.get_bool("OnroadUploads"):
+    return only_offroad(started, params, CP)
+
+  return always_run(started, params, CP)
+
+def or_(*fns):
+  return lambda *args: operator.or_(*(fn(*args) for fn in fns))
+
+def and_(*fns):
+  return lambda *args: operator.and_(*(fn(*args) for fn in fns))
+
+def not_(*fns):
+  return lambda *args: operator.not_(*(fn(*args) for fn in fns))
+
+procs = [
+  DaemonProcess("manage_athenad", "openpilot.system.athena.manage_athenad", "AthenadPid"),
+
+  NativeProcess("loggerd", "openpilot/system/loggerd", ["./loggerd"], logging),
+  NativeProcess("encoderd", "openpilot/system/loggerd", ["./encoderd"], only_onroad),
+  NativeProcess("stream_encoderd", "openpilot/system/loggerd", ["./encoderd", "--stream"], or_(and_(livestream, not_(iscar)), notcar)),
+  PythonProcess("logmessaged", "openpilot.system.logmessaged", always_run),
+
+  NativeProcess("camerad", "openpilot/system/camerad", ["./camerad"], or_(driverview, livestream), enabled=not WEBCAM),
+  PythonProcess("webcamerad", "openpilot.system.camerad.webcam.camerad", driverview, enabled=WEBCAM),
+  PythonProcess("proclogd", "openpilot.system.proclogd", only_onroad, enabled=platform.system() != "Darwin"),
+  PythonProcess("journald", "openpilot.system.journald", only_onroad, platform.system() != "Darwin"),
+  PythonProcess("micd", "openpilot.system.micd", iscar),
+  PythonProcess("timed", "openpilot.system.timed", always_run, enabled=not PC),
+
+  PythonProcess("modeld", "openpilot.selfdrive.modeld.modeld", and_(only_onroad, is_stock_model)),
+  PythonProcess("dmonitoringmodeld", "openpilot.selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(WEBCAM or not PC)),
+
+  PythonProcess("sensord", "openpilot.system.sensord.sensord", only_onroad, enabled=not PC),
+  PythonProcess("ui", "openpilot.selfdrive.ui.ui", always_run),
+  PythonProcess("soundd", "openpilot.selfdrive.ui.soundd", driverview),
+  PythonProcess("locationd", "openpilot.selfdrive.locationd.locationd", only_onroad),
+  NativeProcess("_pandad", "openpilot/selfdrive/pandad", ["./pandad"], always_run, enabled=False),
+  PythonProcess("calibrationd", "openpilot.selfdrive.locationd.calibrationd", only_onroad),
+  PythonProcess("torqued", "openpilot.selfdrive.locationd.torqued", only_onroad),
+  PythonProcess("controlsd", "openpilot.selfdrive.controls.controlsd", and_(not_joystick, iscar)),
+  PythonProcess("joystickd", "openpilot.tools.joystick.joystickd", or_(joystick, notcar)),
+  PythonProcess("selfdrived", "openpilot.selfdrive.selfdrived.selfdrived", only_onroad),
+  PythonProcess("card", "openpilot.selfdrive.car.card", only_onroad),
+  PythonProcess("deleter", "openpilot.system.loggerd.deleter", always_run),
+  PythonProcess("dmonitoringd", "openpilot.selfdrive.monitoring.dmonitoringd", driverview, enabled=(WEBCAM or not PC)),
+  PythonProcess("qcomgpsd", "openpilot.system.qcomgpsd.qcomgpsd", qcomgps, enabled=COMMA_HARDWARE),
+  PythonProcess("pandad", "openpilot.selfdrive.pandad.pandad", always_run),
+  PythonProcess("paramsd", "openpilot.selfdrive.locationd.paramsd", only_onroad),
+  PythonProcess("lagd", "openpilot.selfdrive.locationd.lagd", only_onroad),
+  PythonProcess("ubloxd", "openpilot.system.ubloxd.ubloxd", ublox, enabled=COMMA_HARDWARE),
+  PythonProcess("pigeond", "openpilot.system.ubloxd.pigeond", ublox, enabled=COMMA_HARDWARE),
+  PythonProcess("plannerd", "openpilot.selfdrive.controls.plannerd", not_long_maneuver),
+  PythonProcess("maneuversd", "openpilot.tools.longitudinal_maneuvers.maneuversd", long_maneuver),
+  PythonProcess("lateral_maneuversd", "openpilot.tools.lateral_maneuvers.lateral_maneuversd", lat_maneuver),
+  PythonProcess("radard", "openpilot.selfdrive.controls.radard", only_onroad),
+  PythonProcess("hardwared", "openpilot.system.hardware.hardwared", always_run),
+  PythonProcess("modem", "openpilot.common.hardware.comma.modem", always_run, enabled=COMMA_HARDWARE),
+  PythonProcess("tombstoned", "openpilot.system.tombstoned", always_run, enabled=not PC),
+  PythonProcess("updated", "openpilot.system.updated.updated", only_offroad, enabled=not PC),
+  PythonProcess("uploader", "openpilot.system.loggerd.uploader", uploader_ready),
+  PythonProcess("statsd", "openpilot.sunnypilot.system.statsd", always_run),
+
+  # debug procs
+  NativeProcess("bridge", "openpilot/cereal/messaging", ["./bridge"], notcar),
+  PythonProcess("webrtcd", "openpilot.system.webrtc.webrtcd", or_(and_(livestream, not_(iscar)), notcar)),
+  PythonProcess("joystick", "openpilot.tools.joystick.joystick_control", and_(joystick, iscar)),
+
+  # sunnylink <3
+  DaemonProcess("manage_sunnylinkd", "openpilot.sunnypilot.sunnylink.athena.manage_sunnylinkd", "SunnylinkdPid"),
+  PythonProcess("sunnylink_registration_manager", "openpilot.sunnypilot.sunnylink.registration_manager", sunnylink_need_register_shim),
+  PythonProcess("statsd_sp", "openpilot.sunnypilot.sunnylink.statsd", and_(always_run, sunnylink_ready_shim)),
+]
+
+# sunnypilot
+procs += [
+  # Models
+  PythonProcess("models_manager", "openpilot.sunnypilot.models.manager", only_offroad),
+  NativeProcess("modeld_tinygrad", "openpilot/sunnypilot/modeld_v2", ["./modeld"], and_(only_onroad, is_tinygrad_model)),
+
+  # Backup
+  PythonProcess("backup_manager", "openpilot.sunnypilot.sunnylink.backups.manager", and_(only_offroad, sunnylink_ready_shim)),
+
+  # mapd
+  NativeProcess("mapd", Paths.mapd_root(), ["bash", "-c", f"{MAPD_PATH} > /dev/null 2>&1"], mapd_ready),
+  PythonProcess("mapd_manager", "openpilot.sunnypilot.mapd.mapd_manager", always_run),
+
+  # locationd
+  NativeProcess("locationd_llk", "openpilot/sunnypilot/selfdrive/locationd", ["./locationd"], only_onroad),
+]
+
+if os.path.exists("./github_runner.sh"):
+  procs += [NativeProcess("github_runner_start", "openpilot/system/manager",
+                          ["./github_runner.sh", "start"], and_(only_offroad, use_github_runner), sigkill=False)]
+
+if os.path.exists("../../sunnypilot/sunnylink/uploader.py"):
+  procs += [PythonProcess("sunnylink_uploader", "openpilot.sunnypilot.sunnylink.uploader", use_sunnylink_uploader_shim)]
+
+if os.path.exists("../../third_party/copyparty/copyparty-sfx.py"):
+  sunnypilot_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+  copyparty_args = [f"-v{Paths.crash_log_root()}:/swaglogs:r"]
+  copyparty_args += [f"-v{Paths.log_root()}:/routes:r"]
+  copyparty_args += [f"-v{Paths.model_root()}:/models:rw"]
+  copyparty_args += [f"-v{sunnypilot_root}:/sunnypilot:rw"]
+  copyparty_args += ["-p8080"]
+  copyparty_args += ["-z"]
+  copyparty_args += ["-q"]
+  procs += [NativeProcess("copyparty-sfx", "openpilot/third_party/copyparty", ["./copyparty-sfx.py", *copyparty_args], and_(only_offroad, use_copyparty))]
+
+managed_processes = {p.name: p for p in procs}
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+import signal
+import time
+
+from opendbc.car.structs import car
+from openpilot.common.test import OpenpilotTestCase
+from openpilot.common.params import Params
+import openpilot.system.manager.manager as manager
+from openpilot.system.manager.process import ensure_running
+from openpilot.system.manager.process_config import managed_processes, procs
+from openpilot.common.hardware import HARDWARE
+
+os.environ['FAKEUPLOAD'] = "1"
+
+MAX_STARTUP_TIME = 3
+BLACKLIST_PROCS = ['manage_athenad', 'pandad', 'pigeond']
+
+
+class TestManager(OpenpilotTestCase):
+  def setup_method(self):
+    HARDWARE.set_power_save(False)
+
+    # ensure clean CarParams
+    params = Params()
+    params.clear_all()
+
+  def teardown_method(self):
+    manager.manager_cleanup()
+
+  def test_duplicate_procs(self):
+    assert len(procs) == len(managed_processes), "Duplicate process names"
+
+  def test_blacklisted_procs(self):
+    # TODO: ensure there are blacklisted procs until we have a dedicated test
+    assert len(BLACKLIST_PROCS), "No blacklisted procs to test not_run"
+
+  def test_set_params_with_default_value(self):
+    params = Params()
+    params.clear_all()
+
+    os.environ['PREPAREONLY'] = '1'
+    manager.main()
+    for k in params.all_keys():
+      default_value = params.get_default_value(k)
+      if default_value is not None:
+        assert params.get(k) == default_value
+    assert params.get("OpenpilotEnabledToggle")
+    assert params.get("RouteCount") == 0
+
+  @unittest.skip("this test is flaky the way it's currently written, should be moved to test_onroad")
+  def test_clean_exit(self, subtests):
+    """
+      Ensure all processes exit cleanly when stopped.
+    """
+    HARDWARE.set_power_save(False)
+    manager.manager_init()
+
+    CP = car.CarParams.new_message()
+    procs = ensure_running(managed_processes.values(), True, Params(), CP, not_run=BLACKLIST_PROCS)
+
+    time.sleep(10)
+
+    for p in procs:
+      with subtests.test(proc=p.name):
+        state = p.get_process_state_msg()
+        assert state.running, f"{p.name} not running"
+        exit_code = p.stop(retry=False)
+
+        assert p.name not in BLACKLIST_PROCS, f"{p.name} was started"
+
+        assert exit_code is not None, f"{p.name} failed to exit"
+
+        # TODO: interrupted blocking read exits with 1 in cereal. use a more unique return code
+        exit_codes = [0, 1]
+        if p.sigkill:
+          exit_codes = [-signal.SIGKILL]
+        assert exit_code in exit_codes, f"{p.name} died with {exit_code}"
+
+
+if __name__ == "__main__":
+  unittest.main()
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+import numpy as np
+from functools import cache
+import threading
+
+from openpilot.cereal import messaging
+from openpilot.common.realtime import Ratekeeper
+from openpilot.common.utils import retry
+from openpilot.common.swaglog import cloudlog
+
+RATE = 10
+FFT_SAMPLES = 1600 # 100ms
+REFERENCE_SPL = 2e-5  # newtons/m^2
+SAMPLE_RATE = 16000
+SAMPLE_BUFFER = 800  # 50ms
+
+
+def patch_sounddevice(sd):
+  # TODO: remove once sounddevice uses np.reshape internally.
+  def sounddevice_array(buffer, channels, dtype):
+    return np.frombuffer(buffer, dtype=dtype).reshape(-1, channels)
+
+  sd._array = sounddevice_array
+
+
+@cache
+def get_a_weighting_filter():
+  # Calculate the A-weighting filter
+  # https://en.wikipedia.org/wiki/A-weighting
+  freqs = np.fft.fftfreq(FFT_SAMPLES, d=1 / SAMPLE_RATE)
+  A = 12194 ** 2 * freqs ** 4 / ((freqs ** 2 + 20.6 ** 2) * (freqs ** 2 + 12194 ** 2) * np.sqrt((freqs ** 2 + 107.7 ** 2) * (freqs ** 2 + 737.9 ** 2)))
+  return A / np.max(A)
+
+
+def calculate_spl(measurements):
+  # https://www.engineeringtoolbox.com/sound-pressure-d_711.html
+  sound_pressure = np.sqrt(np.mean(measurements ** 2))  # RMS of amplitudes
+  if sound_pressure > 0:
+    sound_pressure_level = 20 * np.log10(sound_pressure / REFERENCE_SPL)  # dB
+  else:
+    sound_pressure_level = 0
+  return sound_pressure, sound_pressure_level
+
+
+def apply_a_weighting(measurements: np.ndarray) -> np.ndarray:
+  # Generate a Hanning window of the same length as the audio measurements
+  measurements_windowed = measurements * np.hanning(len(measurements))
+
+  # Apply the A-weighting filter to the signal
+  return np.abs(np.fft.ifft(np.fft.fft(measurements_windowed) * get_a_weighting_filter()))
+
+
+class Mic:
+  def __init__(self):
+    self.rk = Ratekeeper(RATE)
+    self.pm = messaging.PubMaster(['soundPressure', 'rawAudioData'])
+
+    self.measurements = np.empty(0)
+
+    self.sound_pressure = 0
+    self.sound_pressure_weighted = 0
+    self.sound_pressure_level_weighted = 0
+
+    self.lock = threading.Lock()
+
+  def update(self):
+    with self.lock:
+      sound_pressure = self.sound_pressure
+      sound_pressure_weighted = self.sound_pressure_weighted
+      sound_pressure_level_weighted = self.sound_pressure_level_weighted
+
+    msg = messaging.new_message('soundPressure', valid=True)
+    msg.soundPressure.soundPressure = float(sound_pressure)
+    msg.soundPressure.soundPressureWeighted = float(sound_pressure_weighted)
+    msg.soundPressure.soundPressureWeightedDb = float(sound_pressure_level_weighted)
+
+    self.pm.send('soundPressure', msg)
+    self.rk.keep_time()
+
+  def callback(self, indata, frames, time, status):
+    """
+    Using amplitude measurements, calculate an uncalibrated sound pressure and sound pressure level.
+    Then apply A-weighting to the raw amplitudes and run the same calculations again.
+
+    Logged A-weighted equivalents are rough approximations of the human-perceived loudness.
+    """
+    msg = messaging.new_message('rawAudioData', valid=True)
+    audio_data_int_16 = (indata[:, 0] * 32767).astype(np.int16)
+    msg.rawAudioData.data = audio_data_int_16.tobytes()
+    msg.rawAudioData.sampleRate = SAMPLE_RATE
+    self.pm.send('rawAudioData', msg)
+
+    with self.lock:
+      self.measurements = np.concatenate((self.measurements, indata[:, 0]))
+
+      while self.measurements.size >= FFT_SAMPLES:
+        measurements = self.measurements[:FFT_SAMPLES]
+
+        self.sound_pressure, _ = calculate_spl(measurements)
+        measurements_weighted = apply_a_weighting(measurements)
+        self.sound_pressure_weighted, self.sound_pressure_level_weighted = calculate_spl(measurements_weighted)
+
+        self.measurements = self.measurements[FFT_SAMPLES:]
+
+  @retry(attempts=10, delay=3)
+  def get_stream(self, sd):
+    # reload sounddevice to reinitialize portaudio
+    sd._terminate()
+    sd._initialize()
+    return sd.InputStream(channels=1, samplerate=SAMPLE_RATE, callback=self.callback, blocksize=SAMPLE_BUFFER)
+
+  def micd_thread(self):
+    # sounddevice must be imported after forking processes
+    import sounddevice as sd
+    patch_sounddevice(sd)
+
+    with self.get_stream(sd) as stream:
+      cloudlog.info(f"micd stream started: {stream.samplerate=} {stream.channels=} {stream.dtype=} {stream.device=}, {stream.blocksize=}")
+      while True:
+        self.update()
+
+
+def main():
+  mic = Mic()
+  mic.micd_thread()
+
+
+if __name__ == "__main__":
+  main()
@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+import os
+from typing import NoReturn, TypedDict
+
+from openpilot.cereal import messaging
+from openpilot.common.realtime import Ratekeeper
+from openpilot.common.swaglog import cloudlog
+
+JIFFY = os.sysconf(os.sysconf_names['SC_CLK_TCK'])
+PAGE_SIZE = os.sysconf(os.sysconf_names['SC_PAGE_SIZE'])
+
+
+def _cpu_times() -> list[dict[str, float]]:
+  cpu_times: list[dict[str, float]] = []
+  try:
+    with open('/proc/stat') as f:
+      lines = f.readlines()[1:]
+    for line in lines:
+      if not line.startswith('cpu') or len(line) < 4 or not line[3].isdigit():
+        break
+      parts = line.split()
+      cpu_times.append({
+        'cpuNum': int(parts[0][3:]),
+        'user': float(parts[1]) / JIFFY,
+        'nice': float(parts[2]) / JIFFY,
+        'system': float(parts[3]) / JIFFY,
+        'idle': float(parts[4]) / JIFFY,
+        'iowait': float(parts[5]) / JIFFY,
+        'irq': float(parts[6]) / JIFFY,
+        'softirq': float(parts[7]) / JIFFY,
+      })
+  except Exception:
+    cloudlog.exception("failed to read /proc/stat")
+  return cpu_times
+
+
+def _mem_info() -> dict[str, int]:
+  keys = ["MemTotal:", "MemFree:", "MemAvailable:", "Buffers:", "Cached:", "Active:", "Inactive:", "Shmem:"]
+  info: dict[str, int] = dict.fromkeys(keys, 0)
+  try:
+    with open('/proc/meminfo') as f:
+      for line in f:
+        parts = line.split()
+        if parts and parts[0] in info:
+          info[parts[0]] = int(parts[1]) * 1024
+  except Exception:
+    cloudlog.exception("failed to read /proc/meminfo")
+  return info
+
+
+_STAT_POS = {
+  'pid': 1,
+  'state': 3,
+  'ppid': 4,
+  'utime': 14,
+  'stime': 15,
+  'cutime': 16,
+  'cstime': 17,
+  'priority': 18,
+  'nice': 19,
+  'num_threads': 20,
+  'starttime': 22,
+  'vsize': 23,
+  'rss': 24,
+  'processor': 39,
+}
+
+class ProcStat(TypedDict):
+  name: str
+  pid: int
+  state: str
+  ppid: int
+  utime: int
+  stime: int
+  cutime: int
+  cstime: int
+  priority: int
+  nice: int
+  num_threads: int
+  starttime: int
+  vms: int
+  rss: int
+  processor: int
+
+
+def _parse_proc_stat(stat: str) -> ProcStat | None:
+  open_paren = stat.find('(')
+  close_paren = stat.rfind(')')
+  if open_paren == -1 or close_paren == -1 or open_paren > close_paren:
+    return None
+  name = stat[open_paren + 1:close_paren]
+  stat = stat[:open_paren] + stat[open_paren:close_paren].replace(' ', '_') + stat[close_paren:]
+  parts = stat.split()
+  if len(parts) < 52:
+    return None
+  try:
+    return {
+      'name': name,
+      'pid': int(parts[_STAT_POS['pid'] - 1]),
+      'state': parts[_STAT_POS['state'] - 1][0],
+      'ppid': int(parts[_STAT_POS['ppid'] - 1]),
+      'utime': int(parts[_STAT_POS['utime'] - 1]),
+      'stime': int(parts[_STAT_POS['stime'] - 1]),
+      'cutime': int(parts[_STAT_POS['cutime'] - 1]),
+      'cstime': int(parts[_STAT_POS['cstime'] - 1]),
+      'priority': int(parts[_STAT_POS['priority'] - 1]),
+      'nice': int(parts[_STAT_POS['nice'] - 1]),
+      'num_threads': int(parts[_STAT_POS['num_threads'] - 1]),
+      'starttime': int(parts[_STAT_POS['starttime'] - 1]),
+      'vms': int(parts[_STAT_POS['vsize'] - 1]),
+      'rss': int(parts[_STAT_POS['rss'] - 1]),
+      'processor': int(parts[_STAT_POS['processor'] - 1]),
+    }
+  except Exception:
+    cloudlog.exception("failed to parse /proc/<pid>/stat")
+    return None
+
+class SmapsData(TypedDict):
+  pss: int       # bytes
+  pss_anon: int  # bytes
+  pss_shmem: int # bytes
+
+
+_SMAPS_KEYS = {b'Pss:', b'Pss_Anon:', b'Pss_Shmem:'}
+
+# smaps_rollup (kernel 4.14+) is ideal but missing on some BSP kernels;
+# fall back to per-VMA smaps (any kernel). Pss_Anon/Pss_Shmem only in 5.x+.
+_smaps_path: str | None = None  # auto-detected on first call
+
+# per-VMA smaps is expensive (kernel walks page tables for every VMA).
+# cache results and only refresh every N cycles to keep CPU low.
+_smaps_cache: dict[int, SmapsData] = {}
+_smaps_cycle = 0
+_SMAPS_EVERY = 20  # refresh every 20th cycle (40s at 0.5Hz)
+
+
+def _read_smaps(pid: int) -> SmapsData:
+  global _smaps_path
+  try:
+    if _smaps_path is None:
+      _smaps_path = 'smaps_rollup' if os.path.exists(f'/proc/{pid}/smaps_rollup') else 'smaps'
+
+    result: SmapsData = {'pss': 0, 'pss_anon': 0, 'pss_shmem': 0}
+    with open(f'/proc/{pid}/{_smaps_path}', 'rb') as f:
+      for line in f:
+        parts = line.split()
+        if len(parts) >= 2 and parts[0] in _SMAPS_KEYS:
+          val = int(parts[1]) * 1024  # kB -> bytes
+          if parts[0] == b'Pss:':
+            result['pss'] += val
+          elif parts[0] == b'Pss_Anon:':
+            result['pss_anon'] += val
+          elif parts[0] == b'Pss_Shmem:':
+            result['pss_shmem'] += val
+    return result
+  except (FileNotFoundError, PermissionError, ProcessLookupError, OSError):
+    return {'pss': 0, 'pss_anon': 0, 'pss_shmem': 0}
+
+
+def _get_smaps_cached(pid: int) -> SmapsData:
+  """Return cached smaps data, refreshing every _SMAPS_EVERY cycles."""
+  if _smaps_cycle == 0 or pid not in _smaps_cache:
+    _smaps_cache[pid] = _read_smaps(pid)
+  return _smaps_cache.get(pid, {'pss': 0, 'pss_anon': 0, 'pss_shmem': 0})
+
+
+class ProcExtra(TypedDict):
+  pid: int
+  name: str
+  exe: str
+  cmdline: list[str]
+
+
+_proc_cache: dict[int, ProcExtra] = {}
+
+
+def _get_proc_extra(pid: int, name: str) -> ProcExtra:
+  cache: ProcExtra | None = _proc_cache.get(pid)
+  if cache is None or cache.get('name') != name:
+    exe = ''
+    cmdline: list[str] = []
+    try:
+      exe = os.readlink(f'/proc/{pid}/exe')
+    except OSError:
+      pass
+    try:
+      with open(f'/proc/{pid}/cmdline', 'rb') as f:
+        cmdline = [c.decode('utf-8', errors='replace') for c in f.read().split(b'\0') if c]
+    except OSError:
+      pass
+    cache = {'pid': pid, 'name': name, 'exe': exe, 'cmdline': cmdline}
+    _proc_cache[pid] = cache
+  return cache
+
+
+def _procs() -> list[ProcStat]:
+  stats: list[ProcStat] = []
+  for pid_str in os.listdir('/proc'):
+    if not pid_str.isdigit():
+      continue
+    try:
+      with open(f'/proc/{pid_str}/stat') as f:
+        stat = f.read()
+      parsed = _parse_proc_stat(stat)
+      if parsed is not None:
+        stats.append(parsed)
+    except OSError:
+      continue
+  return stats
+
+
+def build_proc_log_message(msg) -> None:
+  pl = msg.procLog
+
+  procs = _procs()
+  l = pl.init('procs', len(procs))
+  for i, r in enumerate(procs):
+    proc = l[i]
+    proc.pid = r['pid']
+    proc.state = ord(r['state'][0])
+    proc.ppid = r['ppid']
+    proc.cpuUser = r['utime'] / JIFFY
+    proc.cpuSystem = r['stime'] / JIFFY
+    proc.cpuChildrenUser = r['cutime'] / JIFFY
+    proc.cpuChildrenSystem = r['cstime'] / JIFFY
+    proc.priority = r['priority']
+    proc.nice = r['nice']
+    proc.numThreads = r['num_threads']
+    proc.startTime = r['starttime'] / JIFFY
+    proc.memVms = r['vms']
+    proc.memRss = r['rss'] * PAGE_SIZE
+    proc.processor = r['processor']
+    proc.name = r['name']
+
+    extra = _get_proc_extra(r['pid'], r['name'])
+    proc.exe = extra['exe']
+    cmdline = proc.init('cmdline', len(extra['cmdline']))
+    for j, arg in enumerate(extra['cmdline']):
+      cmdline[j] = arg
+
+    # smaps is expensive (kernel walks page tables); skip small processes, use cache
+    if r['rss'] * PAGE_SIZE > 5 * 1024 * 1024:
+      smaps = _get_smaps_cached(r['pid'])
+      proc.memPss = smaps['pss']
+      proc.memPssAnon = smaps['pss_anon']
+      proc.memPssShmem = smaps['pss_shmem']
+
+  cpu_times = _cpu_times()
+  cpu_list = pl.init('cpuTimes', len(cpu_times))
+  for i, ct in enumerate(cpu_times):
+    cpu = cpu_list[i]
+    cpu.cpuNum = ct['cpuNum']
+    cpu.user = ct['user']
+    cpu.nice = ct['nice']
+    cpu.system = ct['system']
+    cpu.idle = ct['idle']
+    cpu.iowait = ct['iowait']
+    cpu.irq = ct['irq']
+    cpu.softirq = ct['softirq']
+
+  mem_info = _mem_info()
+  pl.mem.total = mem_info["MemTotal:"]
+  pl.mem.free = mem_info["MemFree:"]
+  pl.mem.available = mem_info["MemAvailable:"]
+  pl.mem.buffers = mem_info["Buffers:"]
+  pl.mem.cached = mem_info["Cached:"]
+  pl.mem.active = mem_info["Active:"]
+  pl.mem.inactive = mem_info["Inactive:"]
+  pl.mem.shared = mem_info["Shmem:"]
+
+  global _smaps_cycle
+  _smaps_cycle = (_smaps_cycle + 1) % _SMAPS_EVERY
+
+
+def main() -> NoReturn:
+  pm = messaging.PubMaster(['procLog'])
+  rk = Ratekeeper(0.5)
+  while True:
+    msg = messaging.new_message('procLog', valid=True)
+    build_proc_log_message(msg)
+    pm.send('procLog', msg)
+    rk.keep_time()
+
+
+if __name__ == '__main__':
+  main()
--- a/Show More
+++ b/Show More