dragonpilot v2023.07.05

version: dragonpilot v0.9.4 release
date: 2023-07-05T18:59:41
dp-dev(priv) master commit: 7b0489feab40283a422d2201ef95a9cb8c06f6cd

tools/joystick/README.md

@@ -0,0 +1,64 @@
+# Joystick
+
+**Hardware needed**: device running openpilot, laptop, joystick (optional)
+
+With joystickd, you can connect your laptop to your comma device over the network and debug controls using a joystick or keyboard.
+joystickd uses [inputs](https://pypi.org/project/inputs) which supports many common gamepads and joysticks.
+
+## Usage
+
+The car must be off, and openpilot must be offroad before starting `joystickd`.
+
+### Using a keyboard
+
+SSH into your comma device and start joystickd with the following command:
+
+```shell
+tools/joystick/joystickd.py --keyboard
+```
+
+The available buttons and axes will print showing their key mappings. In general, the WASD keys control gas and brakes and steering torque in 5% increments.
+
+### Joystick on your comma three
+
+Plug the joystick into your comma three aux USB-C port. Then, SSH into the device and start `joystickd.py`.
+
+### Joystick on your laptop
+
+In order to use a joystick over the network, we need to run joystickd locally from your laptop and have it send `testJoystick` packets over the network to the comma device.
+
+1. Connect a joystick to your PC.
+2. Connect your laptop to your comma device's hotspot and open a new SSH shell. Since joystickd is being run on your laptop, we need to write a parameter to let controlsd know to start in joystick debug mode:
+   ```shell
+   # on your comma device
+   echo -n "1" > /data/params/d/JoystickDebugMode
+   ```
+3. Run bridge with your laptop's IP address. This republishes the `testJoystick` packets sent from your laptop so that openpilot can receive them:
+   ```shell
+   # on your comma device
+   cereal/messaging/bridge {LAPTOP_IP} testJoystick
+   ```
+4. Start joystickd on your laptop in ZMQ mode.
+   ```shell
+   # on your laptop
+   export ZMQ=1
+   tools/joystick/joystickd.py
+   ```
+
+### Web joystick on your mobile device
+
+A browser-based virtual joystick designed for touch screens. Starts automatically when installed on comma body (non-car robotics platform).
+For cars, start the web joystick service manually via SSH before starting the car.
+
+```shell
+tools/joystick/web.py
+```
+
+After starting the car/body, open the web joystick app at this URL: `http://[comma three IP address]:5000`
+
+---
+Now start your car and openpilot should go into joystick mode with an alert on startup! The status of the axes will display on the alert, while button statuses print in the shell.
+
+Make sure the conditions are met in the panda to allow controls (e.g. cruise control engaged). You can also make a modification to the panda code to always allow controls.
+
+![](steer.gif)

tools/joystick/joystickd.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+import os
+import argparse
+import threading
+from inputs import get_gamepad
+
+import cereal.messaging as messaging
+from common.realtime import Ratekeeper
+from common.numpy_fast import interp, clip
+from common.params import Params
+from tools.lib.kbhit import KBHit
+
+
+class Keyboard:
+  def __init__(self):
+    self.kb = KBHit()
+    self.axis_increment = 0.05  # 5% of full actuation each key press
+    self.axes_map = {'w': 'gb', 's': 'gb',
+                     'a': 'steer', 'd': 'steer'}
+    self.axes_values = {'gb': 0., 'steer': 0.}
+    self.axes_order = ['gb', 'steer']
+    self.cancel = False
+
+  def update(self):
+    key = self.kb.getch().lower()
+    self.cancel = False
+    if key == 'r':
+      self.axes_values = {ax: 0. for ax in self.axes_values}
+    elif key == 'c':
+      self.cancel = True
+    elif key in self.axes_map:
+      axis = self.axes_map[key]
+      incr = self.axis_increment if key in ['w', 'a'] else -self.axis_increment
+      self.axes_values[axis] = clip(self.axes_values[axis] + incr, -1, 1)
+    else:
+      return False
+    return True
+
+
+class Joystick:
+  def __init__(self, gamepad=False):
+    # TODO: find a way to get this from API, perhaps "inputs" doesn't support it
+    if gamepad:
+      self.cancel_button = 'BTN_NORTH'  # (BTN_NORTH=X, ABS_RZ=Right Trigger)
+      accel_axis = 'ABS_Y'
+      steer_axis = 'ABS_RX'
+    else:
+      self.cancel_button = 'BTN_TRIGGER'
+      accel_axis = 'ABS_Y'
+      steer_axis = 'ABS_RZ'
+    self.min_axis_value = {accel_axis: 0., steer_axis: 0.}
+    self.max_axis_value = {accel_axis: 255., steer_axis: 255.}
+    self.axes_values = {accel_axis: 0., steer_axis: 0.}
+    self.axes_order = [accel_axis, steer_axis]
+    self.cancel = False
+
+  def update(self):
+    joystick_event = get_gamepad()[0]
+    event = (joystick_event.code, joystick_event.state)
+    if event[0] == self.cancel_button:
+      if event[1] == 1:
+        self.cancel = True
+      elif event[1] == 0:   # state 0 is falling edge
+        self.cancel = False
+    elif event[0] in self.axes_values:
+      self.max_axis_value[event[0]] = max(event[1], self.max_axis_value[event[0]])
+      self.min_axis_value[event[0]] = min(event[1], self.min_axis_value[event[0]])
+
+      norm = -interp(event[1], [self.min_axis_value[event[0]], self.max_axis_value[event[0]]], [-1., 1.])
+      self.axes_values[event[0]] = norm if abs(norm) > 0.05 else 0.  # center can be noisy, deadzone of 5%
+    else:
+      return False
+    return True
+
+
+def send_thread(joystick):
+  joystick_sock = messaging.pub_sock('testJoystick')
+  rk = Ratekeeper(100, print_delay_threshold=None)
+  while 1:
+    dat = messaging.new_message('testJoystick')
+    dat.testJoystick.axes = [joystick.axes_values[a] for a in joystick.axes_order]
+    dat.testJoystick.buttons = [joystick.cancel]
+    joystick_sock.send(dat.to_bytes())
+    print('\n' + ', '.join(f'{name}: {round(v, 3)}' for name, v in joystick.axes_values.items()))
+    if "WEB" in os.environ:
+      import requests
+      requests.get("http://"+os.environ["WEB"]+":5000/control/%f/%f" % tuple([joystick.axes_values[a] for a in joystick.axes_order][::-1]), timeout=None)
+    rk.keep_time()
+
+def joystick_thread(joystick):
+  Params().put_bool('JoystickDebugMode', True)
+  threading.Thread(target=send_thread, args=(joystick,), daemon=True).start()
+  while True:
+    joystick.update()
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='Publishes events from your joystick to control your car.\n' +
+                                               'openpilot must be offroad before starting joysticked.',
+                                   formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument('--keyboard', action='store_true', help='Use your keyboard instead of a joystick')
+  parser.add_argument('--gamepad', action='store_true', help='Use gamepad configuration instead of joystick')
+  args = parser.parse_args()
+
+  if not Params().get_bool("IsOffroad") and "ZMQ" not in os.environ and "WEB" not in os.environ:
+    print("The car must be off before running joystickd.")
+    exit()
+
+  print()
+  if args.keyboard:
+    print('Gas/brake control: `W` and `S` keys')
+    print('Steering control: `A` and `D` keys')
+    print('Buttons')
+    print('- `R`: Resets axes')
+    print('- `C`: Cancel cruise control')
+  else:
+    print('Using joystick, make sure to run cereal/messaging/bridge on your device if running over the network!')
+
+  joystick = Keyboard() if args.keyboard else Joystick(args.gamepad)
+  joystick_thread(joystick)

tools/joystick/web.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import time
+import threading
+from flask import Flask
+
+import cereal.messaging as messaging
+
+app = Flask(__name__)
+pm = messaging.PubMaster(['testJoystick'])
+
+index = """
+<html>
+<head>
+<script src="https://github.com/bobboteck/JoyStick/releases/download/v1.1.6/joy.min.js"></script>
+</head>
+<body>
+<div id="joyDiv" style="width:100%;height:100%"></div>
+<script type="text/javascript">
+// Set up gamepad handlers
+let gamepad = null;
+window.addEventListener("gamepadconnected", function(e) {
+  gamepad = e.gamepad;
+});
+window.addEventListener("gamepaddisconnected", function(e) {
+  gamepad = null;
+});
+// Create JoyStick object into the DIV 'joyDiv'
+var joy = new JoyStick('joyDiv');
+setInterval(function(){
+  var x = -joy.GetX()/100;
+  var y = joy.GetY()/100;
+  if (x === 0 && y === 0 && gamepad !== null) {
+    let gamepadstate = navigator.getGamepads()[gamepad.index];
+    x = -gamepadstate.axes[0];
+    y = -gamepadstate.axes[1];
+  }
+  let xhr = new XMLHttpRequest();
+  xhr.open("GET", "/control/"+x+"/"+y);
+  xhr.send();
+}, 50);
+</script>
+"""
+
+@app.route("/")
+def hello_world():
+  return index
+
+last_send_time = time.monotonic()
+@app.route("/control/<x>/<y>")
+def control(x, y):
+  global last_send_time
+  x,y = float(x), float(y)
+  x = max(-1, min(1, x))
+  y = max(-1, min(1, y))
+  dat = messaging.new_message('testJoystick')
+  dat.testJoystick.axes = [y,x]
+  dat.testJoystick.buttons = [False]
+  pm.send('testJoystick', dat)
+  last_send_time = time.monotonic()
+  return ""
+
+def handle_timeout():
+  while 1:
+    this_time = time.monotonic()
+    if (last_send_time+0.5) < this_time:
+      #print("timeout, no web in %.2f s" % (this_time-last_send_time))
+      dat = messaging.new_message('testJoystick')
+      dat.testJoystick.axes = [0,0]
+      dat.testJoystick.buttons = [False]
+      pm.send('testJoystick', dat)
+    time.sleep(0.1)
+
+def main():
+  threading.Thread(target=handle_timeout, daemon=True).start()
+  app.run(host="0.0.0.0")
+
+if __name__ == '__main__':
+  main()

tools/lib/README.md

@@ -0,0 +1,51 @@
+## LogReader
+
+Route is a class for conveniently accessing all the [logs](/system/loggerd/) from your routes. The LogReader class reads the non-video logs, i.e. rlog.bz2 and qlog.bz2. There's also a matching FrameReader class for reading the videos.
+
+```python
+from tools.lib.route import Route
+from tools.lib.logreader import LogReader
+
+r = Route("4cf7a6ad03080c90|2021-09-29--13-46-36")
+
+# get a list of paths for the route's rlog files
+print(r.log_paths())
+
+# and road camera (fcamera.hevc) files
+print(r.camera_paths())
+
+# setup a LogReader to read the route's first rlog
+lr = LogReader(r.log_paths()[0])
+
+# print out all the messages in the log
+import codecs
+codecs.register_error("strict", codecs.backslashreplace_errors)
+for msg in lr:
+  print(msg)
+
+# setup a LogReader for the route's second qlog
+lr = LogReader(r.log_paths()[1])
+
+# print all the steering angles values from the log
+for msg in lr:
+  if msg.which() == "carState":
+    print(msg.carState.steeringAngleDeg)
+```
+
+### MultiLogIterator
+
+`MultiLogIterator` is similar to `LogReader`, but reads multiple logs. 
+
+```python
+from tools.lib.route import Route
+from tools.lib.logreader import MultiLogIterator
+
+# setup a MultiLogIterator to read all the logs in the route
+r = Route("4cf7a6ad03080c90|2021-09-29--13-46-36")
+lr = MultiLogIterator(r.log_paths())
+
+# print all the steering angles values from all the logs in the route
+for msg in lr:
+  if msg.which() == "carState":
+    print(msg.carState.steeringAngleDeg)
+```

tools/lib/api.py

@@ -0,0 +1,35 @@
+import os
+import requests
+from common.params import Params
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com') if not Params().get_bool("dp_api_custom") else Params().get("dp_api_custom_url", encoding='utf-8')
+
+class CommaApi():
+  def __init__(self, token=None):
+    self.session = requests.Session()
+    self.session.headers['User-agent'] = 'OpenpilotTools'
+    if token:
+      self.session.headers['Authorization'] = 'JWT ' + token
+
+  def request(self, method, endpoint, **kwargs):
+    resp = self.session.request(method, API_HOST + '/' + endpoint, **kwargs)
+    resp_json = resp.json()
+    if isinstance(resp_json, dict) and resp_json.get('error'):
+      if resp.status_code in [401, 403]:
+        raise UnauthorizedError('Unauthorized. Authenticate with tools/lib/auth.py')
+
+      e = APIError(str(resp.status_code) + ":" + resp_json.get('description', str(resp_json['error'])))
+      e.status_code = resp.status_code
+      raise e
+    return resp_json
+
+  def get(self, endpoint, **kwargs):
+    return self.request('GET', endpoint, **kwargs)
+
+  def post(self, endpoint, **kwargs):
+    return self.request('POST', endpoint, **kwargs)
+
+class APIError(Exception):
+  pass
+
+class UnauthorizedError(Exception):
+  pass

tools/lib/auth.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+Usage::
+
+  usage: auth.py [-h] [{google,apple,github,jwt}] [jwt]
+
+  Login to your comma account
+
+  positional arguments:
+    {google,apple,github,jwt}
+    jwt
+
+  optional arguments:
+    -h, --help            show this help message and exit
+
+
+Examples::
+
+  ./auth.py  # Log in with google account
+  ./auth.py github  # Log in with GitHub Account
+  ./auth.py jwt ey......hw  # Log in with a JWT from https://jwt.comma.ai, for use in CI
+"""
+
+import argparse
+import sys
+import pprint
+import webbrowser
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Dict
+from urllib.parse import parse_qs, urlencode
+
+from tools.lib.api import APIError, CommaApi, UnauthorizedError
+from tools.lib.auth_config import set_token, get_token
+
+PORT = 3000
+
+
+class ClientRedirectServer(HTTPServer):
+  query_params: Dict[str, Any] = {}
+
+
+class ClientRedirectHandler(BaseHTTPRequestHandler):
+  def do_GET(self):
+    if not self.path.startswith('/auth'):
+      self.send_response(204)
+      return
+
+    query = self.path.split('?', 1)[-1]
+    query_parsed = parse_qs(query, keep_blank_values=True)
+    self.server.query_params = query_parsed
+
+    self.send_response(200)
+    self.send_header('Content-type', 'text/plain')
+    self.end_headers()
+    self.wfile.write(b'Return to the CLI to continue')
+
+  def log_message(self, format, *args):  # pylint: disable=redefined-builtin
+    pass  # this prevent http server from dumping messages to stdout
+
+
+def auth_redirect_link(method):
+  provider_id = {
+    'google': 'g',
+    'apple': 'a',
+    'github': 'h',
+  }[method]
+
+  params = {
+    'redirect_uri': f"https://api.comma.ai/v2/auth/{provider_id}/redirect/",
+    'state': f'service,localhost:{PORT}',
+  }
+
+  if method == 'google':
+    params.update({
+      'type': 'web_server',
+      'client_id': '45471411055-ornt4svd2miog6dnopve7qtmh5mnu6id.apps.googleusercontent.com',
+      'response_type': 'code',
+      'scope': 'https://www.googleapis.com/auth/userinfo.email',
+      'prompt': 'select_account',
+    })
+    return 'https://accounts.google.com/o/oauth2/auth?' + urlencode(params)
+  elif method == 'github':
+    params.update({
+      'client_id': '28c4ecb54bb7272cb5a4',
+      'scope': 'read:user',
+    })
+    return 'https://github.com/login/oauth/authorize?' + urlencode(params)
+  elif method == 'apple':
+    params.update({
+      'client_id': 'ai.comma.login',
+      'response_type': 'code',
+      'response_mode': 'form_post',
+      'scope': 'name email',
+    })
+    return 'https://appleid.apple.com/auth/authorize?' + urlencode(params)
+  else:
+    raise NotImplementedError(f"no redirect implemented for method {method}")
+
+
+def login(method):
+  oauth_uri = auth_redirect_link(method)
+
+  web_server = ClientRedirectServer(('localhost', PORT), ClientRedirectHandler)
+  print(f'To sign in, use your browser and navigate to {oauth_uri}')
+  webbrowser.open(oauth_uri, new=2)
+
+  while True:
+    web_server.handle_request()
+    if 'code' in web_server.query_params:
+      break
+    elif 'error' in web_server.query_params:
+      print('Authentication Error: "{}". Description: "{}" '.format(
+        web_server.query_params['error'],
+        web_server.query_params.get('error_description')), file=sys.stderr)
+      break
+
+  try:
+    auth_resp = CommaApi().post('v2/auth/', data={'code': web_server.query_params['code'], 'provider': web_server.query_params['provider']})
+    set_token(auth_resp['access_token'])
+  except APIError as e:
+    print(f'Authentication Error: {e}', file=sys.stderr)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='Login to your comma account')
+  parser.add_argument('method', default='google', const='google', nargs='?', choices=['google', 'apple', 'github', 'jwt'])
+  parser.add_argument('jwt', nargs='?')
+
+  args = parser.parse_args()
+  if args.method == 'jwt':
+    if args.jwt is None:
+      print("method JWT selected, but no JWT was provided")
+      exit(1)
+
+    set_token(args.jwt)
+  else:
+    login(args.method)
+
+  try:
+    me = CommaApi(token=get_token()).get('/v1/me')
+    print("Authenticated!")
+    pprint.pprint(me)
+  except UnauthorizedError:
+    print("Got invalid JWT")
+    exit(1)

tools/lib/auth_config.py

@@ -0,0 +1,34 @@
+import json
+import os
+from common.file_helpers import mkdirs_exists_ok
+from system.hardware import PC
+
+
+class MissingAuthConfigError(Exception):
+  pass
+
+
+if PC:
+  CONFIG_DIR = os.path.expanduser('~/.comma')
+else:
+  CONFIG_DIR = "/tmp/.comma"
+
+mkdirs_exists_ok(CONFIG_DIR)
+
+
+def get_token():
+  try:
+    with open(os.path.join(CONFIG_DIR, 'auth.json')) as f:
+      auth = json.load(f)
+      return auth['access_token']
+  except Exception:
+    return None
+
+
+def set_token(token):
+  with open(os.path.join(CONFIG_DIR, 'auth.json'), 'w') as f:
+    json.dump({'access_token': token}, f)
+
+
+def clear_token():
+  os.unlink(os.path.join(CONFIG_DIR, 'auth.json'))

tools/lib/bootlog.py

@@ -0,0 +1,63 @@
+import datetime
+import functools
+import re
+from typing import List, Optional
+
+from tools.lib.auth_config import get_token
+from tools.lib.api import CommaApi
+from tools.lib.helpers import RE, timestamp_to_datetime
+
+
+@functools.total_ordering
+class Bootlog:
+  def __init__(self, url: str):
+    self._url = url
+
+    r = re.search(RE.BOOTLOG_NAME, url)
+    if not r:
+      raise Exception(f"Unable to parse: {url}")
+
+    self._dongle_id = r.group('dongle_id')
+    self._timestamp = r.group('timestamp')
+
+  @property
+  def url(self) -> str:
+    return self._url
+
+  @property
+  def dongle_id(self) -> str:
+    return self._dongle_id
+
+  @property
+  def timestamp(self) -> str:
+    return self._timestamp
+
+  @property
+  def datetime(self) -> datetime.datetime:
+    return timestamp_to_datetime(self._timestamp)
+
+  def __str__(self):
+    return f"{self._dongle_id}|{self._timestamp}"
+
+  def __eq__(self, b) -> bool:
+    if not isinstance(b, Bootlog):
+      return False
+    return self.datetime == b.datetime
+
+  def __lt__(self, b) -> bool:
+    if not isinstance(b, Bootlog):
+      return False
+    return self.datetime < b.datetime
+
+def get_bootlog_from_id(bootlog_id: str) -> Optional[Bootlog]:
+  # TODO: implement an API endpoint for this
+  bl = Bootlog(bootlog_id)
+  for b in get_bootlogs(bl.dongle_id):
+    if b == bl:
+      return b
+  return None
+
+def get_bootlogs(dongle_id: str) -> List[Bootlog]:
+  api = CommaApi(get_token())
+  r = api.get(f'v1/devices/{dongle_id}/bootlogs')
+  return [Bootlog(b) for b in r]

tools/lib/cache.py

@@ -0,0 +1,15 @@
+import os
+import urllib.parse
+from common.file_helpers import mkdirs_exists_ok
+
+DEFAULT_CACHE_DIR = os.path.expanduser("~/.commacache")
+
+def cache_path_for_file_path(fn, cache_prefix=None):
+  dir_ = os.path.join(DEFAULT_CACHE_DIR, "local")
+  mkdirs_exists_ok(dir_)
+  fn_parsed = urllib.parse.urlparse(fn)
+  if fn_parsed.scheme == '':
+    cache_fn = os.path.abspath(fn).replace("/", "_")
+  else:
+    cache_fn = f'{fn_parsed.hostname}_{fn_parsed.path.replace("/", "_")}'
+  return os.path.join(dir_, cache_fn)

tools/lib/exceptions.py

@@ -0,0 +1,2 @@
+class DataUnreadableError(Exception):
+  pass

tools/lib/filereader.py

@@ -0,0 +1,11 @@
+import os
+from tools.lib.url_file import URLFile
+
+DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/")
+
+def FileReader(fn, debug=False):
+  if fn.startswith("cd:/"):
+    fn = fn.replace("cd:/", DATA_ENDPOINT)
+  if fn.startswith("http://") or fn.startswith("https://"):
+    return URLFile(fn, debug=debug)
+  return open(fn, "rb")

tools/lib/framereader.py

@@ -0,0 +1,611 @@
+# pylint: skip-file
+import json
+import os
+import pickle
+import struct
+import subprocess
+import tempfile
+import threading
+from enum import IntEnum
+from functools import wraps
+
+import numpy as np
+from lru import LRU
+
+import _io
+from tools.lib.cache import cache_path_for_file_path
+from tools.lib.exceptions import DataUnreadableError
+from common.file_helpers import atomic_write_in_dir
+
+from tools.lib.filereader import FileReader
+
+HEVC_SLICE_B = 0
+HEVC_SLICE_P = 1
+HEVC_SLICE_I = 2
+
+
+class GOPReader:
+  def get_gop(self, num):
+    # returns (start_frame_num, num_frames, frames_to_skip, gop_data)
+    raise NotImplementedError
+
+
+class DoNothingContextManager:
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *x):
+    pass
+
+
+class FrameType(IntEnum):
+  raw = 1
+  h265_stream = 2
+
+
+def fingerprint_video(fn):
+  with FileReader(fn) as f:
+    header = f.read(4)
+  if len(header) == 0:
+    raise DataUnreadableError(f"{fn} is empty")
+  elif header == b"\x00\xc0\x12\x00":
+    return FrameType.raw
+  elif header == b"\x00\x00\x00\x01":
+    if 'hevc' in fn:
+      return FrameType.h265_stream
+    else:
+      raise NotImplementedError(fn)
+  else:
+    raise NotImplementedError(fn)
+
+
+def ffprobe(fn, fmt=None):
+  cmd = ["ffprobe",
+         "-v", "quiet",
+         "-print_format", "json",
+         "-show_format", "-show_streams"]
+  if fmt:
+    cmd += ["-f", fmt]
+  cmd += [fn]
+
+  try:
+    ffprobe_output = subprocess.check_output(cmd)
+  except subprocess.CalledProcessError:
+    raise DataUnreadableError(fn)
+
+  return json.loads(ffprobe_output)
+
+
+def vidindex(fn, typ):
+  vidindex_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vidindex")
+  vidindex = os.path.join(vidindex_dir, "vidindex")
+
+  subprocess.check_call(["make"], cwd=vidindex_dir, stdout=open("/dev/null", "w"))
+
+  with tempfile.NamedTemporaryFile() as prefix_f, \
+       tempfile.NamedTemporaryFile() as index_f:
+    try:
+      subprocess.check_call([vidindex, typ, fn, prefix_f.name, index_f.name])
+    except subprocess.CalledProcessError:
+      raise DataUnreadableError(f"vidindex failed on file {fn}")
+    with open(index_f.name, "rb") as f:
+      index = f.read()
+    with open(prefix_f.name, "rb") as f:
+      prefix = f.read()
+
+  index = np.frombuffer(index, np.uint32).reshape(-1, 2)
+
+  assert index[-1, 0] == 0xFFFFFFFF
+  assert index[-1, 1] == os.path.getsize(fn)
+
+  return index, prefix
+
+
+def cache_fn(func):
+  @wraps(func)
+  def cache_inner(fn, *args, **kwargs):
+    if kwargs.pop('no_cache', None):
+      cache_path = None
+    else:
+      cache_prefix = kwargs.pop('cache_prefix', None)
+      cache_path = cache_path_for_file_path(fn, cache_prefix)
+
+    if cache_path and os.path.exists(cache_path):
+      with open(cache_path, "rb") as cache_file:
+        cache_value = pickle.load(cache_file)
+    else:
+      cache_value = func(fn, *args, **kwargs)
+
+      if cache_path:
+        with atomic_write_in_dir(cache_path, mode="wb", overwrite=True) as cache_file:
+          pickle.dump(cache_value, cache_file, -1)
+
+    return cache_value
+
+  return cache_inner
+
+
+@cache_fn
+def index_stream(fn, typ):
+  assert typ in ("hevc", )
+
+  with FileReader(fn) as f:
+    assert os.path.exists(f.name), fn
+    index, prefix = vidindex(f.name, typ)
+    probe = ffprobe(f.name, typ)
+
+  return {
+    'index': index,
+    'global_prefix': prefix,
+    'probe': probe
+  }
+
+
+def index_videos(camera_paths, cache_prefix=None):
+  """Requires that paths in camera_paths are contiguous and of the same type."""
+  if len(camera_paths) < 1:
+    raise ValueError("must provide at least one video to index")
+
+  frame_type = fingerprint_video(camera_paths[0])
+  for fn in camera_paths:
+    index_video(fn, frame_type, cache_prefix)
+
+
+def index_video(fn, frame_type=None, cache_prefix=None):
+  cache_path = cache_path_for_file_path(fn, cache_prefix)
+
+  if os.path.exists(cache_path):
+    return
+
+  if frame_type is None:
+    frame_type = fingerprint_video(fn[0])
+
+  if frame_type == FrameType.h265_stream:
+    index_stream(fn, "hevc", cache_prefix=cache_prefix)
+  else:
+    raise NotImplementedError("Only h265 supported")
+
+
+def get_video_index(fn, frame_type, cache_prefix=None):
+  cache_path = cache_path_for_file_path(fn, cache_prefix)
+
+  if not os.path.exists(cache_path):
+    index_video(fn, frame_type, cache_prefix)
+
+  if not os.path.exists(cache_path):
+    return None
+  with open(cache_path, "rb") as cache_file:
+    return pickle.load(cache_file)
+
+
+def read_file_check_size(f, sz, cookie):
+  buff = bytearray(sz)
+  bytes_read = f.readinto(buff)
+  assert bytes_read == sz, (bytes_read, sz)
+  return buff
+
+
+def rgb24toyuv(rgb):
+  yuv_from_rgb = np.array([[ 0.299     ,  0.587     ,  0.114      ],
+                           [-0.14714119, -0.28886916,  0.43601035 ],
+                           [ 0.61497538, -0.51496512, -0.10001026 ]])
+  img = np.dot(rgb.reshape(-1, 3), yuv_from_rgb.T).reshape(rgb.shape)
+
+
+
+  ys = img[:, :, 0]
+  us = (img[::2, ::2, 1] + img[1::2, ::2, 1] + img[::2, 1::2, 1] + img[1::2, 1::2, 1]) / 4 + 128
+  vs = (img[::2, ::2, 2] + img[1::2, ::2, 2] + img[::2, 1::2, 2] + img[1::2, 1::2, 2]) / 4 + 128
+
+  return ys, us, vs
+
+
+def rgb24toyuv420(rgb):
+  ys, us, vs = rgb24toyuv(rgb)
+
+  y_len = rgb.shape[0] * rgb.shape[1]
+  uv_len = y_len // 4
+
+  yuv420 = np.empty(y_len + 2 * uv_len, dtype=rgb.dtype)
+  yuv420[:y_len] = ys.reshape(-1)
+  yuv420[y_len:y_len + uv_len] = us.reshape(-1)
+  yuv420[y_len + uv_len:y_len + 2 * uv_len] = vs.reshape(-1)
+
+  return yuv420.clip(0, 255).astype('uint8')
+
+
+def rgb24tonv12(rgb):
+  ys, us, vs = rgb24toyuv(rgb)
+
+  y_len = rgb.shape[0] * rgb.shape[1]
+  uv_len = y_len // 4
+
+  nv12 = np.empty(y_len + 2 * uv_len, dtype=rgb.dtype)
+  nv12[:y_len] = ys.reshape(-1)
+  nv12[y_len::2] = us.reshape(-1)
+  nv12[y_len+1::2] = vs.reshape(-1)
+
+  return nv12.clip(0, 255).astype('uint8')
+
+
+def decompress_video_data(rawdat, vid_fmt, w, h, pix_fmt):
+  # using a tempfile is much faster than proc.communicate for some reason
+
+  with tempfile.TemporaryFile() as tmpf:
+    tmpf.write(rawdat)
+    tmpf.seek(0)
+
+    threads = os.getenv("FFMPEG_THREADS", "0")
+    cuda = os.getenv("FFMPEG_CUDA", "0") == "1"
+    proc = subprocess.Popen(
+      ["ffmpeg",
+       "-threads", threads,
+       "-hwaccel", "none" if not cuda else "cuda",
+       "-c:v", "hevc",
+       "-vsync", "0",
+       "-f", vid_fmt,
+       "-flags2", "showall",
+       "-i", "pipe:0",
+       "-threads", threads,
+       "-f", "rawvideo",
+       "-pix_fmt", pix_fmt,
+       "pipe:1"],
+      stdin=tmpf, stdout=subprocess.PIPE, stderr=open("/dev/null"))
+
+    # dat = proc.communicate()[0]
+    dat = proc.stdout.read()
+    if proc.wait() != 0:
+      raise DataUnreadableError("ffmpeg failed")
+
+  if pix_fmt == "rgb24":
+    ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, h, w, 3)
+  elif pix_fmt == "nv12":
+    ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, (h*w*3//2))
+  elif pix_fmt == "yuv420p":
+    ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, (h*w*3//2))
+  elif pix_fmt == "yuv444p":
+    ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, 3, h, w)
+  else:
+    raise NotImplementedError
+
+  return ret
+
+
+class BaseFrameReader:
+  # properties: frame_type, frame_count, w, h
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *args):
+    self.close()
+
+  def close(self):
+    pass
+
+  def get(self, num, count=1, pix_fmt="yuv420p"):
+    raise NotImplementedError
+
+
+def FrameReader(fn, cache_prefix=None, readahead=False, readbehind=False, index_data=None):
+  frame_type = fingerprint_video(fn)
+  if frame_type == FrameType.raw:
+    return RawFrameReader(fn)
+  elif frame_type in (FrameType.h265_stream,):
+    if not index_data:
+      index_data = get_video_index(fn, frame_type, cache_prefix)
+    return StreamFrameReader(fn, frame_type, index_data, readahead=readahead, readbehind=readbehind)
+  else:
+    raise NotImplementedError(frame_type)
+
+
+class RawData:
+  def __init__(self, f):
+    self.f = _io.FileIO(f, 'rb')
+    self.lenn = struct.unpack("I", self.f.read(4))[0]
+    self.count = os.path.getsize(f) / (self.lenn+4)
+
+  def read(self, i):
+    self.f.seek((self.lenn+4)*i + 4)
+    return self.f.read(self.lenn)
+
+
+class RawFrameReader(BaseFrameReader):
+  def __init__(self, fn):
+    # raw camera
+    self.fn = fn
+    self.frame_type = FrameType.raw
+    self.rawfile = RawData(self.fn)
+    self.frame_count = self.rawfile.count
+    self.w, self.h = 640, 480
+
+  def load_and_debayer(self, img):
+    img = np.frombuffer(img, dtype='uint8').reshape(960, 1280)
+    cimg = np.dstack([img[0::2, 1::2], ((img[0::2, 0::2].astype("uint16") + img[1::2, 1::2].astype("uint16")) >> 1).astype("uint8"), img[1::2, 0::2]])
+    return cimg
+
+  def get(self, num, count=1, pix_fmt="yuv420p"):
+    assert self.frame_count is not None
+    assert num+count <= self.frame_count
+
+    if pix_fmt not in ("nv12", "yuv420p", "rgb24"):
+      raise ValueError(f"Unsupported pixel format {pix_fmt!r}")
+
+    app = []
+    for i in range(num, num+count):
+      dat = self.rawfile.read(i)
+      rgb_dat = self.load_and_debayer(dat)
+      if pix_fmt == "rgb24":
+        app.append(rgb_dat)
+      elif pix_fmt == "nv12":
+        app.append(rgb24tonv12(rgb_dat))
+      elif pix_fmt == "yuv420p":
+        app.append(rgb24toyuv420(rgb_dat))
+      else:
+        raise NotImplementedError
+
+    return app
+
+
+class VideoStreamDecompressor:
+  def __init__(self, fn, vid_fmt, w, h, pix_fmt):
+    self.fn = fn
+    self.vid_fmt = vid_fmt
+    self.w = w
+    self.h = h
+    self.pix_fmt = pix_fmt
+
+    if pix_fmt in ("nv12", "yuv420p"):
+      self.out_size = w*h*3//2  # yuv420p
+    elif pix_fmt in ("rgb24", "yuv444p"):
+      self.out_size = w*h*3
+    else:
+      raise NotImplementedError
+
+    self.proc = None
+    self.t = threading.Thread(target=self.write_thread)
+    self.t.daemon = True
+
+  def write_thread(self):
+    try:
+      with FileReader(self.fn) as f:
+        while True:
+          r = f.read(1024*1024)
+          if len(r) == 0:
+            break
+          self.proc.stdin.write(r)
+    except BrokenPipeError:
+      pass
+    finally:
+      self.proc.stdin.close()
+
+  def read(self):
+    threads = os.getenv("FFMPEG_THREADS", "0")
+    cuda = os.getenv("FFMPEG_CUDA", "0") == "1"
+    cmd = [
+      "ffmpeg",
+      "-threads", threads,
+      "-hwaccel", "none" if not cuda else "cuda",
+      "-c:v", "hevc",
+      # "-avioflags", "direct",
+      "-analyzeduration", "0",
+      "-probesize", "32",
+      "-flush_packets", "0",
+      # "-fflags", "nobuffer",
+      "-vsync", "0",
+      "-f", self.vid_fmt,
+      "-i", "pipe:0",
+      "-threads", threads,
+      "-f", "rawvideo",
+      "-pix_fmt", self.pix_fmt,
+      "pipe:1"
+    ]
+    self.proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    try:
+      self.t.start()
+
+      while True:
+        dat = self.proc.stdout.read(self.out_size)
+        if len(dat) == 0:
+          break
+        assert len(dat) == self.out_size
+        if self.pix_fmt == "rgb24":
+          ret = np.frombuffer(dat, dtype=np.uint8).reshape((self.h, self.w, 3))
+        elif self.pix_fmt == "yuv420p":
+          ret = np.frombuffer(dat, dtype=np.uint8)
+        elif self.pix_fmt == "nv12":
+          ret = np.frombuffer(dat, dtype=np.uint8)
+        elif self.pix_fmt == "yuv444p":
+          ret = np.frombuffer(dat, dtype=np.uint8).reshape((3, self.h, self.w))
+        else:
+          assert False
+        yield ret
+
+      result_code = self.proc.wait()
+      assert result_code == 0, result_code
+    finally:
+      self.proc.kill()
+      self.t.join()
+
+class StreamGOPReader(GOPReader):
+  def __init__(self, fn, frame_type, index_data):
+    assert frame_type == FrameType.h265_stream
+
+    self.fn = fn
+
+    self.frame_type = frame_type
+    self.frame_count = None
+    self.w, self.h = None, None
+
+    self.prefix = None
+    self.index = None
+
+    self.index = index_data['index']
+    self.prefix = index_data['global_prefix']
+    probe = index_data['probe']
+
+    self.prefix_frame_data = None
+    self.num_prefix_frames = 0
+    self.vid_fmt = "hevc"
+
+    i = 0
+    while i < self.index.shape[0] and self.index[i, 0] != HEVC_SLICE_I:
+      i += 1
+    self.first_iframe = i
+
+    assert self.first_iframe == 0
+
+    self.frame_count = len(self.index) - 1
+
+    self.w = probe['streams'][0]['width']
+    self.h = probe['streams'][0]['height']
+
+  def _lookup_gop(self, num):
+    frame_b = num
+    while frame_b > 0 and self.index[frame_b, 0] != HEVC_SLICE_I:
+      frame_b -= 1
+
+    frame_e = num + 1
+    while frame_e < (len(self.index) - 1) and self.index[frame_e, 0] != HEVC_SLICE_I:
+      frame_e += 1
+
+    offset_b = self.index[frame_b, 1]
+    offset_e = self.index[frame_e, 1]
+
+    return (frame_b, frame_e, offset_b, offset_e)
+
+  def get_gop(self, num):
+    frame_b, frame_e, offset_b, offset_e = self._lookup_gop(num)
+    assert frame_b <= num < frame_e
+
+    num_frames = frame_e - frame_b
+
+    with FileReader(self.fn) as f:
+      f.seek(offset_b)
+      rawdat = f.read(offset_e - offset_b)
+
+      if num < self.first_iframe:
+        assert self.prefix_frame_data
+        rawdat = self.prefix_frame_data + rawdat
+
+      rawdat = self.prefix + rawdat
+
+    skip_frames = 0
+    if num < self.first_iframe:
+      skip_frames = self.num_prefix_frames
+
+    return frame_b, num_frames, skip_frames, rawdat
+
+
+class GOPFrameReader(BaseFrameReader):
+  #FrameReader with caching and readahead for formats that are group-of-picture based
+
+  def __init__(self, readahead=False, readbehind=False):
+    self.open_ = True
+
+    self.readahead = readahead
+    self.readbehind = readbehind
+    self.frame_cache = LRU(64)
+
+    if self.readahead:
+      self.cache_lock = threading.RLock()
+      self.readahead_last = None
+      self.readahead_len = 30
+      self.readahead_c = threading.Condition()
+      self.readahead_thread = threading.Thread(target=self._readahead_thread)
+      self.readahead_thread.daemon = True
+      self.readahead_thread.start()
+    else:
+      self.cache_lock = DoNothingContextManager()
+
+  def close(self):
+    if not self.open_:
+      return
+    self.open_ = False
+
+    if self.readahead:
+      self.readahead_c.acquire()
+      self.readahead_c.notify()
+      self.readahead_c.release()
+      self.readahead_thread.join()
+
+  def _readahead_thread(self):
+    while True:
+      self.readahead_c.acquire()
+      try:
+        if not self.open_:
+          break
+        self.readahead_c.wait()
+      finally:
+        self.readahead_c.release()
+      if not self.open_:
+        break
+      assert self.readahead_last
+      num, pix_fmt = self.readahead_last
+
+      if self.readbehind:
+        for k in range(num - 1, max(0, num - self.readahead_len), -1):
+          self._get_one(k, pix_fmt)
+      else:
+        for k in range(num, min(self.frame_count, num + self.readahead_len)):
+          self._get_one(k, pix_fmt)
+
+  def _get_one(self, num, pix_fmt):
+    assert num < self.frame_count
+
+    if (num, pix_fmt) in self.frame_cache:
+      return self.frame_cache[(num, pix_fmt)]
+
+    with self.cache_lock:
+      if (num, pix_fmt) in self.frame_cache:
+        return self.frame_cache[(num, pix_fmt)]
+
+      frame_b, num_frames, skip_frames, rawdat = self.get_gop(num)
+
+      ret = decompress_video_data(rawdat, self.vid_fmt, self.w, self.h, pix_fmt)
+      ret = ret[skip_frames:]
+      assert ret.shape[0] == num_frames
+
+      for i in range(ret.shape[0]):
+        self.frame_cache[(frame_b+i, pix_fmt)] = ret[i]
+
+      return self.frame_cache[(num, pix_fmt)]
+
+  def get(self, num, count=1, pix_fmt="yuv420p"):
+    assert self.frame_count is not None
+
+    if num + count > self.frame_count:
+      raise ValueError(f"{num + count} > {self.frame_count}")
+
+    if pix_fmt not in ("nv12", "yuv420p", "rgb24", "yuv444p"):
+      raise ValueError(f"Unsupported pixel format {pix_fmt!r}")
+
+    ret = [self._get_one(num + i, pix_fmt) for i in range(count)]
+
+    if self.readahead:
+      self.readahead_last = (num+count, pix_fmt)
+      self.readahead_c.acquire()
+      self.readahead_c.notify()
+      self.readahead_c.release()
+
+    return ret
+
+
+class StreamFrameReader(StreamGOPReader, GOPFrameReader):
+  def __init__(self, fn, frame_type, index_data, readahead=False, readbehind=False):
+    StreamGOPReader.__init__(self, fn, frame_type, index_data)
+    GOPFrameReader.__init__(self, readahead, readbehind)
+
+
+def GOPFrameIterator(gop_reader, pix_fmt):
+  dec = VideoStreamDecompressor(gop_reader.fn, gop_reader.vid_fmt, gop_reader.w, gop_reader.h, pix_fmt)
+  yield from dec.read()
+
+
+def FrameIterator(fn, pix_fmt, **kwargs):
+  fr = FrameReader(fn, **kwargs)
+  if isinstance(fr, GOPReader):
+    yield from GOPFrameIterator(fr, pix_fmt)
+  else:
+    for i in range(fr.frame_count):
+      yield fr.get(i, pix_fmt=pix_fmt)[0]

tools/lib/helpers.py

@@ -0,0 +1,32 @@
+import bz2
+import datetime
+
+TIME_FMT = "%Y-%m-%d--%H-%M-%S"
+
+# regex patterns
+class RE:
+  DONGLE_ID =  r'(?P<dongle_id>[a-z0-9]{16})'
+  TIMESTAMP = r'(?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2})'
+  ROUTE_NAME = r'{}[|_/]{}'.format(DONGLE_ID, TIMESTAMP)
+  SEGMENT_NAME = r'{}(?:--|/)(?P<segment_num>[0-9]+)'.format(ROUTE_NAME)
+  BOOTLOG_NAME = ROUTE_NAME
+
+  EXPLORER_FILE = r'^(?P<segment_name>{})--(?P<file_name>[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME)
+  OP_SEGMENT_DIR = r'^(?P<segment_name>{})$'.format(SEGMENT_NAME)
+
+
+def timestamp_to_datetime(t: str) -> datetime.datetime:
+  """
+    Convert an openpilot route timestamp to a python datetime
+  """
+  return datetime.datetime.strptime(t, TIME_FMT)
+
+
+def save_log(dest, log_msgs, compress=True):
+  dat = b"".join(msg.as_builder().to_bytes() for msg in log_msgs)
+
+  if compress:
+    dat = bz2.compress(dat)
+
+  with open(dest, "wb") as f:
+    f.write(dat)

tools/lib/kbhit.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+import sys
+import termios
+import atexit
+from select import select
+
+STDIN_FD = sys.stdin.fileno()
+
+class KBHit:
+  def __init__(self) -> None:
+    ''' Creates a KBHit object that you can call to do various keyboard things.
+    '''
+
+    self.set_kbhit_terminal()
+
+  def set_kbhit_terminal(self) -> None:
+    ''' Save old terminal settings for closure, remove ICANON & ECHO flags.
+    '''
+
+    # Save the terminal settings
+    self.old_term = termios.tcgetattr(STDIN_FD)
+    self.new_term = self.old_term.copy()
+
+    # New terminal setting unbuffered
+    self.new_term[3] &= ~(termios.ICANON | termios.ECHO)
+    termios.tcsetattr(STDIN_FD, termios.TCSAFLUSH, self.new_term)
+
+    # Support normal-terminal reset at exit
+    atexit.register(self.set_normal_term)
+
+  def set_normal_term(self) -> None:
+    ''' Resets to normal terminal. On Windows this is a no-op.
+    '''
+
+    termios.tcsetattr(STDIN_FD, termios.TCSAFLUSH, self.old_term)
+
+  @staticmethod
+  def getch() -> str:
+    ''' Returns a keyboard character after kbhit() has been called.
+      Should not be called in the same program as getarrow().
+    '''
+    return sys.stdin.read(1)
+
+  @staticmethod
+  def getarrow() -> int:
+    ''' Returns an arrow-key code after kbhit() has been called. Codes are
+    0 : up
+    1 : right
+    2 : down
+    3 : left
+    Should not be called in the same program as getch().
+    '''
+
+    c = sys.stdin.read(3)[2]
+    vals = [65, 67, 66, 68]
+
+    return vals.index(ord(c))
+
+  @staticmethod
+  def kbhit():
+    ''' Returns True if keyboard character was hit, False otherwise.
+    '''
+    return select([sys.stdin], [], [], 0)[0] != []
+
+
+# Test
+if __name__ == "__main__":
+
+  kb = KBHit()
+
+  print('Hit any key, or ESC to exit')
+
+  while True:
+
+    if kb.kbhit():
+      c = kb.getch()
+      if c == '\x1b':  # ESC
+        break
+      print(c)
+
+  kb.set_normal_term()

tools/lib/logreader.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+import os
+import sys
+import bz2
+import urllib.parse
+import capnp
+import warnings
+
+
+from cereal import log as capnp_log
+from tools.lib.filereader import FileReader
+from tools.lib.route import Route, SegmentName
+
+# this is an iterator itself, and uses private variables from LogReader
+class MultiLogIterator:
+  def __init__(self, log_paths, sort_by_time=False):
+    self._log_paths = log_paths
+    self.sort_by_time = sort_by_time
+
+    self._first_log_idx = next(i for i in range(len(log_paths)) if log_paths[i] is not None)
+    self._current_log = self._first_log_idx
+    self._idx = 0
+    self._log_readers = [None]*len(log_paths)
+    self.start_time = self._log_reader(self._first_log_idx)._ts[0]
+
+  def _log_reader(self, i):
+    if self._log_readers[i] is None and self._log_paths[i] is not None:
+      log_path = self._log_paths[i]
+      self._log_readers[i] = LogReader(log_path, sort_by_time=self.sort_by_time)
+
+    return self._log_readers[i]
+
+  def __iter__(self):
+    return self
+
+  def _inc(self):
+    lr = self._log_reader(self._current_log)
+    if self._idx < len(lr._ents)-1:
+      self._idx += 1
+    else:
+      self._idx = 0
+      self._current_log = next(i for i in range(self._current_log + 1, len(self._log_readers) + 1)
+                               if i == len(self._log_readers) or self._log_paths[i] is not None)
+      if self._current_log == len(self._log_readers):
+        raise StopIteration
+
+  def __next__(self):
+    while 1:
+      lr = self._log_reader(self._current_log)
+      ret = lr._ents[self._idx]
+      self._inc()
+      return ret
+
+  def tell(self):
+    # returns seconds from start of log
+    return (self._log_reader(self._current_log)._ts[self._idx] - self.start_time) * 1e-9
+
+  def seek(self, ts):
+    # seek to nearest minute
+    minute = int(ts/60)
+    if minute >= len(self._log_paths) or self._log_paths[minute] is None:
+      return False
+
+    self._current_log = minute
+
+    # HACK: O(n) seek afterward
+    self._idx = 0
+    while self.tell() < ts:
+      self._inc()
+    return True
+
+  def reset(self):
+    self.__init__(self._log_paths, sort_by_time=self.sort_by_time)
+
+
+class LogReader:
+  def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False, dat=None):
+    self.data_version = None
+    self._only_union_types = only_union_types
+
+    ext = None
+    if not dat:
+      _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
+      if ext not in ('', '.bz2'):
+        # old rlogs weren't bz2 compressed
+        raise Exception(f"unknown extension {ext}")
+
+      with FileReader(fn) as f:
+        dat = f.read()
+
+    if ext == ".bz2" or dat.startswith(b'BZh9'):
+      dat = bz2.decompress(dat)
+
+    ents = capnp_log.Event.read_multiple_bytes(dat)
+
+    _ents = []
+    try:
+      for e in ents:
+        _ents.append(e)
+    except capnp.KjException:
+      warnings.warn("Corrupted events detected", RuntimeWarning)
+
+    self._ents = list(sorted(_ents, key=lambda x: x.logMonoTime) if sort_by_time else _ents)
+    self._ts = [x.logMonoTime for x in self._ents]
+
+  @classmethod
+  def from_bytes(cls, dat):
+    return cls("", dat=dat)
+
+  def __iter__(self):
+    for ent in self._ents:
+      if self._only_union_types:
+        try:
+          ent.which()
+          yield ent
+        except capnp.lib.capnp.KjException:
+          pass
+      else:
+        yield ent
+
+def logreader_from_route_or_segment(r, sort_by_time=False):
+  sn = SegmentName(r, allow_route_name=True)
+  route = Route(sn.route_name.canonical_name)
+  if sn.segment_num < 0:
+    return MultiLogIterator(route.log_paths(), sort_by_time=sort_by_time)
+  else:
+    return LogReader(route.log_paths()[sn.segment_num], sort_by_time=sort_by_time)
+
+
+if __name__ == "__main__":
+  import codecs
+  # capnproto <= 0.8.0 throws errors converting byte data to string
+  # below line catches those errors and replaces the bytes with \x__
+  codecs.register_error("strict", codecs.backslashreplace_errors)
+  log_path = sys.argv[1]
+  lr = LogReader(log_path, sort_by_time=True)
+  for msg in lr:
+    print(msg)

tools/lib/route.py

@@ -0,0 +1,231 @@
+import os
+import re
+from urllib.parse import urlparse
+from collections import defaultdict
+from itertools import chain
+from typing import Optional
+
+from tools.lib.auth_config import get_token
+from tools.lib.api import CommaApi
+from tools.lib.helpers import RE
+
+QLOG_FILENAMES = ['qlog', 'qlog.bz2']
+QCAMERA_FILENAMES = ['qcamera.ts']
+LOG_FILENAMES = ['rlog', 'rlog.bz2', 'raw_log.bz2']
+CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc']
+DCAMERA_FILENAMES = ['dcamera.hevc']
+ECAMERA_FILENAMES = ['ecamera.hevc']
+
+class Route:
+  def __init__(self, name, data_dir=None):
+    self._name = RouteName(name)
+    self.files = None
+    if data_dir is not None:
+      self._segments = self._get_segments_local(data_dir)
+    else:
+      self._segments = self._get_segments_remote()
+    self.max_seg_number = self._segments[-1].name.segment_num
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def segments(self):
+    return self._segments
+
+  def log_paths(self):
+    log_path_by_seg_num = {s.name.segment_num: s.log_path for s in self._segments}
+    return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def qlog_paths(self):
+    qlog_path_by_seg_num = {s.name.segment_num: s.qlog_path for s in self._segments}
+    return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def camera_paths(self):
+    camera_path_by_seg_num = {s.name.segment_num: s.camera_path for s in self._segments}
+    return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def dcamera_paths(self):
+    dcamera_path_by_seg_num = {s.name.segment_num: s.dcamera_path for s in self._segments}
+    return [dcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def ecamera_paths(self):
+    ecamera_path_by_seg_num = {s.name.segment_num: s.ecamera_path for s in self._segments}
+    return [ecamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def qcamera_paths(self):
+    qcamera_path_by_seg_num = {s.name.segment_num: s.qcamera_path for s in self._segments}
+    return [qcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  # TODO: refactor this, it's super repetitive
+  def _get_segments_remote(self):
+    api = CommaApi(get_token())
+    route_files = api.get('v1/route/' + self.name.canonical_name + '/files')
+    self.files = list(chain.from_iterable(route_files.values()))
+
+    segments = {}
+    for url in self.files:
+      _, dongle_id, time_str, segment_num, fn = urlparse(url).path.rsplit('/', maxsplit=4)
+      segment_name = f'{dongle_id}|{time_str}--{segment_num}'
+      if segments.get(segment_name):
+        segments[segment_name] = Segment(
+          segment_name,
+          url if fn in LOG_FILENAMES else segments[segment_name].log_path,
+          url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path,
+          url if fn in CAMERA_FILENAMES else segments[segment_name].camera_path,
+          url if fn in DCAMERA_FILENAMES else segments[segment_name].dcamera_path,
+          url if fn in ECAMERA_FILENAMES else segments[segment_name].ecamera_path,
+          url if fn in QCAMERA_FILENAMES else segments[segment_name].qcamera_path,
+        )
+      else:
+        segments[segment_name] = Segment(
+          segment_name,
+          url if fn in LOG_FILENAMES else None,
+          url if fn in QLOG_FILENAMES else None,
+          url if fn in CAMERA_FILENAMES else None,
+          url if fn in DCAMERA_FILENAMES else None,
+          url if fn in ECAMERA_FILENAMES else None,
+          url if fn in QCAMERA_FILENAMES else None,
+        )
+
+    return sorted(segments.values(), key=lambda seg: seg.name.segment_num)
+
+  def _get_segments_local(self, data_dir):
+    files = os.listdir(data_dir)
+    segment_files = defaultdict(list)
+
+    for f in files:
+      fullpath = os.path.join(data_dir, f)
+      explorer_match = re.match(RE.EXPLORER_FILE, f)
+      op_match = re.match(RE.OP_SEGMENT_DIR, f)
+
+      if explorer_match:
+        segment_name = explorer_match.group('segment_name')
+        fn = explorer_match.group('file_name')
+        if segment_name.replace('_', '|').startswith(self.name.canonical_name):
+          segment_files[segment_name].append((fullpath, fn))
+      elif op_match and os.path.isdir(fullpath):
+        segment_name = op_match.group('segment_name')
+        if segment_name.startswith(self.name.canonical_name):
+          for seg_f in os.listdir(fullpath):
+            segment_files[segment_name].append((os.path.join(fullpath, seg_f), seg_f))
+      elif f == self.name.canonical_name:
+        for seg_num in os.listdir(fullpath):
+          if not seg_num.isdigit():
+            continue
+
+          segment_name = f'{self.name.canonical_name}--{seg_num}'
+          for seg_f in os.listdir(os.path.join(fullpath, seg_num)):
+            segment_files[segment_name].append((os.path.join(fullpath, seg_num, seg_f), seg_f))
+
+    segments = []
+    for segment, files in segment_files.items():
+
+      try:
+        log_path = next(path for path, filename in files if filename in LOG_FILENAMES)
+      except StopIteration:
+        log_path = None
+
+      try:
+        qlog_path = next(path for path, filename in files if filename in QLOG_FILENAMES)
+      except StopIteration:
+        qlog_path = None
+
+      try:
+        camera_path = next(path for path, filename in files if filename in CAMERA_FILENAMES)
+      except StopIteration:
+        camera_path = None
+
+      try:
+        dcamera_path = next(path for path, filename in files if filename in DCAMERA_FILENAMES)
+      except StopIteration:
+        dcamera_path = None
+
+      try:
+        ecamera_path = next(path for path, filename in files if filename in ECAMERA_FILENAMES)
+      except StopIteration:
+        ecamera_path = None
+
+      try:
+        qcamera_path = next(path for path, filename in files if filename in QCAMERA_FILENAMES)
+      except StopIteration:
+        qcamera_path = None
+
+      segments.append(Segment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path))
+
+    if len(segments) == 0:
+      raise ValueError(f'Could not find segments for route {self.name.canonical_name} in data directory {data_dir}')
+    return sorted(segments, key=lambda seg: seg.name.segment_num)
+
+class Segment:
+  def __init__(self, name, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path):
+    self._name = SegmentName(name)
+    self.log_path = log_path
+    self.qlog_path = qlog_path
+    self.camera_path = camera_path
+    self.dcamera_path = dcamera_path
+    self.ecamera_path = ecamera_path
+    self.qcamera_path = qcamera_path
+
+  @property
+  def name(self):
+    return self._name
+
+class RouteName:
+  def __init__(self, name_str: str):
+    self._name_str = name_str
+    delim = next(c for c in self._name_str if c in ("|", "/"))
+    self._dongle_id, self._time_str = self._name_str.split(delim)
+
+    assert len(self._dongle_id) == 16, self._name_str
+    assert len(self._time_str) == 20, self._name_str
+    self._canonical_name = f"{self._dongle_id}|{self._time_str}"
+
+  @property
+  def canonical_name(self) -> str: return self._canonical_name
+
+  @property
+  def dongle_id(self) -> str: return self._dongle_id
+
+  @property
+  def time_str(self) -> str: return self._time_str
+
+  def __str__(self) -> str: return self._canonical_name
+
+class SegmentName:
+  # TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances
+  # of this class instead of manually constructing a segment name (use canonical_name prop instead)
+  def __init__(self, name_str: str, allow_route_name=False):
+    data_dir_path_separator_index = name_str.rsplit("|", 1)[0].rfind("/")
+    use_data_dir = (data_dir_path_separator_index != -1) and ("|" in name_str)
+    self._name_str = name_str[data_dir_path_separator_index + 1:] if use_data_dir else name_str
+    self._data_dir = name_str[:data_dir_path_separator_index] if use_data_dir else None
+
+    seg_num_delim = "--" if self._name_str.count("--") == 2 else "/"
+    name_parts = self._name_str.rsplit(seg_num_delim, 1)
+    if allow_route_name and len(name_parts) == 1:
+      name_parts.append("-1") # no segment number
+    self._route_name = RouteName(name_parts[0])
+    self._num = int(name_parts[1])
+    self._canonical_name = f"{self._route_name._dongle_id}|{self._route_name._time_str}--{self._num}"
+
+  @property
+  def canonical_name(self) -> str: return self._canonical_name
+
+  @property
+  def dongle_id(self) -> str: return self._route_name.dongle_id
+
+  @property
+  def time_str(self) -> str: return self._route_name.time_str
+
+  @property
+  def segment_num(self) -> int: return self._num
+
+  @property
+  def route_name(self) -> RouteName: return self._route_name
+
+  @property
+  def data_dir(self) -> Optional[str]: return self._data_dir
+
+  def __str__(self) -> str: return self._canonical_name

tools/lib/tests/test_caching.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import unittest
+
+os.environ["COMMA_CACHE"] = "/tmp/__test_cache__"
+from tools.lib.url_file import URLFile, CACHE_DIR
+
+
+class TestFileDownload(unittest.TestCase):
+
+  def compare_loads(self, url, start=0, length=None):
+    """Compares range between cached and non cached version"""
+    shutil.rmtree(CACHE_DIR)
+
+    file_cached = URLFile(url, cache=True)
+    file_downloaded = URLFile(url, cache=False)
+
+    file_cached.seek(start)
+    file_downloaded.seek(start)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertLessEqual(length + start if length is not None else 0, file_downloaded.get_length())
+
+    response_cached = file_cached.read(ll=length)
+    response_downloaded = file_downloaded.read(ll=length)
+
+    self.assertEqual(response_cached, response_downloaded)
+
+    # Now test with cache in place
+    file_cached = URLFile(url, cache=True)
+    file_cached.seek(start)
+    response_cached = file_cached.read(ll=length)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertEqual(response_cached, response_downloaded)
+
+  def test_small_file(self):
+    # Make sure we don't force cache
+    os.environ["FILEREADER_CACHE"] = "0"
+    small_file_url = "https://raw.githubusercontent.com/commaai/openpilot/master/docs/SAFETY.md"
+    #  If you want large file to be larger than a chunk
+    #  large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/fcamera.hevc"
+
+    #  Load full small file
+    self.compare_loads(small_file_url)
+
+    file_small = URLFile(small_file_url)
+    length = file_small.get_length()
+
+    self.compare_loads(small_file_url, length - 100, 100)
+    self.compare_loads(small_file_url, 50, 100)
+
+    #  Load small file 100 bytes at a time
+    for i in range(length // 100):
+      self.compare_loads(small_file_url, 100 * i, 100)
+
+  def test_large_file(self):
+    large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+    #  Load the end 100 bytes of both files
+    file_large = URLFile(large_file_url)
+    length = file_large.get_length()
+
+    self.compare_loads(large_file_url, length - 100, 100)
+    self.compare_loads(large_file_url)
+
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/tests/test_readers.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import unittest
+import requests
+import tempfile
+
+from collections import defaultdict
+import numpy as np
+from tools.lib.framereader import FrameReader
+from tools.lib.logreader import LogReader
+
+
+class TestReaders(unittest.TestCase):
+  @unittest.skip("skip for bandwidth reasons")
+  def test_logreader(self):
+    def _check_data(lr):
+      hist = defaultdict(int)
+      for l in lr:
+        hist[l.which()] += 1
+
+      self.assertEqual(hist['carControl'], 6000)
+      self.assertEqual(hist['logMessage'], 6857)
+
+    with tempfile.NamedTemporaryFile(suffix=".bz2") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true", timeout=10)
+      fp.write(r.content)
+      fp.flush()
+
+      lr_file = LogReader(fp.name)
+      _check_data(lr_file)
+
+    lr_url = LogReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true")
+    _check_data(lr_url)
+
+  @unittest.skip("skip for bandwidth reasons")
+  def test_framereader(self):
+    def _check_data(f):
+      self.assertEqual(f.frame_count, 1200)
+      self.assertEqual(f.w, 1164)
+      self.assertEqual(f.h, 874)
+
+      frame_first_30 = f.get(0, 30)
+      self.assertEqual(len(frame_first_30), 30)
+
+      print(frame_first_30[15])
+
+      print("frame_0")
+      frame_0 = f.get(0, 1)
+      frame_15 = f.get(15, 1)
+
+      print(frame_15[0])
+
+      assert np.all(frame_first_30[0] == frame_0[0])
+      assert np.all(frame_first_30[15] == frame_15[0])
+
+    with tempfile.NamedTemporaryFile(suffix=".hevc") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true", timeout=10)
+      fp.write(r.content)
+      fp.flush()
+
+      fr_file = FrameReader(fp.name)
+      _check_data(fr_file)
+
+    fr_url = FrameReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true")
+    _check_data(fr_url)
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/tests/test_route_library.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import unittest
+from collections import namedtuple
+
+from tools.lib.route import SegmentName
+
+class TestRouteLibrary(unittest.TestCase):
+  def test_segment_name_formats(self):
+    Case = namedtuple('Case', ['input', 'expected_route', 'expected_segment_num', 'expected_data_dir'])
+
+    cases = [ Case("4cf7a6ad03080c90|2021-09-29--13-46-36", "4cf7a6ad03080c90|2021-09-29--13-46-36", -1, None),
+              Case("4cf7a6ad03080c90/2021-09-29--13-46-36--1", "4cf7a6ad03080c90|2021-09-29--13-46-36", 1, None),
+              Case("4cf7a6ad03080c90|2021-09-29--13-46-36/2", "4cf7a6ad03080c90|2021-09-29--13-46-36", 2, None),
+              Case("4cf7a6ad03080c90/2021-09-29--13-46-36/3", "4cf7a6ad03080c90|2021-09-29--13-46-36", 3, None),
+              Case("/data/media/0/realdata/4cf7a6ad03080c90|2021-09-29--13-46-36", "4cf7a6ad03080c90|2021-09-29--13-46-36", -1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/4cf7a6ad03080c90|2021-09-29--13-46-36--1", "4cf7a6ad03080c90|2021-09-29--13-46-36", 1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/4cf7a6ad03080c90|2021-09-29--13-46-36/2", "4cf7a6ad03080c90|2021-09-29--13-46-36", 2, "/data/media/0/realdata") ]
+
+    def _validate(case):
+      route_or_segment_name = case.input
+
+      s = SegmentName(route_or_segment_name, allow_route_name=True)
+
+      self.assertEqual(str(s.route_name), case.expected_route)
+      self.assertEqual(s.segment_num, case.expected_segment_num)
+      self.assertEqual(s.data_dir, case.expected_data_dir)
+
+    for case in cases:
+      _validate(case)
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/url_file.py

@@ -0,0 +1,199 @@
+# pylint: skip-file
+
+import os
+import time
+import tempfile
+import threading
+import urllib.parse
+import pycurl
+from hashlib import sha256
+from io import BytesIO
+from tenacity import retry, wait_random_exponential, stop_after_attempt
+from common.file_helpers import mkdirs_exists_ok, atomic_write_in_dir
+#  Cache chunk size
+K = 1000
+CHUNK_SIZE = 1000 * K
+
+CACHE_DIR = os.environ.get("COMMA_CACHE", "/tmp/comma_download_cache/")
+
+
+def hash_256(link):
+  hsh = str(sha256((link.split("?")[0]).encode('utf-8')).hexdigest())
+  return hsh
+
+
+class URLFile:
+  _tlocal = threading.local()
+
+  def __init__(self, url, debug=False, cache=None):
+    self._url = url
+    self._pos = 0
+    self._length = None
+    self._local_file = None
+    self._debug = debug
+    #  True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input
+    self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0"))
+    if cache is not None:
+      self._force_download = not cache
+
+    try:
+      self._curl = self._tlocal.curl
+    except AttributeError:
+      self._curl = self._tlocal.curl = pycurl.Curl()
+    mkdirs_exists_ok(CACHE_DIR)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    if self._local_file is not None:
+      os.remove(self._local_file.name)
+      self._local_file.close()
+      self._local_file = None
+
+  @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
+  def get_length_online(self):
+    c = self._curl
+    c.reset()
+    c.setopt(pycurl.NOSIGNAL, 1)
+    c.setopt(pycurl.TIMEOUT_MS, 500000)
+    c.setopt(pycurl.FOLLOWLOCATION, True)
+    c.setopt(pycurl.URL, self._url)
+    c.setopt(c.NOBODY, 1)
+    c.perform()
+    length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD))
+    c.reset()
+    return length
+
+  def get_length(self):
+    if self._length is not None:
+      return self._length
+    file_length_path = os.path.join(CACHE_DIR, hash_256(self._url) + "_length")
+    if os.path.exists(file_length_path) and not self._force_download:
+      with open(file_length_path) as file_length:
+          content = file_length.read()
+          self._length = int(content)
+          return self._length
+
+    self._length = self.get_length_online()
+    if not self._force_download:
+      with atomic_write_in_dir(file_length_path, mode="w") as file_length:
+        file_length.write(str(self._length))
+    return self._length
+
+  def read(self, ll=None):
+    if self._force_download:
+      return self.read_aux(ll=ll)
+
+    file_begin = self._pos
+    file_end = self._pos + ll if ll is not None else self.get_length()
+    assert file_end != -1, f"Remote file is empty or doesn't exist: {self._url}"
+    #  We have to align with chunks we store. Position is the begginiing of the latest chunk that starts before or at our file
+    position = (file_begin // CHUNK_SIZE) * CHUNK_SIZE
+    response = b""
+    while True:
+      self._pos = position
+      chunk_number = self._pos / CHUNK_SIZE
+      file_name = hash_256(self._url) + "_" + str(chunk_number)
+      full_path = os.path.join(CACHE_DIR, str(file_name))
+      data = None
+      #  If we don't have a file, download it
+      if not os.path.exists(full_path):
+        data = self.read_aux(ll=CHUNK_SIZE)
+        with atomic_write_in_dir(full_path, mode="wb") as new_cached_file:
+          new_cached_file.write(data)
+      else:
+        with open(full_path, "rb") as cached_file:
+          data = cached_file.read()
+
+      response += data[max(0, file_begin - position): min(CHUNK_SIZE, file_end - position)]
+
+      position += CHUNK_SIZE
+      if position >= file_end:
+        self._pos = file_end
+        return response
+
+  @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
+  def read_aux(self, ll=None):
+    download_range = False
+    headers = ["Connection: keep-alive"]
+    if self._pos != 0 or ll is not None:
+      if ll is None:
+        end = self.get_length() - 1
+      else:
+        end = min(self._pos + ll, self.get_length()) - 1
+      if self._pos >= end:
+        return b""
+      headers.append(f"Range: bytes={self._pos}-{end}")
+      download_range = True
+
+    dats = BytesIO()
+    c = self._curl
+    c.setopt(pycurl.URL, self._url)
+    c.setopt(pycurl.WRITEDATA, dats)
+    c.setopt(pycurl.NOSIGNAL, 1)
+    c.setopt(pycurl.TIMEOUT_MS, 500000)
+    c.setopt(pycurl.HTTPHEADER, headers)
+    c.setopt(pycurl.FOLLOWLOCATION, True)
+
+    if self._debug:
+      print("downloading", self._url)
+
+      def header(x):
+        if b'MISS' in x:
+          print(x.strip())
+
+      c.setopt(pycurl.HEADERFUNCTION, header)
+
+      def test(debug_type, debug_msg):
+       print("  debug(%d): %s" % (debug_type, debug_msg.strip()))
+
+      c.setopt(pycurl.VERBOSE, 1)
+      c.setopt(pycurl.DEBUGFUNCTION, test)
+      t1 = time.time()
+
+    c.perform()
+
+    if self._debug:
+      t2 = time.time()
+      if t2 - t1 > 0.1:
+        print(f"get {self._url} {headers!r} {t2 - t1:.f} slow")
+
+    response_code = c.getinfo(pycurl.RESPONSE_CODE)
+    if response_code == 416:  # Requested Range Not Satisfiable
+      raise Exception(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+    if download_range and response_code != 206:  # Partial Content
+      raise Exception(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+    if (not download_range) and response_code != 200:  # OK
+      raise Exception(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+
+    ret = dats.getvalue()
+    self._pos += len(ret)
+    return ret
+
+  def seek(self, pos):
+    self._pos = pos
+
+  @property
+  def name(self):
+    """Returns a local path to file with the URLFile's contents.
+
+       This can be used to interface with modules that require local files.
+    """
+    if self._local_file is None:
+      _, ext = os.path.splitext(urllib.parse.urlparse(self._url).path)
+      local_fd, local_path = tempfile.mkstemp(suffix=ext)
+      try:
+        os.write(local_fd, self.read())
+        local_file = open(local_path, "rb")
+      except Exception:
+        os.remove(local_path)
+        raise
+      finally:
+        os.close(local_fd)
+
+      self._local_file = local_file
+      self.read = self._local_file.read
+      self.seek = self._local_file.seek
+
+    return self._local_file.name

tools/lib/vidindex/.gitignore

@@ -0,0 +1 @@
+vidindex

tools/lib/vidindex/Makefile

@@ -0,0 +1,6 @@
+CC := gcc
+
+vidindex: bitstream.c bitstream.h vidindex.c
+	$(eval $@_TMP := $(shell mktemp))
+	$(CC) -std=c99 bitstream.c vidindex.c -o $($@_TMP)
+	mv $($@_TMP) $@

tools/lib/vidindex/bitstream.h

@@ -0,0 +1,26 @@
+#ifndef bitstream_H
+#define bitstream_H
+
+
+#include <stddef.h>
+#include <stdint.h>
+
+struct bitstream {
+  const uint8_t *buffer_ptr;
+  const uint8_t *buffer_end;
+  uint64_t value;
+  uint32_t pos;
+  uint32_t shift;
+  size_t size;
+};
+
+void bs_init(struct bitstream *bs, const uint8_t *buffer, size_t input_size);
+void bs_seek(struct bitstream *bs, size_t new_pos);
+uint32_t bs_get(struct bitstream *bs, int n);
+uint32_t bs_peek(struct bitstream *bs, int n);
+size_t bs_remain(struct bitstream *bs);
+int bs_eof(struct bitstream *bs);
+uint32_t bs_ue(struct bitstream *bs);
+int32_t bs_se(struct bitstream *bs);
+
+#endif

tools/replay/camera.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <unistd.h>
+#include "cereal/visionipc/visionipc_server.h"
+#include "common/queue.h"
+#include "tools/replay/framereader.h"
+#include "tools/replay/logreader.h"
+
+class CameraServer {
+public:
+  CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr);
+  ~CameraServer();
+  void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx);
+  void waitForSent();
+
+protected:
+  struct Camera {
+    CameraType type;
+    VisionStreamType stream_type;
+    int width;
+    int height;
+    std::thread thread;
+    SafeQueue<std::pair<FrameReader*, const cereal::EncodeIndex::Reader>> queue;
+    int cached_id = -1;
+    int cached_seg = -1;
+    VisionBuf * cached_buf;
+  };
+  void startVipcServer();
+  void cameraThread(Camera &cam);
+
+  Camera cameras_[MAX_CAMERAS] = {
+      {.type = RoadCam, .stream_type = VISION_STREAM_ROAD},
+      {.type = DriverCam, .stream_type = VISION_STREAM_DRIVER},
+      {.type = WideRoadCam, .stream_type = VISION_STREAM_WIDE_ROAD},
+  };
+  std::atomic<int> publishing_ = 0;
+  std::unique_ptr<VisionIpcServer> vipc_server_;
+};

tools/replay/consoleui.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <array>
+#include <QBasicTimer>
+#include <QObject>
+#include <QSocketNotifier>
+#include <QTimer>
+#include <QTimerEvent>
+
+#include "tools/replay/replay.h"
+#include <ncurses.h>
+
+class ConsoleUI : public QObject {
+  Q_OBJECT
+
+public:
+  ConsoleUI(Replay *replay, QObject *parent = 0);
+  ~ConsoleUI();
+
+private:
+  void initWindows();
+  void handleKey(char c);
+  void displayHelp();
+  void displayTimelineDesc();
+  void updateTimeline();
+  void updateSummary();
+  void updateStatus();
+  void pauseReplay(bool pause);
+
+  enum Status { Waiting, Playing, Paused };
+  enum Win { Title, Stats, Log, LogBorder, DownloadBar, Timeline, TimelineDesc, Help, CarState, Max};
+  std::array<WINDOW*, Win::Max> w{};
+  SubMaster sm;
+  Replay *replay;
+  QBasicTimer getch_timer;
+  QTimer sm_timer;
+  QSocketNotifier notifier{0, QSocketNotifier::Read, this};
+  int max_width, max_height;
+  Status status = Status::Waiting;
+
+signals:
+  void updateProgressBarSignal(uint64_t cur, uint64_t total, bool success);
+  void logMessageSignal(ReplyMsgType type, const QString &msg);
+
+private slots:
+  void readyRead();
+  void timerEvent(QTimerEvent *ev);
+  void updateProgressBar(uint64_t cur, uint64_t total, bool success);
+  void logMessage(ReplyMsgType type, const QString &msg);
+};

tools/replay/filereader.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <atomic>
+#include <string>
+
+class FileReader {
+public:
+  FileReader(bool cache_to_local, size_t chunk_size = 0, int retries = 3)
+      : cache_to_local_(cache_to_local), chunk_size_(chunk_size), max_retries_(retries) {}
+  virtual ~FileReader() {}
+  std::string read(const std::string &file, std::atomic<bool> *abort = nullptr);
+
+private:
+  std::string download(const std::string &url, std::atomic<bool> *abort);
+  size_t chunk_size_;
+  int max_retries_;
+  bool cache_to_local_;
+};
+
+std::string cacheFilePath(const std::string &url);

tools/replay/framereader.h

@@ -0,0 +1,51 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tools/replay/filereader.h"
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+}
+
+struct AVFrameDeleter {
+  void operator()(AVFrame* frame) const { av_frame_free(&frame); }
+};
+
+class FrameReader {
+public:
+  FrameReader();
+  ~FrameReader();
+  bool load(const std::string &url, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr, bool local_cache = false,
+            int chunk_size = -1, int retries = 0);
+  bool load(const std::byte *data, size_t size, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr);
+  bool get(int idx, uint8_t *yuv);
+  int getYUVSize() const { return width * height * 3 / 2; }
+  size_t getFrameCount() const { return packets.size(); }
+  bool valid() const { return valid_; }
+
+  int width = 0, height = 0;
+  int aligned_width = 0, aligned_height = 0;
+
+private:
+  bool initHardwareDecoder(AVHWDeviceType hw_device_type);
+  bool decode(int idx, uint8_t *yuv);
+  AVFrame * decodeFrame(AVPacket *pkt);
+  bool copyBuffers(AVFrame *f, uint8_t *yuv);
+
+  std::vector<AVPacket*> packets;
+  std::unique_ptr<AVFrame, AVFrameDeleter>av_frame_, hw_frame;
+  AVFormatContext *input_ctx = nullptr;
+  AVCodecContext *decoder_ctx = nullptr;
+  int key_frames_count_ = 0;
+  bool valid_ = false;
+  AVIOContext *avio_ctx_ = nullptr;
+
+  AVPixelFormat hw_pix_fmt = AV_PIX_FMT_NONE;
+  AVBufferRef *hw_device_ctx = nullptr;
+  int prev_idx = -1;
+  inline static std::atomic<bool> has_hw_decoder = true;
+};

tools/replay/logreader.h

@@ -0,0 +1,67 @@
+#pragma once
+
+#if __has_include(<memory_resource>)
+#define HAS_MEMORY_RESOURCE 1
+#include <memory_resource>
+#endif
+
+#include <set>
+
+#include "cereal/gen/cpp/log.capnp.h"
+#include "system/camerad/cameras/camera_common.h"
+#include "tools/replay/filereader.h"
+
+const CameraType ALL_CAMERAS[] = {RoadCam, DriverCam, WideRoadCam};
+const int MAX_CAMERAS = std::size(ALL_CAMERAS);
+const int DEFAULT_EVENT_MEMORY_POOL_BLOCK_SIZE = 65000;
+
+class Event {
+public:
+  Event(cereal::Event::Which which, uint64_t mono_time) : reader(kj::ArrayPtr<capnp::word>{}) {
+    // construct a dummy Event for binary search, e.g std::upper_bound
+    this->which = which;
+    this->mono_time = mono_time;
+  }
+  Event(const kj::ArrayPtr<const capnp::word> &amsg, bool frame = false);
+  inline kj::ArrayPtr<const capnp::byte> bytes() const { return words.asBytes(); }
+
+  struct lessThan {
+    inline bool operator()(const Event *l, const Event *r) {
+      return l->mono_time < r->mono_time || (l->mono_time == r->mono_time && l->which < r->which);
+    }
+  };
+
+#if HAS_MEMORY_RESOURCE
+  void *operator new(size_t size, std::pmr::monotonic_buffer_resource *mbr) {
+    return mbr->allocate(size);
+  }
+  void operator delete(void *ptr) {
+    // No-op. memory used by EventMemoryPool increases monotonically until the logReader is destroyed.
+  }
+#endif
+
+  uint64_t mono_time;
+  cereal::Event::Which which;
+  cereal::Event::Reader event;
+  capnp::FlatArrayMessageReader reader;
+  kj::ArrayPtr<const capnp::word> words;
+  bool frame;
+};
+
+class LogReader {
+public:
+  LogReader(size_t memory_pool_block_size = DEFAULT_EVENT_MEMORY_POOL_BLOCK_SIZE);
+  ~LogReader();
+  bool load(const std::string &url, std::atomic<bool> *abort = nullptr, const std::set<cereal::Event::Which> &allow = {},
+            bool local_cache = false, int chunk_size = -1, int retries = 0);
+  bool load(const std::byte *data, size_t size, std::atomic<bool> *abort = nullptr);
+  std::vector<Event*> events;
+
+private:
+  bool parse(const std::set<cereal::Event::Which> &allow, std::atomic<bool> *abort);
+  std::string raw_;
+#ifdef HAS_MEMORY_RESOURCE
+  std::pmr::monotonic_buffer_resource *mbr_ = nullptr;
+  void *pool_buffer_ = nullptr;
+#endif
+};

tools/replay/replay.h

@@ -0,0 +1,141 @@
+#pragma once
+
+#include <optional>
+
+#include <QThread>
+
+#include "tools/replay/camera.h"
+#include "tools/replay/route.h"
+
+const QString DEMO_ROUTE = "4cf7a6ad03080c90|2021-09-29--13-46-36";
+
+// one segment uses about 100M of memory
+constexpr int MIN_SEGMENTS_CACHE = 5;
+
+enum REPLAY_FLAGS {
+  REPLAY_FLAG_NONE = 0x0000,
+  REPLAY_FLAG_DCAM = 0x0002,
+  REPLAY_FLAG_ECAM = 0x0004,
+  REPLAY_FLAG_NO_LOOP = 0x0010,
+  REPLAY_FLAG_NO_FILE_CACHE = 0x0020,
+  REPLAY_FLAG_QCAMERA = 0x0040,
+  REPLAY_FLAG_NO_HW_DECODER = 0x0100,
+  REPLAY_FLAG_FULL_SPEED = 0x0200,
+  REPLAY_FLAG_NO_VIPC = 0x0400,
+  REPLAY_FLAG_ALL_SERVICES = 0x0800,
+};
+
+enum class FindFlag {
+  nextEngagement,
+  nextDisEngagement,
+  nextUserFlag,
+  nextInfo,
+  nextWarning,
+  nextCritical
+};
+
+enum class TimelineType { None, Engaged, AlertInfo, AlertWarning, AlertCritical, UserFlag };
+typedef bool (*replayEventFilter)(const Event *, void *);
+
+class Replay : public QObject {
+  Q_OBJECT
+
+public:
+  Replay(QString route, QStringList allow, QStringList block, QStringList base_blacklist, SubMaster *sm = nullptr,
+          uint32_t flags = REPLAY_FLAG_NONE, QString data_dir = "", QObject *parent = 0);
+  ~Replay();
+  bool load();
+  void start(int seconds = 0);
+  void stop();
+  void pause(bool pause);
+  void seekToFlag(FindFlag flag);
+  void seekTo(double seconds, bool relative);
+  inline bool isPaused() const { return paused_; }
+  // the filter is called in streaming thread.try to return quickly from it to avoid blocking streaming.
+  // the filter function must return true if the event should be filtered.
+  // otherwise it must return false.
+  inline void installEventFilter(replayEventFilter filter, void *opaque) {
+    filter_opaque = opaque;
+    event_filter = filter;
+  }
+  inline int segmentCacheLimit() const { return segment_cache_limit; }
+  inline void setSegmentCacheLimit(int n) { segment_cache_limit = std::max(MIN_SEGMENTS_CACHE, n); }
+  inline bool hasFlag(REPLAY_FLAGS flag) const { return flags_ & flag; }
+  inline void addFlag(REPLAY_FLAGS flag) { flags_ |= flag; }
+  inline void removeFlag(REPLAY_FLAGS flag) { flags_ &= ~flag; }
+  inline const Route* route() const { return route_.get(); }
+  inline double currentSeconds() const { return double(cur_mono_time_ - route_start_ts_) / 1e9; }
+  inline QDateTime currentDateTime() const { return route_->datetime().addSecs(currentSeconds()); }
+  inline uint64_t routeStartTime() const { return route_start_ts_; }
+  inline int toSeconds(uint64_t mono_time) const { return (mono_time - route_start_ts_) / 1e9; }
+  inline int totalSeconds() const { return (!segments_.empty()) ? (segments_.rbegin()->first + 1) * 60 : 0; }
+  inline void setSpeed(float speed) { speed_ = speed; }
+  inline float getSpeed() const { return speed_; }
+  inline const std::vector<Event *> *events() const { return events_.get(); }
+  inline const std::map<int, std::unique_ptr<Segment>> &segments() const { return segments_; };
+  inline const std::string &carFingerprint() const { return car_fingerprint_; }
+  inline const std::vector<std::tuple<int, int, TimelineType>> getTimeline() {
+    std::lock_guard lk(timeline_lock);
+    return timeline;
+  }
+
+signals:
+  void streamStarted();
+  void segmentsMerged();
+  void seekedTo(double sec);
+
+protected slots:
+  void segmentLoadFinished(bool success);
+
+protected:
+  typedef std::map<int, std::unique_ptr<Segment>> SegmentMap;
+  std::optional<uint64_t> find(FindFlag flag);
+  void startStream(const Segment *cur_segment);
+  void stream();
+  void setCurrentSegment(int n);
+  void queueSegment();
+  void mergeSegments(const SegmentMap::iterator &begin, const SegmentMap::iterator &end);
+  void updateEvents(const std::function<bool()>& lambda);
+  void publishMessage(const Event *e);
+  void publishFrame(const Event *e);
+  void buildTimeline();
+  inline bool isSegmentMerged(int n) {
+    return std::find(segments_merged_.begin(), segments_merged_.end(), n) != segments_merged_.end();
+  }
+
+  QThread *stream_thread_ = nullptr;
+
+  // logs
+  std::mutex stream_lock_;
+  std::condition_variable stream_cv_;
+  std::atomic<bool> updating_events_ = false;
+  std::atomic<int> current_segment_ = 0;
+  SegmentMap segments_;
+  // the following variables must be protected with stream_lock_
+  std::atomic<bool> exit_ = false;
+  bool paused_ = false;
+  bool events_updated_ = false;
+  uint64_t route_start_ts_ = 0;
+  std::atomic<uint64_t> cur_mono_time_ = 0;
+  std::unique_ptr<std::vector<Event *>> events_;
+  std::unique_ptr<std::vector<Event *>> new_events_;
+  std::vector<int> segments_merged_;
+
+  // messaging
+  SubMaster *sm = nullptr;
+  std::unique_ptr<PubMaster> pm;
+  std::vector<const char*> sockets_;
+  std::unique_ptr<Route> route_;
+  std::unique_ptr<CameraServer> camera_server_;
+  std::atomic<uint32_t> flags_ = REPLAY_FLAG_NONE;
+
+  std::mutex timeline_lock;
+  QFuture<void> timeline_future;
+  std::vector<std::tuple<int, int, TimelineType>> timeline;
+  std::set<cereal::Event::Which> allow_list;
+  std::string car_fingerprint_;
+  float speed_ = 1.0;
+  replayEventFilter event_filter = nullptr;
+  void *filter_opaque = nullptr;
+  int segment_cache_limit = MIN_SEGMENTS_CACHE;
+};

tools/replay/route.h

@@ -0,0 +1,72 @@
+#pragma once
+
+#include <QDateTime>
+#include <QFutureSynchronizer>
+
+#include "tools/replay/framereader.h"
+#include "tools/replay/logreader.h"
+#include "tools/replay/util.h"
+
+struct RouteIdentifier {
+  QString dongle_id;
+  QString timestamp;
+  int segment_id;
+  QString str;
+};
+
+struct SegmentFile {
+  QString rlog;
+  QString qlog;
+  QString road_cam;
+  QString driver_cam;
+  QString wide_road_cam;
+  QString qcamera;
+};
+
+class Route {
+public:
+  Route(const QString &route, const QString &data_dir = {});
+  bool load();
+  inline const QString &name() const { return route_.str; }
+  inline const QDateTime datetime() const { return date_time_; }
+  inline const QString &dir() const { return data_dir_; }
+  inline const RouteIdentifier &identifier() const { return route_; }
+  inline const std::map<int, SegmentFile> &segments() const { return segments_; }
+  inline const SegmentFile &at(int n) { return segments_.at(n); }
+  static RouteIdentifier parseRoute(const QString &str);
+
+protected:
+  bool loadFromLocal();
+  bool loadFromServer();
+  bool loadFromJson(const QString &json);
+  void addFileToSegment(int seg_num, const QString &file);
+  RouteIdentifier route_ = {};
+  QString data_dir_;
+  std::map<int, SegmentFile> segments_;
+  QDateTime date_time_;
+};
+
+class Segment : public QObject {
+  Q_OBJECT
+
+public:
+  Segment(int n, const SegmentFile &files, uint32_t flags, const std::set<cereal::Event::Which> &allow = {});
+  ~Segment();
+  inline bool isLoaded() const { return !loading_ && !abort_; }
+
+  const int seg_num = 0;
+  std::unique_ptr<LogReader> log;
+  std::unique_ptr<FrameReader> frames[MAX_CAMERAS] = {};
+
+signals:
+  void loadFinished(bool success);
+
+protected:
+  void loadFile(int id, const std::string file);
+
+  std::atomic<bool> abort_ = false;
+  std::atomic<int> loading_ = 0;
+  QFutureSynchronizer<void> synchronizer_;
+  uint32_t flags;
+  std::set<cereal::Event::Which> allow;
+};

tools/replay/util.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <atomic>
+#include <functional>
+#include <string>
+
+enum class ReplyMsgType {
+  Info,
+  Debug,
+  Warning,
+  Critical
+};
+
+typedef std::function<void(ReplyMsgType type, const std::string msg)> ReplayMessageHandler;
+void installMessageHandler(ReplayMessageHandler);
+void logMessage(ReplyMsgType type, const char* fmt, ...);
+
+#define rInfo(fmt, ...) ::logMessage(ReplyMsgType::Info, fmt,  ## __VA_ARGS__)
+#define rDebug(fmt, ...) ::logMessage(ReplyMsgType::Debug, fmt,  ## __VA_ARGS__)
+#define rWarning(fmt, ...) ::logMessage(ReplyMsgType::Warning, fmt,  ## __VA_ARGS__)
+#define rError(fmt, ...) ::logMessage(ReplyMsgType::Critical , fmt,  ## __VA_ARGS__)
+
+std::string sha256(const std::string &str);
+void precise_nano_sleep(long sleep_ns);
+std::string decompressBZ2(const std::string &in, std::atomic<bool> *abort = nullptr);
+std::string decompressBZ2(const std::byte *in, size_t in_size, std::atomic<bool> *abort = nullptr);
+std::string getUrlWithoutQuery(const std::string &url);
+size_t getRemoteFileSize(const std::string &url, std::atomic<bool> *abort = nullptr);
+std::string httpGet(const std::string &url, size_t chunk_size = 0, std::atomic<bool> *abort = nullptr);
+
+typedef std::function<void(uint64_t cur, uint64_t total, bool success)> DownloadProgressHandler;
+void installDownloadProgressHandler(DownloadProgressHandler);
+bool httpDownload(const std::string &url, const std::string &file, size_t chunk_size = 0, std::atomic<bool> *abort = nullptr);
+std::string formattedDataSize(size_t size);