openpilot v0.8.13

tools/lib/README.md

@@ -0,0 +1,33 @@
+## LogReader
+
+Route is a class for conviently accessing all the [logs](/selfdrive/loggerd/) from your routes. The LogReader class reads the non-video logs, i.e. rlog.bz2 and qlog.bz2. There's also a matching FrameReader class for reading the videos.
+
+```python
+from tools.lib.route import Route
+from tools.lib.logreader import LogReader
+
+r = Route("4cf7a6ad03080c90|2021-09-29--13-46-36")
+
+# get a list of paths for the route's rlog files
+print(r.log_paths())
+
+# and road camera (fcamera.hevc) files
+print(r.camera_paths())
+
+# setup a LogReader to read the route's first rlog
+lr = LogReader(r.log_paths()[0])
+
+# print out all the messages in the log
+import codecs
+codecs.register_error("strict", codecs.backslashreplace_errors)
+for msg in lr:
+  print(msg)
+
+# setup a LogReader for the route's second qlog
+lr = LogReader(r.log_paths()[1])
+
+# print all the steering angles values from the log
+for msg in lr:
+  if msg.which() == "carState":
+    print(msg.carState.steeringAngleDeg)
+```

tools/lib/api.py

@@ -0,0 +1,34 @@
+import os
+import requests
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
+
+class CommaApi():
+  def __init__(self, token=None):
+    self.session = requests.Session()
+    self.session.headers['User-agent'] = 'OpenpilotTools'
+    if token:
+      self.session.headers['Authorization'] = 'JWT ' + token
+
+  def request(self, method, endpoint, **kwargs):
+    resp = self.session.request(method, API_HOST + '/' + endpoint, **kwargs)
+    resp_json = resp.json()
+    if isinstance(resp_json, dict) and resp_json.get('error'):
+      if resp.status_code in [401, 403]:
+        raise UnauthorizedError('Unauthorized. Authenticate with tools/lib/auth.py')
+
+      e = APIError(str(resp.status_code) + ":" + resp_json.get('description', str(resp_json['error'])))
+      e.status_code = resp.status_code
+      raise e
+    return resp_json
+
+  def get(self, endpoint, **kwargs):
+    return self.request('GET', endpoint, **kwargs)
+
+  def post(self, endpoint, **kwargs):
+    return self.request('POST', endpoint, **kwargs)
+
+class APIError(Exception):
+  pass
+
+class UnauthorizedError(Exception):
+  pass

tools/lib/auth.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+Usage::
+
+  usage: auth.py [-h] [{google,apple,github,jwt}] [jwt]
+
+  Login to your comma account
+
+  positional arguments:
+    {google,apple,github,jwt}
+    jwt
+
+  optional arguments:
+    -h, --help            show this help message and exit
+
+
+Examples::
+
+  ./auth.py  # Log in with google account
+  ./auth.py github  # Log in with GitHub Account
+  ./auth.py jwt ey......hw  # Log in with a JWT from https://jwt.comma.ai, for use in CI
+"""
+
+import argparse
+import sys
+import pprint
+import webbrowser
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Dict
+from urllib.parse import parse_qs, urlencode
+
+from tools.lib.api import APIError, CommaApi, UnauthorizedError
+from tools.lib.auth_config import set_token, get_token
+
+PORT = 3000
+
+
+class ClientRedirectServer(HTTPServer):
+  query_params: Dict[str, Any] = {}
+
+
+class ClientRedirectHandler(BaseHTTPRequestHandler):
+  def do_GET(self):
+    if not self.path.startswith('/auth'):
+      self.send_response(204)
+      return
+
+    query = self.path.split('?', 1)[-1]
+    query = parse_qs(query, keep_blank_values=True)
+    self.server.query_params = query
+
+    self.send_response(200)
+    self.send_header('Content-type', 'text/plain')
+    self.end_headers()
+    self.wfile.write(b'Return to the CLI to continue')
+
+  def log_message(self, format, *args):  # pylint: disable=redefined-builtin
+    pass  # this prevent http server from dumping messages to stdout
+
+
+def auth_redirect_link(method):
+  provider_id = {
+    'google': 'g',
+    'apple': 'a',
+    'github': 'h',
+  }[method]
+
+  params = {
+    'redirect_uri': f"https://api.comma.ai/v2/auth/{provider_id}/redirect/",
+    'state': f'service,localhost:{PORT}',
+  }
+
+  if method == 'google':
+    params.update({
+      'type': 'web_server',
+      'client_id': '45471411055-ornt4svd2miog6dnopve7qtmh5mnu6id.apps.googleusercontent.com',
+      'response_type': 'code',
+      'scope': 'https://www.googleapis.com/auth/userinfo.email',
+      'prompt': 'select_account',
+    })
+    return 'https://accounts.google.com/o/oauth2/auth?' + urlencode(params)
+  elif method == 'github':
+    params.update({
+      'client_id': '28c4ecb54bb7272cb5a4',
+      'scope': 'read:user',
+    })
+    return 'https://github.com/login/oauth/authorize?' + urlencode(params)
+  elif method == 'apple':
+      params.update({
+        'client_id': 'ai.comma.login',
+        'response_type': 'code',
+        'response_mode': 'form_post',
+        'scope': 'name email',
+      })
+      return 'https://appleid.apple.com/auth/authorize?' + urlencode(params)
+  else:
+    raise NotImplementedError(f"no redirect implemented for method {method}")
+
+
+def login(method):
+  oauth_uri = auth_redirect_link(method)
+
+  web_server = ClientRedirectServer(('localhost', PORT), ClientRedirectHandler)
+  print(f'To sign in, use your browser and navigate to {oauth_uri}')
+  webbrowser.open(oauth_uri, new=2)
+
+  while True:
+    web_server.handle_request()
+    if 'code' in web_server.query_params:
+      break
+    elif 'error' in web_server.query_params:
+      print('Authentication Error: "{}". Description: "{}" '.format(
+        web_server.query_params['error'],
+        web_server.query_params.get('error_description')), file=sys.stderr)
+      break
+
+  try:
+    auth_resp = CommaApi().post('v2/auth/', data={'code': web_server.query_params['code'], 'provider': web_server.query_params['provider']})
+    set_token(auth_resp['access_token'])
+  except APIError as e:
+    print(f'Authentication Error: {e}', file=sys.stderr)
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser(description='Login to your comma account')
+  parser.add_argument('method', default='google', const='google', nargs='?', choices=['google', 'apple', 'github', 'jwt'])
+  parser.add_argument('jwt', nargs='?')
+
+  args = parser.parse_args()
+  if args.method == 'jwt':
+    if args.jwt is None:
+      print("method JWT selected, but no JWT was provided")
+      exit(1)
+
+    set_token(args.jwt)
+  else:
+    login(args.method)
+
+  try:
+    me = CommaApi(token=get_token()).get('/v1/me')
+    print("Authenticated!")
+    pprint.pprint(me)
+  except UnauthorizedError:
+    print("Got invalid JWT")
+    exit(1)

tools/lib/auth_config.py

@@ -0,0 +1,34 @@
+import json
+import os
+from common.file_helpers import mkdirs_exists_ok
+from selfdrive.hardware import PC
+
+
+class MissingAuthConfigError(Exception):
+  pass
+
+
+if PC:
+  CONFIG_DIR = os.path.expanduser('~/.comma')
+else:
+  CONFIG_DIR = "/tmp/.comma"
+
+mkdirs_exists_ok(CONFIG_DIR)
+
+
+def get_token():
+  try:
+    with open(os.path.join(CONFIG_DIR, 'auth.json')) as f:
+      auth = json.load(f)
+      return auth['access_token']
+  except Exception:
+    return None
+
+
+def set_token(token):
+  with open(os.path.join(CONFIG_DIR, 'auth.json'), 'w') as f:
+    json.dump({'access_token': token}, f)
+
+
+def clear_token():
+  os.unlink(os.path.join(CONFIG_DIR, 'auth.json'))

tools/lib/bootlog.py

@@ -0,0 +1,48 @@
+import datetime
+import functools
+import re
+
+from tools.lib.auth_config import get_token
+from tools.lib.api import CommaApi
+from tools.lib.helpers import RE, timestamp_to_datetime
+
+
+@functools.total_ordering
+class Bootlog:
+  def __init__(self, url: str):
+    self._url = url
+
+    r = re.search(RE.BOOTLOG_NAME, url)
+    if not r:
+      raise Exception(f"Unable to parse: {url}")
+
+    self._dongle_id = r.group('dongle_id')
+    self._timestamp = r.group('timestamp')
+
+  @property
+  def url(self) -> str:
+    return self._url
+
+  @property
+  def dongle_id(self) -> str:
+    return self._dongle_id
+
+  @property
+  def timestamp(self) -> str:
+    return self._timestamp
+
+  @property
+  def datetime(self) -> datetime.datetime:
+    return timestamp_to_datetime(self._timestamp)
+
+  def __eq__(self, b) -> bool:
+    return self.datetime == b.datetime
+
+  def __lt__(self, b) -> bool:
+    return self.datetime < b.datetime
+
+
+def get_bootlogs(dongle_id: str):
+  api = CommaApi(get_token())
+  r = api.get(f'v1/devices/{dongle_id}/bootlogs')
+  return [Bootlog(b) for b in r]

tools/lib/cache.py

@@ -0,0 +1,15 @@
+import os
+import urllib.parse
+from common.file_helpers import mkdirs_exists_ok
+
+DEFAULT_CACHE_DIR = os.path.expanduser("~/.commacache")
+
+def cache_path_for_file_path(fn, cache_prefix=None):
+  dir_ = os.path.join(DEFAULT_CACHE_DIR, "local")
+  mkdirs_exists_ok(dir_)
+  fn_parsed = urllib.parse.urlparse(fn)
+  if fn_parsed.scheme == '':
+    cache_fn = os.path.abspath(fn).replace("/", "_")
+  else:
+    cache_fn = f'{fn_parsed.hostname}_{fn_parsed.path.replace("/", "_")}'
+  return os.path.join(dir_, cache_fn)

tools/lib/exceptions.py

@@ -0,0 +1,2 @@
+class DataUnreadableError(Exception):
+  pass

tools/lib/filereader.py

@@ -0,0 +1,11 @@
+import os
+from tools.lib.url_file import URLFile
+
+DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.internal/")
+
+def FileReader(fn, debug=False):
+  if fn.startswith("cd:/"):
+    fn = fn.replace("cd:/", DATA_ENDPOINT)
+  if fn.startswith("http://") or fn.startswith("https://"):
+    return URLFile(fn, debug=debug)
+  return open(fn, "rb")

tools/lib/framereader.py

@@ -0,0 +1,581 @@
+# pylint: skip-file
+import json
+import os
+import pickle
+import struct
+import subprocess
+import tempfile
+import threading
+from enum import IntEnum
+from functools import wraps
+
+import numpy as np
+from lru import LRU
+
+import _io
+from tools.lib.cache import cache_path_for_file_path
+from tools.lib.exceptions import DataUnreadableError
+from common.file_helpers import atomic_write_in_dir
+
+from tools.lib.filereader import FileReader
+
+HEVC_SLICE_B = 0
+HEVC_SLICE_P = 1
+HEVC_SLICE_I = 2
+
+
+class GOPReader:
+  def get_gop(self, num):
+    # returns (start_frame_num, num_frames, frames_to_skip, gop_data)
+    raise NotImplementedError
+
+
+class DoNothingContextManager:
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *x):
+    pass
+
+
+class FrameType(IntEnum):
+  raw = 1
+  h265_stream = 2
+
+
+def fingerprint_video(fn):
+  with FileReader(fn) as f:
+    header = f.read(4)
+  if len(header) == 0:
+    raise DataUnreadableError(f"{fn} is empty")
+  elif header == b"\x00\xc0\x12\x00":
+    return FrameType.raw
+  elif header == b"\x00\x00\x00\x01":
+    if 'hevc' in fn:
+      return FrameType.h265_stream
+    else:
+      raise NotImplementedError(fn)
+  else:
+    raise NotImplementedError(fn)
+
+
+def ffprobe(fn, fmt=None):
+  cmd = ["ffprobe",
+         "-v", "quiet",
+         "-print_format", "json",
+         "-show_format", "-show_streams"]
+  if fmt:
+    cmd += ["-f", fmt]
+  cmd += [fn]
+
+  try:
+    ffprobe_output = subprocess.check_output(cmd)
+  except subprocess.CalledProcessError:
+    raise DataUnreadableError(fn)
+
+  return json.loads(ffprobe_output)
+
+
+def vidindex(fn, typ):
+  vidindex_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vidindex")
+  vidindex = os.path.join(vidindex_dir, "vidindex")
+
+  subprocess.check_call(["make"], cwd=vidindex_dir, stdout=open("/dev/null", "w"))
+
+  with tempfile.NamedTemporaryFile() as prefix_f, \
+       tempfile.NamedTemporaryFile() as index_f:
+    try:
+      subprocess.check_call([vidindex, typ, fn, prefix_f.name, index_f.name])
+    except subprocess.CalledProcessError:
+      raise DataUnreadableError(f"vidindex failed on file {fn}")
+    with open(index_f.name, "rb") as f:
+      index = f.read()
+    with open(prefix_f.name, "rb") as f:
+      prefix = f.read()
+
+  index = np.frombuffer(index, np.uint32).reshape(-1, 2)
+
+  assert index[-1, 0] == 0xFFFFFFFF
+  assert index[-1, 1] == os.path.getsize(fn)
+
+  return index, prefix
+
+
+def cache_fn(func):
+  @wraps(func)
+  def cache_inner(fn, *args, **kwargs):
+    if kwargs.pop('no_cache', None):
+      cache_path = None
+    else:
+      cache_prefix = kwargs.pop('cache_prefix', None)
+      cache_path = cache_path_for_file_path(fn, cache_prefix)
+
+    if cache_path and os.path.exists(cache_path):
+      with open(cache_path, "rb") as cache_file:
+        cache_value = pickle.load(cache_file)
+    else:
+      cache_value = func(fn, *args, **kwargs)
+
+      if cache_path:
+        with atomic_write_in_dir(cache_path, mode="wb", overwrite=True) as cache_file:
+          pickle.dump(cache_value, cache_file, -1)
+
+    return cache_value
+
+  return cache_inner
+
+
+@cache_fn
+def index_stream(fn, typ):
+  assert typ in ("hevc", )
+
+  with FileReader(fn) as f:
+    assert os.path.exists(f.name), fn
+    index, prefix = vidindex(f.name, typ)
+    probe = ffprobe(f.name, typ)
+
+  return {
+    'index': index,
+    'global_prefix': prefix,
+    'probe': probe
+  }
+
+
+def index_videos(camera_paths, cache_prefix=None):
+  """Requires that paths in camera_paths are contiguous and of the same type."""
+  if len(camera_paths) < 1:
+    raise ValueError("must provide at least one video to index")
+
+  frame_type = fingerprint_video(camera_paths[0])
+  for fn in camera_paths:
+    index_video(fn, frame_type, cache_prefix)
+
+
+def index_video(fn, frame_type=None, cache_prefix=None):
+  cache_path = cache_path_for_file_path(fn, cache_prefix)
+
+  if os.path.exists(cache_path):
+    return
+
+  if frame_type is None:
+    frame_type = fingerprint_video(fn[0])
+
+  if frame_type == FrameType.h265_stream:
+    index_stream(fn, "hevc", cache_prefix=cache_prefix)
+  else:
+    raise NotImplementedError("Only h265 supported")
+
+
+def get_video_index(fn, frame_type, cache_prefix=None):
+  cache_path = cache_path_for_file_path(fn, cache_prefix)
+
+  if not os.path.exists(cache_path):
+    index_video(fn, frame_type, cache_prefix)
+
+  if not os.path.exists(cache_path):
+    return None
+  with open(cache_path, "rb") as cache_file:
+    return pickle.load(cache_file)
+
+
+def read_file_check_size(f, sz, cookie):
+  buff = bytearray(sz)
+  bytes_read = f.readinto(buff)
+  assert bytes_read == sz, (bytes_read, sz)
+  return buff
+
+
+def rgb24toyuv420(rgb):
+  yuv_from_rgb = np.array([[ 0.299     ,  0.587     ,  0.114      ],
+                           [-0.14714119, -0.28886916,  0.43601035 ],
+                           [ 0.61497538, -0.51496512, -0.10001026 ]])
+  img = np.dot(rgb.reshape(-1, 3), yuv_from_rgb.T).reshape(rgb.shape)
+
+  y_len = img.shape[0] * img.shape[1]
+  uv_len = y_len // 4
+
+  ys = img[:, :, 0]
+  us = (img[::2, ::2, 1] + img[1::2, ::2, 1] + img[::2, 1::2, 1] + img[1::2, 1::2, 1]) / 4 + 128
+  vs = (img[::2, ::2, 2] + img[1::2, ::2, 2] + img[::2, 1::2, 2] + img[1::2, 1::2, 2]) / 4 + 128
+
+  yuv420 = np.empty(y_len + 2 * uv_len, dtype=img.dtype)
+  yuv420[:y_len] = ys.reshape(-1)
+  yuv420[y_len:y_len + uv_len] = us.reshape(-1)
+  yuv420[y_len + uv_len:y_len + 2 * uv_len] = vs.reshape(-1)
+
+  return yuv420.clip(0, 255).astype('uint8')
+
+
+def decompress_video_data(rawdat, vid_fmt, w, h, pix_fmt):
+  # using a tempfile is much faster than proc.communicate for some reason
+
+  with tempfile.TemporaryFile() as tmpf:
+    tmpf.write(rawdat)
+    tmpf.seek(0)
+
+    threads = os.getenv("FFMPEG_THREADS", "0")
+    cuda = os.getenv("FFMPEG_CUDA", "0") == "1"
+    proc = subprocess.Popen(
+      ["ffmpeg",
+       "-threads", threads,
+       "-hwaccel", "none" if not cuda else "cuda",
+       "-c:v", "hevc",
+       "-vsync", "0",
+       "-f", vid_fmt,
+       "-flags2", "showall",
+       "-i", "pipe:0",
+       "-threads", threads,
+       "-f", "rawvideo",
+       "-pix_fmt", pix_fmt,
+       "pipe:1"],
+      stdin=tmpf, stdout=subprocess.PIPE, stderr=open("/dev/null"))
+
+    # dat = proc.communicate()[0]
+    dat = proc.stdout.read()
+    if proc.wait() != 0:
+      raise DataUnreadableError("ffmpeg failed")
+
+  if pix_fmt == "rgb24":
+    ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, h, w, 3)
+  elif pix_fmt == "yuv420p":
+    ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, (h*w*3//2))
+  elif pix_fmt == "yuv444p":
+    ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, 3, h, w)
+  else:
+    raise NotImplementedError
+
+  return ret
+
+
+class BaseFrameReader:
+  # properties: frame_type, frame_count, w, h
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, *args):
+    self.close()
+
+  def close(self):
+    pass
+
+  def get(self, num, count=1, pix_fmt="yuv420p"):
+    raise NotImplementedError
+
+
+def FrameReader(fn, cache_prefix=None, readahead=False, readbehind=False, index_data=None):
+  frame_type = fingerprint_video(fn)
+  if frame_type == FrameType.raw:
+    return RawFrameReader(fn)
+  elif frame_type in (FrameType.h265_stream,):
+    if not index_data:
+      index_data = get_video_index(fn, frame_type, cache_prefix)
+    return StreamFrameReader(fn, frame_type, index_data, readahead=readahead, readbehind=readbehind)
+  else:
+    raise NotImplementedError(frame_type)
+
+
+class RawData:
+  def __init__(self, f):
+    self.f = _io.FileIO(f, 'rb')
+    self.lenn = struct.unpack("I", self.f.read(4))[0]
+    self.count = os.path.getsize(f) / (self.lenn+4)
+
+  def read(self, i):
+    self.f.seek((self.lenn+4)*i + 4)
+    return self.f.read(self.lenn)
+
+
+class RawFrameReader(BaseFrameReader):
+  def __init__(self, fn):
+    # raw camera
+    self.fn = fn
+    self.frame_type = FrameType.raw
+    self.rawfile = RawData(self.fn)
+    self.frame_count = self.rawfile.count
+    self.w, self.h = 640, 480
+
+  def load_and_debayer(self, img):
+    img = np.frombuffer(img, dtype='uint8').reshape(960, 1280)
+    cimg = np.dstack([img[0::2, 1::2], ((img[0::2, 0::2].astype("uint16") + img[1::2, 1::2].astype("uint16")) >> 1).astype("uint8"), img[1::2, 0::2]])
+    return cimg
+
+  def get(self, num, count=1, pix_fmt="yuv420p"):
+    assert self.frame_count is not None
+    assert num+count <= self.frame_count
+
+    if pix_fmt not in ("yuv420p", "rgb24"):
+      raise ValueError(f"Unsupported pixel format {pix_fmt!r}")
+
+    app = []
+    for i in range(num, num+count):
+      dat = self.rawfile.read(i)
+      rgb_dat = self.load_and_debayer(dat)
+      if pix_fmt == "rgb24":
+        app.append(rgb_dat)
+      elif pix_fmt == "yuv420p":
+        app.append(rgb24toyuv420(rgb_dat))
+      else:
+        raise NotImplementedError
+
+    return app
+
+
+class VideoStreamDecompressor:
+  def __init__(self, fn, vid_fmt, w, h, pix_fmt):
+    self.fn = fn
+    self.vid_fmt = vid_fmt
+    self.w = w
+    self.h = h
+    self.pix_fmt = pix_fmt
+
+    if pix_fmt == "yuv420p":
+      self.out_size = w*h*3//2  # yuv420p
+    elif pix_fmt in ("rgb24", "yuv444p"):
+      self.out_size = w*h*3
+    else:
+      raise NotImplementedError
+
+    self.proc = None
+    self.t = threading.Thread(target=self.write_thread)
+    self.t.daemon = True
+
+  def write_thread(self):
+    try:
+      with FileReader(self.fn) as f:
+        while True:
+          r = f.read(1024*1024)
+          if len(r) == 0:
+            break
+          self.proc.stdin.write(r)
+    finally:
+      self.proc.stdin.close()
+
+  def read(self):
+    threads = os.getenv("FFMPEG_THREADS", "0")
+    cuda = os.getenv("FFMPEG_CUDA", "0") == "1"
+    cmd = [
+      "ffmpeg",
+      "-threads", threads,
+      "-hwaccel", "none" if not cuda else "cuda",
+      "-c:v", "hevc",
+      # "-avioflags", "direct",
+      "-analyzeduration", "0",
+      "-probesize", "32",
+      "-flush_packets", "0",
+      # "-fflags", "nobuffer",
+      "-vsync", "0",
+      "-f", self.vid_fmt,
+      "-i", "pipe:0",
+      "-threads", threads,
+      "-f", "rawvideo",
+      "-pix_fmt", self.pix_fmt,
+      "pipe:1"
+    ]
+    self.proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    try:
+      self.t.start()
+
+      while True:
+        dat = self.proc.stdout.read(self.out_size)
+        if len(dat) == 0:
+          break
+        assert len(dat) == self.out_size
+        if self.pix_fmt == "rgb24":
+          ret = np.frombuffer(dat, dtype=np.uint8).reshape((self.h, self.w, 3))
+        elif self.pix_fmt == "yuv420p":
+          ret = np.frombuffer(dat, dtype=np.uint8)
+        elif self.pix_fmt == "yuv444p":
+          ret = np.frombuffer(dat, dtype=np.uint8).reshape((3, self.h, self.w))
+        else:
+          assert False
+        yield ret
+
+      result_code = self.proc.wait()
+      assert result_code == 0, result_code
+    finally:
+      self.proc.kill()
+      self.t.join()
+
+class StreamGOPReader(GOPReader):
+  def __init__(self, fn, frame_type, index_data):
+    assert frame_type == FrameType.h265_stream
+
+    self.fn = fn
+
+    self.frame_type = frame_type
+    self.frame_count = None
+    self.w, self.h = None, None
+
+    self.prefix = None
+    self.index = None
+
+    self.index = index_data['index']
+    self.prefix = index_data['global_prefix']
+    probe = index_data['probe']
+
+    self.prefix_frame_data = None
+    self.num_prefix_frames = 0
+    self.vid_fmt = "hevc"
+
+    i = 0
+    while i < self.index.shape[0] and self.index[i, 0] != HEVC_SLICE_I:
+      i += 1
+    self.first_iframe = i
+
+    assert self.first_iframe == 0
+
+    self.frame_count = len(self.index) - 1
+
+    self.w = probe['streams'][0]['width']
+    self.h = probe['streams'][0]['height']
+
+  def _lookup_gop(self, num):
+    frame_b = num
+    while frame_b > 0 and self.index[frame_b, 0] != HEVC_SLICE_I:
+      frame_b -= 1
+
+    frame_e = num + 1
+    while frame_e < (len(self.index) - 1) and self.index[frame_e, 0] != HEVC_SLICE_I:
+      frame_e += 1
+
+    offset_b = self.index[frame_b, 1]
+    offset_e = self.index[frame_e, 1]
+
+    return (frame_b, frame_e, offset_b, offset_e)
+
+  def get_gop(self, num):
+    frame_b, frame_e, offset_b, offset_e = self._lookup_gop(num)
+    assert frame_b <= num < frame_e
+
+    num_frames = frame_e - frame_b
+
+    with FileReader(self.fn) as f:
+      f.seek(offset_b)
+      rawdat = f.read(offset_e - offset_b)
+
+      if num < self.first_iframe:
+        assert self.prefix_frame_data
+        rawdat = self.prefix_frame_data + rawdat
+
+      rawdat = self.prefix + rawdat
+
+    skip_frames = 0
+    if num < self.first_iframe:
+      skip_frames = self.num_prefix_frames
+
+    return frame_b, num_frames, skip_frames, rawdat
+
+
+class GOPFrameReader(BaseFrameReader):
+  #FrameReader with caching and readahead for formats that are group-of-picture based
+
+  def __init__(self, readahead=False, readbehind=False):
+    self.open_ = True
+
+    self.readahead = readahead
+    self.readbehind = readbehind
+    self.frame_cache = LRU(64)
+
+    if self.readahead:
+      self.cache_lock = threading.RLock()
+      self.readahead_last = None
+      self.readahead_len = 30
+      self.readahead_c = threading.Condition()
+      self.readahead_thread = threading.Thread(target=self._readahead_thread)
+      self.readahead_thread.daemon = True
+      self.readahead_thread.start()
+    else:
+      self.cache_lock = DoNothingContextManager()
+
+  def close(self):
+    if not self.open_:
+      return
+    self.open_ = False
+
+    if self.readahead:
+      self.readahead_c.acquire()
+      self.readahead_c.notify()
+      self.readahead_c.release()
+      self.readahead_thread.join()
+
+  def _readahead_thread(self):
+    while True:
+      self.readahead_c.acquire()
+      try:
+        if not self.open_:
+          break
+        self.readahead_c.wait()
+      finally:
+        self.readahead_c.release()
+      if not self.open_:
+        break
+      assert self.readahead_last
+      num, pix_fmt = self.readahead_last
+
+      if self.readbehind:
+        for k in range(num - 1, max(0, num - self.readahead_len), -1):
+          self._get_one(k, pix_fmt)
+      else:
+        for k in range(num, min(self.frame_count, num + self.readahead_len)):
+          self._get_one(k, pix_fmt)
+
+  def _get_one(self, num, pix_fmt):
+    assert num < self.frame_count
+
+    if (num, pix_fmt) in self.frame_cache:
+      return self.frame_cache[(num, pix_fmt)]
+
+    with self.cache_lock:
+      if (num, pix_fmt) in self.frame_cache:
+        return self.frame_cache[(num, pix_fmt)]
+
+      frame_b, num_frames, skip_frames, rawdat = self.get_gop(num)
+
+      ret = decompress_video_data(rawdat, self.vid_fmt, self.w, self.h, pix_fmt)
+      ret = ret[skip_frames:]
+      assert ret.shape[0] == num_frames
+
+      for i in range(ret.shape[0]):
+        self.frame_cache[(frame_b+i, pix_fmt)] = ret[i]
+
+      return self.frame_cache[(num, pix_fmt)]
+
+  def get(self, num, count=1, pix_fmt="yuv420p"):
+    assert self.frame_count is not None
+
+    if num + count > self.frame_count:
+      raise ValueError(f"{num + count} > {self.frame_count}")
+
+    if pix_fmt not in ("yuv420p", "rgb24", "yuv444p"):
+      raise ValueError(f"Unsupported pixel format {pix_fmt!r}")
+
+    ret = [self._get_one(num + i, pix_fmt) for i in range(count)]
+
+    if self.readahead:
+      self.readahead_last = (num+count, pix_fmt)
+      self.readahead_c.acquire()
+      self.readahead_c.notify()
+      self.readahead_c.release()
+
+    return ret
+
+
+class StreamFrameReader(StreamGOPReader, GOPFrameReader):
+  def __init__(self, fn, frame_type, index_data, readahead=False, readbehind=False):
+    StreamGOPReader.__init__(self, fn, frame_type, index_data)
+    GOPFrameReader.__init__(self, readahead, readbehind)
+
+
+def GOPFrameIterator(gop_reader, pix_fmt):
+  dec = VideoStreamDecompressor(gop_reader.fn, gop_reader.vid_fmt, gop_reader.w, gop_reader.h, pix_fmt)
+  yield from dec.read()
+
+
+def FrameIterator(fn, pix_fmt, **kwargs):
+  fr = FrameReader(fn, **kwargs)
+  if isinstance(fr, GOPReader):
+    yield from GOPFrameIterator(fr, pix_fmt)
+  else:
+    for i in range(fr.frame_count):
+      yield fr.get(i, pix_fmt=pix_fmt)[0]

tools/lib/helpers.py

@@ -0,0 +1,20 @@
+import datetime
+
+TIME_FMT = "%Y-%m-%d--%H-%M-%S"
+
+# regex patterns
+class RE:
+  DONGLE_ID =  r'(?P<dongle_id>[a-z0-9]{16})'
+  TIMESTAMP = r'(?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2})'
+  ROUTE_NAME = r'{}[|_/]{}'.format(DONGLE_ID, TIMESTAMP)
+  SEGMENT_NAME = r'{}(?:--|/)(?P<segment_num>[0-9]+)'.format(ROUTE_NAME)
+  BOOTLOG_NAME = ROUTE_NAME
+
+  EXPLORER_FILE = r'^(?P<segment_name>{})--(?P<file_name>[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME)
+  OP_SEGMENT_DIR = r'^(?P<segment_name>{})$'.format(SEGMENT_NAME)
+
+def timestamp_to_datetime(t: str) -> datetime.datetime:
+  """
+    Convert an openpilot route timestamp to a python datetime
+  """
+  return datetime.datetime.strptime(t, TIME_FMT)

tools/lib/kbhit.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+import sys
+import termios
+import atexit
+from select import select
+
+STDIN_FD = sys.stdin.fileno()
+
+class KBHit:
+  def __init__(self) -> None:
+    ''' Creates a KBHit object that you can call to do various keyboard things.
+    '''
+
+    self.set_kbhit_terminal()
+
+  def set_kbhit_terminal(self) -> None:
+    ''' Save old terminal settings for closure, remove ICANON & ECHO flags.
+    '''
+
+    # Save the terminal settings
+    self.old_term = termios.tcgetattr(STDIN_FD)
+    self.new_term = self.old_term.copy()
+
+    # New terminal setting unbuffered
+    self.new_term[3] &= ~(termios.ICANON | termios.ECHO)  # type: ignore
+    termios.tcsetattr(STDIN_FD, termios.TCSAFLUSH, self.new_term)
+
+    # Support normal-terminal reset at exit
+    atexit.register(self.set_normal_term)
+
+  def set_normal_term(self) -> None:
+    ''' Resets to normal terminal. On Windows this is a no-op.
+    '''
+
+    termios.tcsetattr(STDIN_FD, termios.TCSAFLUSH, self.old_term)
+
+  @staticmethod
+  def getch() -> str:
+    ''' Returns a keyboard character after kbhit() has been called.
+      Should not be called in the same program as getarrow().
+    '''
+    return sys.stdin.read(1)
+
+  @staticmethod
+  def getarrow() -> int:
+    ''' Returns an arrow-key code after kbhit() has been called. Codes are
+    0 : up
+    1 : right
+    2 : down
+    3 : left
+    Should not be called in the same program as getch().
+    '''
+
+    c = sys.stdin.read(3)[2]
+    vals = [65, 67, 66, 68]
+
+    return vals.index(ord(c))
+
+  @staticmethod
+  def kbhit():
+    ''' Returns True if keyboard character was hit, False otherwise.
+    '''
+    return select([sys.stdin], [], [], 0)[0] != []
+
+
+# Test
+if __name__ == "__main__":
+
+  kb = KBHit()
+
+  print('Hit any key, or ESC to exit')
+
+  while True:
+
+    if kb.kbhit():
+      c = kb.getch()
+      if c == '\x1b':  # ESC
+        break
+      print(c)
+
+  kb.set_normal_term()

tools/lib/logreader.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+import os
+import sys
+import bz2
+import urllib.parse
+import capnp
+
+from tools.lib.filereader import FileReader
+from cereal import log as capnp_log
+
+# this is an iterator itself, and uses private variables from LogReader
+class MultiLogIterator:
+  def __init__(self, log_paths, sort_by_time=False):
+    self._log_paths = log_paths
+    self.sort_by_time = sort_by_time
+
+    self._first_log_idx = next(i for i in range(len(log_paths)) if log_paths[i] is not None)
+    self._current_log = self._first_log_idx
+    self._idx = 0
+    self._log_readers = [None]*len(log_paths)
+    self.start_time = self._log_reader(self._first_log_idx)._ts[0]
+
+  def _log_reader(self, i):
+    if self._log_readers[i] is None and self._log_paths[i] is not None:
+      log_path = self._log_paths[i]
+      self._log_readers[i] = LogReader(log_path, sort_by_time=self.sort_by_time)
+
+    return self._log_readers[i]
+
+  def __iter__(self):
+    return self
+
+  def _inc(self):
+    lr = self._log_reader(self._current_log)
+    if self._idx < len(lr._ents)-1:
+      self._idx += 1
+    else:
+      self._idx = 0
+      self._current_log = next(i for i in range(self._current_log + 1, len(self._log_readers) + 1)
+                               if i == len(self._log_readers) or self._log_paths[i] is not None)
+      if self._current_log == len(self._log_readers):
+        raise StopIteration
+
+  def __next__(self):
+    while 1:
+      lr = self._log_reader(self._current_log)
+      ret = lr._ents[self._idx]
+      self._inc()
+      return ret
+
+  def tell(self):
+    # returns seconds from start of log
+    return (self._log_reader(self._current_log)._ts[self._idx] - self.start_time) * 1e-9
+
+  def seek(self, ts):
+    # seek to nearest minute
+    minute = int(ts/60)
+    if minute >= len(self._log_paths) or self._log_paths[minute] is None:
+      return False
+
+    self._current_log = minute
+
+    # HACK: O(n) seek afterward
+    self._idx = 0
+    while self.tell() < ts:
+      self._inc()
+    return True
+
+
+class LogReader:
+  def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False):
+    data_version = None
+    _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
+    with FileReader(fn) as f:
+      dat = f.read()
+
+    if ext == "":
+      # old rlogs weren't bz2 compressed
+      ents = capnp_log.Event.read_multiple_bytes(dat)
+    elif ext == ".bz2":
+      dat = bz2.decompress(dat)
+      ents = capnp_log.Event.read_multiple_bytes(dat)
+    else:
+      raise Exception(f"unknown extension {ext}")
+
+    self._ents = list(sorted(ents, key=lambda x: x.logMonoTime) if sort_by_time else ents)
+    self._ts = [x.logMonoTime for x in self._ents]
+    self.data_version = data_version
+    self._only_union_types = only_union_types
+
+  def __iter__(self):
+    for ent in self._ents:
+      if self._only_union_types:
+        try:
+          ent.which()
+          yield ent
+        except capnp.lib.capnp.KjException:
+          pass
+      else:
+        yield ent
+
+if __name__ == "__main__":
+  import codecs
+  # capnproto <= 0.8.0 throws errors converting byte data to string
+  # below line catches those errors and replaces the bytes with \x__
+  codecs.register_error("strict", codecs.backslashreplace_errors)
+  log_path = sys.argv[1]
+  lr = LogReader(log_path, sort_by_time=True)
+  for msg in lr:
+    print(msg)

tools/lib/robust_logreader.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import os
+import bz2
+import urllib.parse
+import subprocess
+import tqdm
+import glob
+from tempfile import TemporaryDirectory
+import capnp
+
+from tools.lib.logreader import FileReader, LogReader
+from cereal import log as capnp_log
+
+
+class RobustLogReader(LogReader):
+  def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False):  # pylint: disable=super-init-not-called
+    data_version = None
+    _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
+    with FileReader(fn) as f:
+      dat = f.read()
+
+    if ext == "":
+      pass
+    elif ext == ".bz2":
+      try:
+        dat = bz2.decompress(dat)
+      except ValueError:
+        print("Failed to decompress, falling back to bzip2recover")
+        with TemporaryDirectory() as directory:
+          # Run bzip2recovery on log
+          with open(os.path.join(directory, 'out.bz2'), 'wb') as f:
+            f.write(dat)
+          subprocess.check_call(["bzip2recover", "out.bz2"], cwd=directory)
+
+          # Decompress and concatenate parts
+          dat = b""
+          for n in sorted(glob.glob(f"{directory}/rec*.bz2")):
+            print(f"Decompressing {n}")
+            with open(n, 'rb') as f:
+              dat += bz2.decompress(f.read())
+    else:
+      raise Exception(f"unknown extension {ext}")
+
+    progress = None
+    while True:
+      try:
+        ents = capnp_log.Event.read_multiple_bytes(dat)
+        self._ents = list(sorted(ents, key=lambda x: x.logMonoTime) if sort_by_time else ents)
+        break
+      except capnp.lib.capnp.KjException:
+        if progress is None:
+          progress = tqdm.tqdm(total=len(dat))
+
+        # Cut off bytes at the end until capnp is able to read
+        dat = dat[:-1]
+        progress.update(1)
+
+    self._ts = [x.logMonoTime for x in self._ents]
+    self.data_version = data_version
+    self._only_union_types = only_union_types

tools/lib/route.py

@@ -0,0 +1,223 @@
+import os
+import re
+from urllib.parse import urlparse
+from collections import defaultdict
+from itertools import chain
+
+from tools.lib.auth_config import get_token
+from tools.lib.api import CommaApi
+from tools.lib.helpers import RE
+
+QLOG_FILENAMES = ['qlog.bz2']
+QCAMERA_FILENAMES = ['qcamera.ts']
+LOG_FILENAMES = ['rlog.bz2', 'raw_log.bz2']
+CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc']
+DCAMERA_FILENAMES = ['dcamera.hevc']
+ECAMERA_FILENAMES = ['ecamera.hevc']
+
+class Route:
+  def __init__(self, name, data_dir=None):
+    self._name = RouteName(name)
+    self.files = None
+    if data_dir is not None:
+      self._segments = self._get_segments_local(data_dir)
+    else:
+      self._segments = self._get_segments_remote()
+    self.max_seg_number = self._segments[-1].name.segment_num
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def segments(self):
+    return self._segments
+
+  def log_paths(self):
+    log_path_by_seg_num = {s.name.segment_num: s.log_path for s in self._segments}
+    return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def qlog_paths(self):
+    qlog_path_by_seg_num = {s.name.segment_num: s.qlog_path for s in self._segments}
+    return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def camera_paths(self):
+    camera_path_by_seg_num = {s.name.segment_num: s.camera_path for s in self._segments}
+    return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def dcamera_paths(self):
+    dcamera_path_by_seg_num = {s.name.segment_num: s.dcamera_path for s in self._segments}
+    return [dcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def ecamera_paths(self):
+    ecamera_path_by_seg_num = {s.name.segment_num: s.ecamera_path for s in self._segments}
+    return [ecamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def qcamera_paths(self):
+    qcamera_path_by_seg_num = {s.name.segment_num: s.qcamera_path for s in self._segments}
+    return [qcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  # TODO: refactor this, it's super repetitive
+  def _get_segments_remote(self):
+    api = CommaApi(get_token())
+    route_files = api.get('v1/route/' + self.name.canonical_name + '/files')
+    self.files = list(chain.from_iterable(route_files.values()))
+
+    segments = {}
+    for url in self.files:
+      _, dongle_id, time_str, segment_num, fn = urlparse(url).path.rsplit('/', maxsplit=4)
+      segment_name = f'{dongle_id}|{time_str}--{segment_num}'
+      if segments.get(segment_name):
+        segments[segment_name] = Segment(
+          segment_name,
+          url if fn in LOG_FILENAMES else segments[segment_name].log_path,
+          url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path,
+          url if fn in CAMERA_FILENAMES else segments[segment_name].camera_path,
+          url if fn in DCAMERA_FILENAMES else segments[segment_name].dcamera_path,
+          url if fn in ECAMERA_FILENAMES else segments[segment_name].ecamera_path,
+          url if fn in QCAMERA_FILENAMES else segments[segment_name].qcamera_path,
+        )
+      else:
+        segments[segment_name] = Segment(
+          segment_name,
+          url if fn in LOG_FILENAMES else None,
+          url if fn in QLOG_FILENAMES else None,
+          url if fn in CAMERA_FILENAMES else None,
+          url if fn in DCAMERA_FILENAMES else None,
+          url if fn in ECAMERA_FILENAMES else None,
+          url if fn in QCAMERA_FILENAMES else None,
+        )
+
+    return sorted(segments.values(), key=lambda seg: seg.name.segment_num)
+
+  def _get_segments_local(self, data_dir):
+    files = os.listdir(data_dir)
+    segment_files = defaultdict(list)
+
+    for f in files:
+      fullpath = os.path.join(data_dir, f)
+      explorer_match = re.match(RE.EXPLORER_FILE, f)
+      op_match = re.match(RE.OP_SEGMENT_DIR, f)
+
+      if explorer_match:
+        segment_name = explorer_match.group('segment_name')
+        fn = explorer_match.group('file_name')
+        if segment_name.replace('_', '|').startswith(self.name.canonical_name):
+          segment_files[segment_name].append((fullpath, fn))
+      elif op_match and os.path.isdir(fullpath):
+        segment_name = op_match.group('segment_name')
+        if segment_name.startswith(self.name.canonical_name):
+          for seg_f in os.listdir(fullpath):
+            segment_files[segment_name].append((os.path.join(fullpath, seg_f), seg_f))
+      elif f == self.name.canonical_name:
+        for seg_num in os.listdir(fullpath):
+          if not seg_num.isdigit():
+            continue
+
+          segment_name = f'{self.name.canonical_name}--{seg_num}'
+          for seg_f in os.listdir(os.path.join(fullpath, seg_num)):
+            segment_files[segment_name].append((os.path.join(fullpath, seg_num, seg_f), seg_f))
+
+    segments = []
+    for segment, files in segment_files.items():
+
+      try:
+        log_path = next(path for path, filename in files if filename in LOG_FILENAMES)
+      except StopIteration:
+        log_path = None
+
+      try:
+        qlog_path = next(path for path, filename in files if filename in QLOG_FILENAMES)
+      except StopIteration:
+        qlog_path = None
+
+      try:
+        camera_path = next(path for path, filename in files if filename in CAMERA_FILENAMES)
+      except StopIteration:
+        camera_path = None
+
+      try:
+        dcamera_path = next(path for path, filename in files if filename in DCAMERA_FILENAMES)
+      except StopIteration:
+        dcamera_path = None
+
+      try:
+        ecamera_path = next(path for path, filename in files if filename in ECAMERA_FILENAMES)
+      except StopIteration:
+        ecamera_path = None
+
+      try:
+        qcamera_path = next(path for path, filename in files if filename in QCAMERA_FILENAMES)
+      except StopIteration:
+        qcamera_path = None
+
+      segments.append(Segment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path))
+
+    if len(segments) == 0:
+      raise ValueError(f'Could not find segments for route {self.name.canonical_name} in data directory {data_dir}')
+    return sorted(segments, key=lambda seg: seg.name.segment_num)
+
+class Segment:
+  def __init__(self, name, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path):
+    self._name = SegmentName(name)
+    self.log_path = log_path
+    self.qlog_path = qlog_path
+    self.camera_path = camera_path
+    self.dcamera_path = dcamera_path
+    self.ecamera_path = ecamera_path
+    self.qcamera_path = qcamera_path
+
+  @property
+  def name(self):
+    return self._name
+
+class RouteName:
+  def __init__(self, name_str: str):
+    self._name_str = name_str
+    delim = next(c for c in self._name_str if c in ("|", "/"))
+    self._dongle_id, self._time_str = self._name_str.split(delim)
+
+    assert len(self._dongle_id) == 16, self._name_str
+    assert len(self._time_str) == 20, self._name_str
+    self._canonical_name = f"{self._dongle_id}|{self._time_str}"
+
+  @property
+  def canonical_name(self) -> str: return self._canonical_name
+
+  @property
+  def dongle_id(self) -> str: return self._dongle_id
+
+  @property
+  def time_str(self) -> str: return self._time_str
+
+  def __str__(self) -> str: return self._canonical_name
+
+class SegmentName:
+  # TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances
+  # of this class instead of manually constructing a segment name (use canonical_name prop instead)
+  def __init__(self, name_str: str, allow_route_name=False):
+    self._name_str = name_str
+    seg_num_delim = "--" if self._name_str.count("--") == 2 else "/"
+    name_parts = self._name_str.rsplit(seg_num_delim, 1)
+    if allow_route_name and len(name_parts) == 1:
+      name_parts.append("-1") # no segment number
+    self._route_name = RouteName(name_parts[0])
+    self._num = int(name_parts[1])
+    self._canonical_name = f"{self._route_name._dongle_id}|{self._route_name._time_str}--{self._num}"
+
+  @property
+  def canonical_name(self) -> str: return self._canonical_name
+
+  @property
+  def dongle_id(self) -> str: return self._route_name.dongle_id
+
+  @property
+  def time_str(self) -> str: return self._route_name.time_str
+
+  @property
+  def segment_num(self) -> int: return self._num
+
+  @property
+  def route_name(self) -> RouteName: return self._route_name
+
+  def __str__(self) -> str: return self._canonical_name

tools/lib/tests/test_caching.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import unittest
+
+os.environ["COMMA_CACHE"] = "/tmp/__test_cache__"
+from tools.lib.url_file import URLFile, CACHE_DIR
+
+
+class TestFileDownload(unittest.TestCase):
+
+  def compare_loads(self, url, start=0, length=None):
+    """Compares range between cached and non cached version"""
+    shutil.rmtree(CACHE_DIR)
+
+    file_cached = URLFile(url, cache=True)
+    file_downloaded = URLFile(url, cache=False)
+
+    file_cached.seek(start)
+    file_downloaded.seek(start)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertLessEqual(length + start if length is not None else 0, file_downloaded.get_length())
+
+    response_cached = file_cached.read(ll=length)
+    response_downloaded = file_downloaded.read(ll=length)
+
+    self.assertEqual(response_cached, response_downloaded)
+
+    # Now test with cache in place
+    file_cached = URLFile(url, cache=True)
+    file_cached.seek(start)
+    response_cached = file_cached.read(ll=length)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertEqual(response_cached, response_downloaded)
+
+  def test_small_file(self):
+    # Make sure we don't force cache
+    os.environ["FILEREADER_CACHE"] = "0"
+    small_file_url = "https://raw.githubusercontent.com/commaai/openpilot/master/docs/SAFETY.md"
+    #  If you want large file to be larger than a chunk
+    #  large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/fcamera.hevc"
+
+    #  Load full small file
+    self.compare_loads(small_file_url)
+
+    file_small = URLFile(small_file_url)
+    length = file_small.get_length()
+
+    self.compare_loads(small_file_url, length - 100, 100)
+    self.compare_loads(small_file_url, 50, 100)
+
+    #  Load small file 100 bytes at a time
+    for i in range(length // 100):
+      self.compare_loads(small_file_url, 100 * i, 100)
+
+  def test_large_file(self):
+    large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+    #  Load the end 100 bytes of both files
+    file_large = URLFile(large_file_url)
+    length = file_large.get_length()
+
+    self.compare_loads(large_file_url, length - 100, 100)
+    self.compare_loads(large_file_url)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/lib/tests/test_readers.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import unittest
+import requests
+import tempfile
+
+from collections import defaultdict
+import numpy as np
+from tools.lib.framereader import FrameReader
+from tools.lib.logreader import LogReader
+
+
+class TestReaders(unittest.TestCase):
+  @unittest.skip("skip for bandwith reasons")
+  def test_logreader(self):
+    def _check_data(lr):
+      hist = defaultdict(int)
+      for l in lr:
+        hist[l.which()] += 1
+
+      self.assertEqual(hist['carControl'], 6000)
+      self.assertEqual(hist['logMessage'], 6857)
+
+    with tempfile.NamedTemporaryFile(suffix=".bz2") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true")
+      fp.write(r.content)
+      fp.flush()
+
+      lr_file = LogReader(fp.name)
+      _check_data(lr_file)
+
+    lr_url = LogReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true")
+    _check_data(lr_url)
+
+  @unittest.skip("skip for bandwith reasons")
+  def test_framereader(self):
+    def _check_data(f):
+      self.assertEqual(f.frame_count, 1200)
+      self.assertEqual(f.w, 1164)
+      self.assertEqual(f.h, 874)
+
+      frame_first_30 = f.get(0, 30)
+      self.assertEqual(len(frame_first_30), 30)
+
+      print(frame_first_30[15])
+
+      print("frame_0")
+      frame_0 = f.get(0, 1)
+      frame_15 = f.get(15, 1)
+
+      print(frame_15[0])
+
+      assert np.all(frame_first_30[0] == frame_0[0])
+      assert np.all(frame_first_30[15] == frame_15[0])
+
+    with tempfile.NamedTemporaryFile(suffix=".hevc") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true")
+      fp.write(r.content)
+      fp.flush()
+
+      fr_file = FrameReader(fp.name)
+      _check_data(fr_file)
+
+    fr_url = FrameReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true")
+    _check_data(fr_url)
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/url_file.py

@@ -0,0 +1,198 @@
+# pylint: skip-file
+
+import os
+import time
+import tempfile
+import threading
+import urllib.parse
+import pycurl
+from hashlib import sha256
+from io import BytesIO
+from tenacity import retry, wait_random_exponential, stop_after_attempt
+from common.file_helpers import mkdirs_exists_ok, atomic_write_in_dir
+#  Cache chunk size
+K = 1000
+CHUNK_SIZE = 1000 * K
+
+CACHE_DIR = os.environ.get("COMMA_CACHE", "/tmp/comma_download_cache/")
+
+
+def hash_256(link):
+  hsh = str(sha256((link.split("?")[0]).encode('utf-8')).hexdigest())
+  return hsh
+
+
+class URLFile:
+  _tlocal = threading.local()
+
+  def __init__(self, url, debug=False, cache=None):
+    self._url = url
+    self._pos = 0
+    self._length = None
+    self._local_file = None
+    self._debug = debug
+    #  True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input
+    self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0"))
+    if cache is not None:
+      self._force_download = not cache
+
+    try:
+      self._curl = self._tlocal.curl
+    except AttributeError:
+      self._curl = self._tlocal.curl = pycurl.Curl()
+    mkdirs_exists_ok(CACHE_DIR)
+
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    if self._local_file is not None:
+      os.remove(self._local_file.name)
+      self._local_file.close()
+      self._local_file = None
+
+  @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
+  def get_length_online(self):
+    c = self._curl
+    c.reset()
+    c.setopt(pycurl.NOSIGNAL, 1)
+    c.setopt(pycurl.TIMEOUT_MS, 500000)
+    c.setopt(pycurl.FOLLOWLOCATION, True)
+    c.setopt(pycurl.URL, self._url)
+    c.setopt(c.NOBODY, 1)
+    c.perform()
+    length = int(c.getinfo(c.CONTENT_LENGTH_DOWNLOAD))
+    c.reset()
+    return length
+
+  def get_length(self):
+    if self._length is not None:
+      return self._length
+    file_length_path = os.path.join(CACHE_DIR, hash_256(self._url) + "_length")
+    if os.path.exists(file_length_path) and not self._force_download:
+      with open(file_length_path) as file_length:
+          content = file_length.read()
+          self._length = int(content)
+          return self._length
+
+    self._length = self.get_length_online()
+    if not self._force_download:
+      with atomic_write_in_dir(file_length_path, mode="w") as file_length:
+        file_length.write(str(self._length))
+    return self._length
+
+  def read(self, ll=None):
+    if self._force_download:
+      return self.read_aux(ll=ll)
+
+    file_begin = self._pos
+    file_end = self._pos + ll if ll is not None else self.get_length()
+    #  We have to align with chunks we store. Position is the begginiing of the latest chunk that starts before or at our file
+    position = (file_begin // CHUNK_SIZE) * CHUNK_SIZE
+    response = b""
+    while True:
+      self._pos = position
+      chunk_number = self._pos / CHUNK_SIZE
+      file_name = hash_256(self._url) + "_" + str(chunk_number)
+      full_path = os.path.join(CACHE_DIR, str(file_name))
+      data = None
+      #  If we don't have a file, download it
+      if not os.path.exists(full_path):
+        data = self.read_aux(ll=CHUNK_SIZE)
+        with atomic_write_in_dir(full_path, mode="wb") as new_cached_file:
+          new_cached_file.write(data)
+      else:
+        with open(full_path, "rb") as cached_file:
+          data = cached_file.read()
+
+      response += data[max(0, file_begin - position): min(CHUNK_SIZE, file_end - position)]
+
+      position += CHUNK_SIZE
+      if position >= file_end:
+        self._pos = file_end
+        return response
+
+  @retry(wait=wait_random_exponential(multiplier=1, max=5), stop=stop_after_attempt(3), reraise=True)
+  def read_aux(self, ll=None):
+    download_range = False
+    headers = ["Connection: keep-alive"]
+    if self._pos != 0 or ll is not None:
+      if ll is None:
+        end = self.get_length() - 1
+      else:
+        end = min(self._pos + ll, self.get_length()) - 1
+      if self._pos >= end:
+        return b""
+      headers.append(f"Range: bytes={self._pos}-{end}")
+      download_range = True
+
+    dats = BytesIO()
+    c = self._curl
+    c.setopt(pycurl.URL, self._url)
+    c.setopt(pycurl.WRITEDATA, dats)
+    c.setopt(pycurl.NOSIGNAL, 1)
+    c.setopt(pycurl.TIMEOUT_MS, 500000)
+    c.setopt(pycurl.HTTPHEADER, headers)
+    c.setopt(pycurl.FOLLOWLOCATION, True)
+
+    if self._debug:
+      print("downloading", self._url)
+
+      def header(x):
+        if b'MISS' in x:
+          print(x.strip())
+
+      c.setopt(pycurl.HEADERFUNCTION, header)
+
+      def test(debug_type, debug_msg):
+       print("  debug(%d): %s" % (debug_type, debug_msg.strip()))
+
+      c.setopt(pycurl.VERBOSE, 1)
+      c.setopt(pycurl.DEBUGFUNCTION, test)
+      t1 = time.time()
+
+    c.perform()
+
+    if self._debug:
+      t2 = time.time()
+      if t2 - t1 > 0.1:
+        print(f"get {self._url} {headers!r} {t2 - t1:.f} slow")
+
+    response_code = c.getinfo(pycurl.RESPONSE_CODE)
+    if response_code == 416:  # Requested Range Not Satisfiable
+      raise Exception(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+    if download_range and response_code != 206:  # Partial Content
+      raise Exception(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+    if (not download_range) and response_code != 200:  # OK
+      raise Exception(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+
+    ret = dats.getvalue()
+    self._pos += len(ret)
+    return ret
+
+  def seek(self, pos):
+    self._pos = pos
+
+  @property
+  def name(self):
+    """Returns a local path to file with the URLFile's contents.
+
+       This can be used to interface with modules that require local files.
+    """
+    if self._local_file is None:
+      _, ext = os.path.splitext(urllib.parse.urlparse(self._url).path)
+      local_fd, local_path = tempfile.mkstemp(suffix=ext)
+      try:
+        os.write(local_fd, self.read())
+        local_file = open(local_path, "rb")
+      except Exception:
+        os.remove(local_path)
+        raise
+      finally:
+        os.close(local_fd)
+
+      self._local_file = local_file
+      self.read = self._local_file.read
+      self.seek = self._local_file.seek
+
+    return self._local_file.name

tools/lib/vidindex/.gitignore

@@ -0,0 +1 @@
+vidindex

tools/lib/vidindex/Makefile

@@ -0,0 +1,6 @@
+CC := gcc
+
+vidindex: bitstream.c bitstream.h vidindex.c
+	$(eval $@_TMP := $(shell mktemp))
+	$(CC) -std=c99 bitstream.c vidindex.c -o $($@_TMP)
+	mv $($@_TMP) $@

tools/lib/vidindex/bitstream.c

@@ -0,0 +1,118 @@
+#include <stdbool.h>
+#include <assert.h>
+
+#include "bitstream.h"
+
+static const uint32_t BS_MASKS[33] = {
+    0,           0x1L,        0x3L,       0x7L,       0xFL,       0x1FL,
+    0x3FL,       0x7FL,       0xFFL,      0x1FFL,     0x3FFL,     0x7FFL,
+    0xFFFL,      0x1FFFL,     0x3FFFL,    0x7FFFL,    0xFFFFL,    0x1FFFFL,
+    0x3FFFFL,    0x7FFFFL,    0xFFFFFL,   0x1FFFFFL,  0x3FFFFFL,  0x7FFFFFL,
+    0xFFFFFFL,   0x1FFFFFFL,  0x3FFFFFFL, 0x7FFFFFFL, 0xFFFFFFFL, 0x1FFFFFFFL,
+    0x3FFFFFFFL, 0x7FFFFFFFL, 0xFFFFFFFFL};
+
+void bs_init(struct bitstream* bs, const uint8_t* buffer, size_t input_size) {
+  bs->buffer_ptr = buffer;
+  bs->buffer_end = buffer + input_size;
+  bs->value = 0;
+  bs->pos = 0;
+  bs->shift = 8;
+  bs->size = input_size * 8;
+}
+
+uint32_t bs_get(struct bitstream* bs, int n) {
+  if (n > 32)
+    return 0;
+
+  bs->pos += n;
+  bs->shift += n;
+  while (bs->shift > 8) {
+    if (bs->buffer_ptr < bs->buffer_end) {
+      bs->value <<= 8;
+      bs->value |= *bs->buffer_ptr++;
+      bs->shift -= 8;
+    } else {
+      bs_seek(bs, bs->pos - n);
+      return 0;
+      // bs->value <<= 8;
+      // bs->shift -= 8;
+    }
+  }
+  return (bs->value >> (8 - bs->shift)) & BS_MASKS[n];
+}
+
+void bs_seek(struct bitstream* bs, size_t new_pos) {
+  bs->pos = (new_pos / 32) * 32;
+  bs->shift = 8;
+  bs->value = 0;
+  bs_get(bs, new_pos % 32);
+}
+
+uint32_t bs_peek(struct bitstream* bs, int n) {
+  struct bitstream bak = *bs;
+  return bs_get(&bak, n);
+}
+
+size_t bs_remain(struct bitstream* bs) {
+  return bs->size - bs->pos;
+}
+
+int bs_eof(struct bitstream* bs) {
+  return bs_remain(bs) == 0;
+}
+
+uint32_t bs_ue(struct bitstream* bs) {
+  static const uint8_t exp_golomb_bits[256] = {
+      8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3,
+      3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+      2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+      1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  };
+  uint32_t bits, read = 0;
+  int bits_left;
+  uint8_t coded;
+  int done = 0;
+  bits = 0;
+  // we want to read 8 bits at a time - if we don't have 8 bits,
+  // read what's left, and shift.  The exp_golomb_bits calc remains the
+  // same.
+  while (!done) {
+    bits_left = bs_remain(bs);
+    if (bits_left < 8) {
+      read = bs_peek(bs, bits_left) << (8 - bits_left);
+      done = 1;
+    } else {
+      read = bs_peek(bs, 8);
+      if (read == 0) {
+        bs_get(bs, 8);
+        bits += 8;
+      } else {
+        done = 1;
+      }
+    }
+  }
+  coded = exp_golomb_bits[read];
+  bs_get(bs, coded);
+  bits += coded;
+
+  //  printf("ue - bits %d\n", bits);
+  return bs_get(bs, bits + 1) - 1;
+}
+
+int32_t bs_se(struct bitstream* bs) {
+  uint32_t ret;
+  ret = bs_ue(bs);
+  if ((ret & 0x1) == 0) {
+    ret >>= 1;
+    int32_t temp = 0 - ret;
+    return temp;
+  }
+  return (ret + 1) >> 1;
+}

tools/lib/vidindex/bitstream.h

@@ -0,0 +1,26 @@
+#ifndef bitstream_H
+#define bitstream_H
+
+
+#include <stddef.h>
+#include <stdint.h>
+
+struct bitstream {
+  const uint8_t *buffer_ptr;
+  const uint8_t *buffer_end;
+  uint64_t value;
+  uint32_t pos;
+  uint32_t shift;
+  size_t size;
+};
+
+void bs_init(struct bitstream *bs, const uint8_t *buffer, size_t input_size);
+void bs_seek(struct bitstream *bs, size_t new_pos);
+uint32_t bs_get(struct bitstream *bs, int n);
+uint32_t bs_peek(struct bitstream *bs, int n);
+size_t bs_remain(struct bitstream *bs);
+int bs_eof(struct bitstream *bs);
+uint32_t bs_ue(struct bitstream *bs);
+int32_t bs_se(struct bitstream *bs);
+
+#endif

tools/lib/vidindex/vidindex.c

@@ -0,0 +1,307 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <assert.h>
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#include "bitstream.h"
+
+#define START_CODE 0x000001
+
+static uint32_t read24be(const uint8_t* ptr) {
+    return (ptr[0] << 16) | (ptr[1] << 8) | ptr[2];
+}
+static void write32le(FILE *of, uint32_t v) {
+  uint8_t va[4] = {
+    v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff
+  };
+  fwrite(va, 1, sizeof(va), of);
+}
+
+// Table 7-1
+enum hevc_nal_type {
+  HEVC_NAL_TYPE_TRAIL_N = 0,
+  HEVC_NAL_TYPE_TRAIL_R = 1,
+  HEVC_NAL_TYPE_TSA_N = 2,
+  HEVC_NAL_TYPE_TSA_R = 3,
+  HEVC_NAL_TYPE_STSA_N = 4,
+  HEVC_NAL_TYPE_STSA_R = 5,
+  HEVC_NAL_TYPE_RADL_N = 6,
+  HEVC_NAL_TYPE_RADL_R = 7,
+  HEVC_NAL_TYPE_RASL_N = 8,
+  HEVC_NAL_TYPE_RASL_R = 9,
+  HEVC_NAL_TYPE_BLA_W_LP = 16,
+  HEVC_NAL_TYPE_BLA_W_RADL = 17,
+  HEVC_NAL_TYPE_BLA_N_LP = 18,
+  HEVC_NAL_TYPE_IDR_W_RADL = 19,
+  HEVC_NAL_TYPE_IDR_N_LP = 20,
+  HEVC_NAL_TYPE_CRA_NUT = 21,
+  HEVC_NAL_TYPE_RSV_IRAP_VCL23 = 23,
+  HEVC_NAL_TYPE_VPS_NUT = 32,
+  HEVC_NAL_TYPE_SPS_NUT = 33,
+  HEVC_NAL_TYPE_PPS_NUT = 34,
+  HEVC_NAL_TYPE_AUD_NUT = 35,
+  HEVC_NAL_TYPE_EOS_NUT = 36,
+  HEVC_NAL_TYPE_EOB_NUT = 37,
+  HEVC_NAL_TYPE_FD_NUT = 38,
+  HEVC_NAL_TYPE_PREFIX_SEI_NUT = 39,
+  HEVC_NAL_TYPE_SUFFIX_SEI_NUT = 40,
+};
+
+// Table 7-7
+enum hevc_slice_type {
+  HEVC_SLICE_B = 0,
+  HEVC_SLICE_P = 1,
+  HEVC_SLICE_I = 2,
+};
+
+static void hevc_index(const uint8_t *data, size_t file_size, FILE *of_prefix, FILE *of_index) {
+  const uint8_t* ptr = data;
+  const uint8_t* ptr_end = data + file_size;
+
+  assert(ptr[0] == 0);
+  ptr++;
+  assert(read24be(ptr) == START_CODE);
+
+  // pps. ignore for now
+  uint32_t num_extra_slice_header_bits = 0;
+  uint32_t dependent_slice_segments_enabled_flag = 0;
+
+  while (ptr < ptr_end) {
+    const uint8_t* next = ptr+1;
+    for (; next < ptr_end-4; next++) {
+      if (read24be(next) == START_CODE) break;
+    }
+    size_t nal_size = next - ptr;
+    if (nal_size < 6) {
+      break;
+    }
+
+    {
+      struct bitstream bs = {0};
+      bs_init(&bs, ptr, nal_size);
+
+      uint32_t start_code = bs_get(&bs, 24);
+      assert(start_code == 0x000001);
+
+      // nal_unit_header
+      uint32_t forbidden_zero_bit = bs_get(&bs, 1);
+      uint32_t nal_unit_type = bs_get(&bs, 6);
+      uint32_t nuh_layer_id = bs_get(&bs, 6);
+      uint32_t nuh_temporal_id_plus1 = bs_get(&bs, 3);
+
+      // if (nal_unit_type != 1) printf("%3d -- %3d %10d %lu\n", nal_unit_type, frame_num, (uint32_t)(ptr-data), nal_size);
+
+      switch (nal_unit_type) {
+      case HEVC_NAL_TYPE_VPS_NUT:
+      case HEVC_NAL_TYPE_SPS_NUT:
+      case HEVC_NAL_TYPE_PPS_NUT:
+        fwrite(ptr, 1, nal_size, of_prefix);
+        break;
+      case HEVC_NAL_TYPE_TRAIL_N:
+      case HEVC_NAL_TYPE_TRAIL_R:
+      case HEVC_NAL_TYPE_TSA_N:
+      case HEVC_NAL_TYPE_TSA_R:
+      case HEVC_NAL_TYPE_STSA_N:
+      case HEVC_NAL_TYPE_STSA_R:
+      case HEVC_NAL_TYPE_RADL_N:
+      case HEVC_NAL_TYPE_RADL_R:
+      case HEVC_NAL_TYPE_RASL_N:
+      case HEVC_NAL_TYPE_RASL_R:
+      case HEVC_NAL_TYPE_BLA_W_LP:
+      case HEVC_NAL_TYPE_BLA_W_RADL:
+      case HEVC_NAL_TYPE_BLA_N_LP:
+      case HEVC_NAL_TYPE_IDR_W_RADL:
+      case HEVC_NAL_TYPE_IDR_N_LP:
+      case HEVC_NAL_TYPE_CRA_NUT: {
+        // slice_segment_header
+        uint32_t first_slice_segment_in_pic_flag = bs_get(&bs, 1);
+        if (nal_unit_type >= HEVC_NAL_TYPE_BLA_W_LP && nal_unit_type <= HEVC_NAL_TYPE_RSV_IRAP_VCL23) {
+          uint32_t no_output_of_prior_pics_flag = bs_get(&bs, 1);
+        }
+        uint32_t slice_pic_parameter_set_id = bs_get(&bs, 1);
+        if (!first_slice_segment_in_pic_flag) {
+          // ...
+          break;
+        }
+
+        if (!dependent_slice_segments_enabled_flag) {
+          for (int i=0; i<num_extra_slice_header_bits; i++) {
+            bs_get(&bs, 1);
+          }
+          uint32_t slice_type = bs_ue(&bs);
+
+          // write the index
+          write32le(of_index, slice_type);
+          write32le(of_index, ptr - data);
+
+          // ...
+        }
+
+        break;
+      }
+      }
+
+      //...
+      // emulation_prevention_three_byte
+    }
+
+    ptr = next;
+  }
+
+  write32le(of_index, -1);
+  write32le(of_index, file_size);
+}
+
+// Table 7-1
+enum h264_nal_type {
+  H264_NAL_SLICE           = 1,
+  H264_NAL_DPA             = 2,
+  H264_NAL_DPB             = 3,
+  H264_NAL_DPC             = 4,
+  H264_NAL_IDR_SLICE       = 5,
+  H264_NAL_SEI             = 6,
+  H264_NAL_SPS             = 7,
+  H264_NAL_PPS             = 8,
+  H264_NAL_AUD             = 9,
+  H264_NAL_END_SEQUENCE    = 10,
+  H264_NAL_END_STREAM      = 11,
+  H264_NAL_FILLER_DATA     = 12,
+  H264_NAL_SPS_EXT         = 13,
+  H264_NAL_AUXILIARY_SLICE = 19,
+};
+
+enum h264_slice_type {
+  H264_SLICE_P = 0,
+  H264_SLICE_B = 1,
+  H264_SLICE_I = 2,
+  // ...
+};
+
+static void h264_index(const uint8_t *data, size_t file_size, FILE *of_prefix, FILE *of_index) {
+  const uint8_t* ptr = data;
+  const uint8_t* ptr_end = data + file_size;
+
+  assert(ptr[0] == 0);
+  ptr++;
+  assert(read24be(ptr) == START_CODE);
+
+
+  uint32_t sps_log2_max_frame_num_minus4;
+
+
+  int last_frame_num = -1;
+
+  while (ptr < ptr_end) {
+    const uint8_t* next = ptr+1;
+    for (; next < ptr_end-4; next++) {
+      if (read24be(next) == START_CODE) break;
+    }
+    size_t nal_size = next - ptr;
+    if (nal_size < 5) {
+      break;
+    }
+
+    {
+      struct bitstream bs = {0};
+      bs_init(&bs, ptr, nal_size);
+
+      uint32_t start_code = bs_get(&bs, 24);
+      assert(start_code == 0x000001);
+
+      // nal_unit_header
+      uint32_t forbidden_zero_bit = bs_get(&bs, 1);
+      uint32_t nal_ref_idx = bs_get(&bs, 2);
+      uint32_t nal_unit_type = bs_get(&bs, 5);
+
+      switch (nal_unit_type) {
+      case H264_NAL_SPS:
+
+        {
+          uint32_t profile_idx = bs_get(&bs, 8);
+          uint32_t constraint_sets = bs_get(&bs, 4);
+          uint32_t reserved = bs_get(&bs, 5);
+          uint32_t level_idc = bs_get(&bs, 5);
+          uint32_t seq_parameter_set_id = bs_ue(&bs);
+          sps_log2_max_frame_num_minus4 = bs_ue(&bs);
+        }
+
+        // fallthrough
+      case H264_NAL_PPS:
+        fwrite(ptr, 1, nal_size, of_prefix);
+        break;
+
+      case H264_NAL_SLICE:
+      case H264_NAL_IDR_SLICE: {
+        // slice header
+        uint32_t first_mb_in_slice = bs_ue(&bs);
+        uint32_t slice_type = bs_ue(&bs);
+        uint32_t pic_parameter_set_id = bs_ue(&bs);
+
+        uint32_t frame_num = bs_get(&bs, sps_log2_max_frame_num_minus4+4);
+
+        if (first_mb_in_slice == 0) {
+          write32le(of_index, slice_type);
+          write32le(of_index, ptr - data);
+        }
+
+        break;
+      }
+
+      }
+
+    }
+
+    ptr = next;
+  }
+
+  write32le(of_index, -1);
+  write32le(of_index, file_size);
+}
+
+int main(int argc, char** argv) {
+  if (argc != 5) {
+    fprintf(stderr, "usage: %s h264|hevc file_path out_prefix out_index\n", argv[0]);
+    exit(1);
+  }
+
+  const char* file_type = argv[1];
+  const char* file_path = argv[2];
+
+  int fd = open(file_path, O_RDONLY, 0);
+  if (fd < 0) {
+    fprintf(stderr, "error: couldn't open %s\n", file_path);
+    exit(1);
+  }
+
+  FILE *of_prefix = fopen(argv[3], "wb");
+  assert(of_prefix);
+  FILE *of_index = fopen(argv[4], "wb");
+  assert(of_index);
+
+  off_t file_size = lseek(fd, 0, SEEK_END);
+  lseek(fd, 0, SEEK_SET);
+
+  assert(file_size > 4);
+
+  const uint8_t* data = (const uint8_t*)mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
+  assert(data != MAP_FAILED);
+
+  if (strcmp(file_type, "hevc") == 0) {
+    hevc_index(data, file_size, of_prefix, of_index);
+  } else if (strcmp(file_type, "h264") == 0) {
+    h264_index(data, file_size, of_prefix, of_index);
+  } else {
+    assert(false);
+  }
+
+  munmap((void*)data, file_size);
+  close(fd);
+
+  return 0;
+}