openpilot v0.8.13 release

tools/lib/auth.py

@@ -109,7 +109,7 @@ def login(method):
     if 'code' in web_server.query_params:
       break
     elif 'error' in web_server.query_params:
-      print('Authentication Error: "%s". Description: "%s" ' % (
+      print('Authentication Error: "{}". Description: "{}" '.format(
         web_server.query_params['error'],
         web_server.query_params.get('error_description')), file=sys.stderr)
       break

tools/lib/bootlog.py

@@ -0,0 +1,48 @@
+import datetime
+import functools
+import re
+
+from tools.lib.auth_config import get_token
+from tools.lib.api import CommaApi
+from tools.lib.helpers import RE, timestamp_to_datetime
+
+
+@functools.total_ordering
+class Bootlog:
+  def __init__(self, url: str):
+    self._url = url
+
+    r = re.search(RE.BOOTLOG_NAME, url)
+    if not r:
+      raise Exception(f"Unable to parse: {url}")
+
+    self._dongle_id = r.group('dongle_id')
+    self._timestamp = r.group('timestamp')
+
+  @property
+  def url(self) -> str:
+    return self._url
+
+  @property
+  def dongle_id(self) -> str:
+    return self._dongle_id
+
+  @property
+  def timestamp(self) -> str:
+    return self._timestamp
+
+  @property
+  def datetime(self) -> datetime.datetime:
+    return timestamp_to_datetime(self._timestamp)
+
+  def __eq__(self, b) -> bool:
+    return self.datetime == b.datetime
+
+  def __lt__(self, b) -> bool:
+    return self.datetime < b.datetime
+
+
+def get_bootlogs(dongle_id: str):
+  api = CommaApi(get_token())
+  r = api.get(f'v1/devices/{dongle_id}/bootlogs')
+  return [Bootlog(b) for b in r]

tools/lib/filereader.py

@@ -1,6 +1,11 @@
+import os
 from tools.lib.url_file import URLFile
 
+DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.internal/")
+
 def FileReader(fn, debug=False):
+  if fn.startswith("cd:/"):
+    fn = fn.replace("cd:/", DATA_ENDPOINT)
   if fn.startswith("http://") or fn.startswith("https://"):
     return URLFile(fn, debug=debug)
   return open(fn, "rb")

tools/lib/framereader.py

@@ -17,10 +17,7 @@ from tools.lib.cache import cache_path_for_file_path
 from tools.lib.exceptions import DataUnreadableError
 from common.file_helpers import atomic_write_in_dir
 
-try:
-  from xx.chffr.lib.filereader import FileReader
-except ImportError:
-  from tools.lib.filereader import FileReader
+from tools.lib.filereader import FileReader
 
 HEVC_SLICE_B = 0
 HEVC_SLICE_P = 1
@@ -50,7 +47,7 @@ def fingerprint_video(fn):
   with FileReader(fn) as f:
     header = f.read(4)
   if len(header) == 0:
-    raise DataUnreadableError("%s is empty" % fn)
+    raise DataUnreadableError(f"{fn} is empty")
   elif header == b"\x00\xc0\x12\x00":
     return FrameType.raw
   elif header == b"\x00\x00\x00\x01":
@@ -90,7 +87,7 @@ def vidindex(fn, typ):
     try:
       subprocess.check_call([vidindex, typ, fn, prefix_f.name, index_f.name])
     except subprocess.CalledProcessError:
-      raise DataUnreadableError("vidindex failed on file %s" % fn)
+      raise DataUnreadableError(f"vidindex failed on file {fn}")
     with open(index_f.name, "rb") as f:
       index = f.read()
     with open(prefix_f.name, "rb") as f:
@@ -308,7 +305,7 @@ class RawFrameReader(BaseFrameReader):
     assert num+count <= self.frame_count
 
     if pix_fmt not in ("yuv420p", "rgb24"):
-      raise ValueError("Unsupported pixel format %r" % pix_fmt)
+      raise ValueError(f"Unsupported pixel format {pix_fmt!r}")
 
     app = []
     for i in range(num, num+count):
@@ -548,10 +545,10 @@ class GOPFrameReader(BaseFrameReader):
     assert self.frame_count is not None
 
     if num + count > self.frame_count:
-      raise ValueError("{} > {}".format(num + count, self.frame_count))
+      raise ValueError(f"{num + count} > {self.frame_count}")
 
     if pix_fmt not in ("yuv420p", "rgb24", "yuv444p"):
-      raise ValueError("Unsupported pixel format %r" % pix_fmt)
+      raise ValueError(f"Unsupported pixel format {pix_fmt!r}")
 
     ret = [self._get_one(num + i, pix_fmt) for i in range(count)]
 
@@ -572,15 +569,13 @@ class StreamFrameReader(StreamGOPReader, GOPFrameReader):
 
 def GOPFrameIterator(gop_reader, pix_fmt):
   dec = VideoStreamDecompressor(gop_reader.fn, gop_reader.vid_fmt, gop_reader.w, gop_reader.h, pix_fmt)
-  for frame in dec.read():
-    yield frame
+  yield from dec.read()
 
 
 def FrameIterator(fn, pix_fmt, **kwargs):
   fr = FrameReader(fn, **kwargs)
   if isinstance(fr, GOPReader):
-    for v in GOPFrameIterator(fr, pix_fmt):
-      yield v
+    yield from GOPFrameIterator(fr, pix_fmt)
   else:
     for i in range(fr.frame_count):
       yield fr.get(i, pix_fmt=pix_fmt)[0]

tools/lib/helpers.py

@@ -0,0 +1,20 @@
+import datetime
+
+TIME_FMT = "%Y-%m-%d--%H-%M-%S"
+
+# regex patterns
+class RE:
+  DONGLE_ID =  r'(?P<dongle_id>[a-z0-9]{16})'
+  TIMESTAMP = r'(?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2})'
+  ROUTE_NAME = r'{}[|_/]{}'.format(DONGLE_ID, TIMESTAMP)
+  SEGMENT_NAME = r'{}(?:--|/)(?P<segment_num>[0-9]+)'.format(ROUTE_NAME)
+  BOOTLOG_NAME = ROUTE_NAME
+
+  EXPLORER_FILE = r'^(?P<segment_name>{})--(?P<file_name>[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME)
+  OP_SEGMENT_DIR = r'^(?P<segment_name>{})$'.format(SEGMENT_NAME)
+
+def timestamp_to_datetime(t: str) -> datetime.datetime:
+  """
+    Convert an openpilot route timestamp to a python datetime
+  """
+  return datetime.datetime.strptime(t, TIME_FMT)

tools/lib/logreader.py

@@ -5,17 +5,14 @@ import bz2
 import urllib.parse
 import capnp
 
-try:
-  from xx.chffr.lib.filereader import FileReader
-except ImportError:
-  from tools.lib.filereader import FileReader
+from tools.lib.filereader import FileReader
 from cereal import log as capnp_log
 
 # this is an iterator itself, and uses private variables from LogReader
-class MultiLogIterator(object):
-  def __init__(self, log_paths, wraparound=False):
+class MultiLogIterator:
+  def __init__(self, log_paths, sort_by_time=False):
     self._log_paths = log_paths
-    self._wraparound = wraparound
+    self.sort_by_time = sort_by_time
 
     self._first_log_idx = next(i for i in range(len(log_paths)) if log_paths[i] is not None)
     self._current_log = self._first_log_idx
@@ -26,7 +23,7 @@ class MultiLogIterator(object):
   def _log_reader(self, i):
     if self._log_readers[i] is None and self._log_paths[i] is not None:
       log_path = self._log_paths[i]
-      self._log_readers[i] = LogReader(log_path)
+      self._log_readers[i] = LogReader(log_path, sort_by_time=self.sort_by_time)
 
     return self._log_readers[i]
 
@@ -41,12 +38,8 @@ class MultiLogIterator(object):
       self._idx = 0
       self._current_log = next(i for i in range(self._current_log + 1, len(self._log_readers) + 1)
                                if i == len(self._log_readers) or self._log_paths[i] is not None)
-      # wraparound
       if self._current_log == len(self._log_readers):
-        if self._wraparound:
-          self._current_log = self._first_log_idx
-        else:
-          raise StopIteration
+        raise StopIteration
 
   def __next__(self):
     while 1:
@@ -74,8 +67,8 @@ class MultiLogIterator(object):
     return True
 
 
-class LogReader(object):
-  def __init__(self, fn, canonicalize=True, only_union_types=False):
+class LogReader:
+  def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False):
     data_version = None
     _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
     with FileReader(fn) as f:
@@ -90,7 +83,7 @@ class LogReader(object):
     else:
       raise Exception(f"unknown extension {ext}")
 
-    self._ents = list(ents)
+    self._ents = list(sorted(ents, key=lambda x: x.logMonoTime) if sort_by_time else ents)
     self._ts = [x.logMonoTime for x in self._ents]
     self.data_version = data_version
     self._only_union_types = only_union_types
@@ -112,6 +105,6 @@ if __name__ == "__main__":
   # below line catches those errors and replaces the bytes with \x__
   codecs.register_error("strict", codecs.backslashreplace_errors)
   log_path = sys.argv[1]
-  lr = LogReader(log_path)
+  lr = LogReader(log_path, sort_by_time=True)
   for msg in lr:
     print(msg)

tools/lib/robust_logreader.py

@@ -13,7 +13,7 @@ from cereal import log as capnp_log
 
 
 class RobustLogReader(LogReader):
-  def __init__(self, fn, canonicalize=True, only_union_types=False):  # pylint: disable=super-init-not-called
+  def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False):  # pylint: disable=super-init-not-called
     data_version = None
     _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
     with FileReader(fn) as f:
@@ -45,7 +45,7 @@ class RobustLogReader(LogReader):
     while True:
       try:
         ents = capnp_log.Event.read_multiple_bytes(dat)
-        self._ents = list(ents)
+        self._ents = list(sorted(ents, key=lambda x: x.logMonoTime) if sort_by_time else ents)
         break
       except capnp.lib.capnp.KjException:
         if progress is None:

tools/lib/route.py

@@ -6,10 +6,7 @@ from itertools import chain
 
 from tools.lib.auth_config import get_token
 from tools.lib.api import CommaApi
-
-SEGMENT_NAME_RE = r'[a-z0-9]{16}[|_][0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2}--[0-9]+'
-EXPLORER_FILE_RE = r'^({})--([a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE)
-OP_SEGMENT_DIR_RE = r'^({})$'.format(SEGMENT_NAME_RE)
+from tools.lib.helpers import RE
 
 QLOG_FILENAMES = ['qlog.bz2']
 QCAMERA_FILENAMES = ['qcamera.ts']
@@ -18,48 +15,52 @@ CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc']
 DCAMERA_FILENAMES = ['dcamera.hevc']
 ECAMERA_FILENAMES = ['ecamera.hevc']
 
-class Route(object):
-  def __init__(self, route_name, data_dir=None):
+class Route:
+  def __init__(self, name, data_dir=None):
+    self._name = RouteName(name)
     self.files = None
-    self.route_name = route_name.replace('_', '|')
     if data_dir is not None:
       self._segments = self._get_segments_local(data_dir)
     else:
       self._segments = self._get_segments_remote()
-    self.max_seg_number = self._segments[-1].canonical_name.segment_num
+    self.max_seg_number = self._segments[-1].name.segment_num
+
+  @property
+  def name(self):
+    return self._name
 
   @property
   def segments(self):
     return self._segments
 
   def log_paths(self):
-    log_path_by_seg_num = {s.canonical_name.segment_num: s.log_path for s in self._segments}
+    log_path_by_seg_num = {s.name.segment_num: s.log_path for s in self._segments}
     return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def qlog_paths(self):
-    qlog_path_by_seg_num = {s.canonical_name.segment_num: s.qlog_path for s in self._segments}
+    qlog_path_by_seg_num = {s.name.segment_num: s.qlog_path for s in self._segments}
     return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def camera_paths(self):
-    camera_path_by_seg_num = {s.canonical_name.segment_num: s.camera_path for s in self._segments}
+    camera_path_by_seg_num = {s.name.segment_num: s.camera_path for s in self._segments}
     return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def dcamera_paths(self):
-    dcamera_path_by_seg_num = {s.canonical_name.segment_num: s.dcamera_path for s in self._segments}
+    dcamera_path_by_seg_num = {s.name.segment_num: s.dcamera_path for s in self._segments}
     return [dcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def ecamera_paths(self):
-    ecamera_path_by_seg_num = {s.canonical_name.segment_num: s.ecamera_path for s in self._segments}
+    ecamera_path_by_seg_num = {s.name.segment_num: s.ecamera_path for s in self._segments}
     return [ecamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def qcamera_paths(self):
-    qcamera_path_by_seg_num = {s.canonical_name.segment_num: s.qcamera_path for s in self._segments}
+    qcamera_path_by_seg_num = {s.name.segment_num: s.qcamera_path for s in self._segments}
     return [qcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   # TODO: refactor this, it's super repetitive
   def _get_segments_remote(self):
     api = CommaApi(get_token())
-    route_files = api.get('v1/route/' + self.route_name + '/files')
+    route_files = api.get('v1/route/' + self.name.canonical_name + '/files')
     self.files = list(chain.from_iterable(route_files.values()))
 
     segments = {}
@@ -67,7 +68,7 @@ class Route(object):
       _, dongle_id, time_str, segment_num, fn = urlparse(url).path.rsplit('/', maxsplit=4)
       segment_name = f'{dongle_id}|{time_str}--{segment_num}'
       if segments.get(segment_name):
-        segments[segment_name] = RouteSegment(
+        segments[segment_name] = Segment(
           segment_name,
           url if fn in LOG_FILENAMES else segments[segment_name].log_path,
           url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path,
@@ -77,7 +78,7 @@ class Route(object):
           url if fn in QCAMERA_FILENAMES else segments[segment_name].qcamera_path,
         )
       else:
-        segments[segment_name] = RouteSegment(
+        segments[segment_name] = Segment(
           segment_name,
           url if fn in LOG_FILENAMES else None,
           url if fn in QLOG_FILENAMES else None,
@@ -87,7 +88,7 @@ class Route(object):
           url if fn in QCAMERA_FILENAMES else None,
         )
 
-    return sorted(segments.values(), key=lambda seg: seg.canonical_name.segment_num)
+    return sorted(segments.values(), key=lambda seg: seg.name.segment_num)
 
   def _get_segments_local(self, data_dir):
     files = os.listdir(data_dir)
@@ -95,24 +96,25 @@ class Route(object):
 
     for f in files:
       fullpath = os.path.join(data_dir, f)
-      explorer_match = re.match(EXPLORER_FILE_RE, f)
-      op_match = re.match(OP_SEGMENT_DIR_RE, f)
+      explorer_match = re.match(RE.EXPLORER_FILE, f)
+      op_match = re.match(RE.OP_SEGMENT_DIR, f)
 
       if explorer_match:
-        segment_name, fn = explorer_match.groups()
-        if segment_name.replace('_', '|').startswith(self.route_name):
+        segment_name = explorer_match.group('segment_name')
+        fn = explorer_match.group('file_name')
+        if segment_name.replace('_', '|').startswith(self.name.canonical_name):
           segment_files[segment_name].append((fullpath, fn))
       elif op_match and os.path.isdir(fullpath):
-        segment_name, = op_match.groups()
-        if segment_name.startswith(self.route_name):
+        segment_name = op_match.group('segment_name')
+        if segment_name.startswith(self.name.canonical_name):
           for seg_f in os.listdir(fullpath):
             segment_files[segment_name].append((os.path.join(fullpath, seg_f), seg_f))
-      elif f == self.route_name:
+      elif f == self.name.canonical_name:
         for seg_num in os.listdir(fullpath):
           if not seg_num.isdigit():
             continue
 
-          segment_name = '{}--{}'.format(self.route_name, seg_num)
+          segment_name = f'{self.name.canonical_name}--{seg_num}'
           for seg_f in os.listdir(os.path.join(fullpath, seg_num)):
             segment_files[segment_name].append((os.path.join(fullpath, seg_num, seg_f), seg_f))
 
@@ -149,15 +151,15 @@ class Route(object):
       except StopIteration:
         qcamera_path = None
 
-      segments.append(RouteSegment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path))
+      segments.append(Segment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path))
 
     if len(segments) == 0:
-      raise ValueError('Could not find segments for route {} in data directory {}'.format(self.route_name, data_dir))
-    return sorted(segments, key=lambda seg: seg.canonical_name.segment_num)
+      raise ValueError(f'Could not find segments for route {self.name.canonical_name} in data directory {data_dir}')
+    return sorted(segments, key=lambda seg: seg.name.segment_num)
 
-class RouteSegment(object):
+class Segment:
   def __init__(self, name, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path):
-    self._name = RouteSegmentName(name)
+    self._name = SegmentName(name)
     self.log_path = log_path
     self.qlog_path = qlog_path
     self.camera_path = camera_path
@@ -167,21 +169,55 @@ class RouteSegment(object):
 
   @property
   def name(self):
-    return str(self._name)
-
-  @property
-  def canonical_name(self):
     return self._name
 
-class RouteSegmentName(object):
-  def __init__(self, name_str):
-    self._segment_name_str = name_str
-    self._route_name_str, num_str = self._segment_name_str.rsplit("--", 1)
-    self._num = int(num_str)
+class RouteName:
+  def __init__(self, name_str: str):
+    self._name_str = name_str
+    delim = next(c for c in self._name_str if c in ("|", "/"))
+    self._dongle_id, self._time_str = self._name_str.split(delim)
+
+    assert len(self._dongle_id) == 16, self._name_str
+    assert len(self._time_str) == 20, self._name_str
+    self._canonical_name = f"{self._dongle_id}|{self._time_str}"
 
   @property
-  def segment_num(self):
-    return self._num
+  def canonical_name(self) -> str: return self._canonical_name
 
-  def __str__(self):
-    return self._segment_name_str
+  @property
+  def dongle_id(self) -> str: return self._dongle_id
+
+  @property
+  def time_str(self) -> str: return self._time_str
+
+  def __str__(self) -> str: return self._canonical_name
+
+class SegmentName:
+  # TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances
+  # of this class instead of manually constructing a segment name (use canonical_name prop instead)
+  def __init__(self, name_str: str, allow_route_name=False):
+    self._name_str = name_str
+    seg_num_delim = "--" if self._name_str.count("--") == 2 else "/"
+    name_parts = self._name_str.rsplit(seg_num_delim, 1)
+    if allow_route_name and len(name_parts) == 1:
+      name_parts.append("-1") # no segment number
+    self._route_name = RouteName(name_parts[0])
+    self._num = int(name_parts[1])
+    self._canonical_name = f"{self._route_name._dongle_id}|{self._route_name._time_str}--{self._num}"
+
+  @property
+  def canonical_name(self) -> str: return self._canonical_name
+
+  @property
+  def dongle_id(self) -> str: return self._route_name.dongle_id
+
+  @property
+  def time_str(self) -> str: return self._route_name.time_str
+
+  @property
+  def segment_num(self) -> int: return self._num
+
+  @property
+  def route_name(self) -> RouteName: return self._route_name
+
+  def __str__(self) -> str: return self._canonical_name

tools/lib/url_file.py

@@ -22,7 +22,7 @@ def hash_256(link):
   return hsh
 
 
-class URLFile(object):
+class URLFile:
   _tlocal = threading.local()
 
   def __init__(self, url, debug=False, cache=None):
@@ -70,7 +70,7 @@ class URLFile(object):
       return self._length
     file_length_path = os.path.join(CACHE_DIR, hash_256(self._url) + "_length")
     if os.path.exists(file_length_path) and not self._force_download:
-      with open(file_length_path, "r") as file_length:
+      with open(file_length_path) as file_length:
           content = file_length.read()
           self._length = int(content)
           return self._length
@@ -156,7 +156,7 @@ class URLFile(object):
     if self._debug:
       t2 = time.time()
       if t2 - t1 > 0.1:
-        print("get %s %r %.f slow" % (self._url, headers, t2 - t1))
+        print(f"get {self._url} {headers!r} {t2 - t1:.f} slow")
 
     response_code = c.getinfo(pycurl.RESPONSE_CODE)
     if response_code == 416:  # Requested Range Not Satisfiable