openpilot v0.8.11 release

tools/lib/README.md

@@ -1,6 +1,4 @@
-# tools libs
-
-## [route.py](route.py) & [logreader.py](logreader.py)
+## LogReader
 
 Route is a class for conviently accessing all the [logs](/selfdrive/loggerd/) from your routes. The LogReader class reads the non-video logs, i.e. rlog.bz2 and qlog.bz2. There's also a matching FrameReader class for reading the videos.
 

tools/lib/auth.py

@@ -1,20 +1,47 @@
 #!/usr/bin/env python3
+"""
+Usage::
+
+  usage: auth.py [-h] [{google,apple,github,jwt}] [jwt]
+
+  Login to your comma account
+
+  positional arguments:
+    {google,apple,github,jwt}
+    jwt
+
+  optional arguments:
+    -h, --help            show this help message and exit
+
+
+Examples::
+
+  ./auth.py  # Log in with google account
+  ./auth.py github  # Log in with GitHub Account
+  ./auth.py jwt ey......hw  # Log in with a JWT from https://jwt.comma.ai, for use in CI
+"""
+
+import argparse
 import sys
+import pprint
 import webbrowser
-from http.server import HTTPServer, BaseHTTPRequestHandler
-from urllib.parse import urlencode, parse_qs
-from tools.lib.api import CommaApi, APIError
-from tools.lib.auth_config import set_token
-from typing import Dict, Any
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Dict
+from urllib.parse import parse_qs, urlencode
+
+from tools.lib.api import APIError, CommaApi, UnauthorizedError
+from tools.lib.auth_config import set_token, get_token
 
 PORT = 3000
 
+
 class ClientRedirectServer(HTTPServer):
   query_params: Dict[str, Any] = {}
 
+
 class ClientRedirectHandler(BaseHTTPRequestHandler):
   def do_GET(self):
-    if not self.path.startswith('/auth/g/redirect'):
+    if not self.path.startswith('/auth'):
       self.send_response(204)
       return
 
@@ -30,21 +57,48 @@ class ClientRedirectHandler(BaseHTTPRequestHandler):
   def log_message(self, format, *args):  # pylint: disable=redefined-builtin
     pass  # this prevent http server from dumping messages to stdout
 
-def auth_redirect_link():
-  redirect_uri = f'http://localhost:{PORT}/auth/g/redirect'
+
+def auth_redirect_link(method):
+  provider_id = {
+    'google': 'g',
+    'apple': 'a',
+    'github': 'h',
+  }[method]
+
   params = {
-    'type': 'web_server',
-    'client_id': '45471411055-ornt4svd2miog6dnopve7qtmh5mnu6id.apps.googleusercontent.com',
-    'redirect_uri': redirect_uri,
-    'response_type': 'code',
-    'scope': 'https://www.googleapis.com/auth/userinfo.email',
-    'prompt': 'select_account',
+    'redirect_uri': f"https://api.comma.ai/v2/auth/{provider_id}/redirect/",
+    'state': f'service,localhost:{PORT}',
   }
 
-  return (redirect_uri, 'https://accounts.google.com/o/oauth2/auth?' + urlencode(params))
+  if method == 'google':
+    params.update({
+      'type': 'web_server',
+      'client_id': '45471411055-ornt4svd2miog6dnopve7qtmh5mnu6id.apps.googleusercontent.com',
+      'response_type': 'code',
+      'scope': 'https://www.googleapis.com/auth/userinfo.email',
+      'prompt': 'select_account',
+    })
+    return 'https://accounts.google.com/o/oauth2/auth?' + urlencode(params)
+  elif method == 'github':
+    params.update({
+      'client_id': '28c4ecb54bb7272cb5a4',
+      'scope': 'read:user',
+    })
+    return 'https://github.com/login/oauth/authorize?' + urlencode(params)
+  elif method == 'apple':
+      params.update({
+        'client_id': 'ai.comma.login',
+        'response_type': 'code',
+        'response_mode': 'form_post',
+        'scope': 'name email',
+      })
+      return 'https://appleid.apple.com/auth/authorize?' + urlencode(params)
+  else:
+    raise NotImplementedError(f"no redirect implemented for method {method}")
 
-def login():
-  redirect_uri, oauth_uri = auth_redirect_link()
+
+def login(method):
+  oauth_uri = auth_redirect_link(method)
 
   web_server = ClientRedirectServer(('localhost', PORT), ClientRedirectHandler)
   print(f'To sign in, use your browser and navigate to {oauth_uri}')
@@ -53,7 +107,6 @@ def login():
   while True:
     web_server.handle_request()
     if 'code' in web_server.query_params:
-      code = web_server.query_params['code']
       break
     elif 'error' in web_server.query_params:
       print('Authentication Error: "%s". Description: "%s" ' % (
@@ -62,11 +115,31 @@ def login():
       break
 
   try:
-    auth_resp = CommaApi().post('v2/auth/', data={'code': code, 'redirect_uri': redirect_uri})
+    auth_resp = CommaApi().post('v2/auth/', data={'code': web_server.query_params['code'], 'provider': web_server.query_params['provider']})
     set_token(auth_resp['access_token'])
-    print('Authenticated')
   except APIError as e:
     print(f'Authentication Error: {e}', file=sys.stderr)
 
+
 if __name__ == '__main__':
-  login()
+  parser = argparse.ArgumentParser(description='Login to your comma account')
+  parser.add_argument('method', default='google', const='google', nargs='?', choices=['google', 'apple', 'github', 'jwt'])
+  parser.add_argument('jwt', nargs='?')
+
+  args = parser.parse_args()
+  if args.method == 'jwt':
+    if args.jwt is None:
+      print("method JWT selected, but no JWT was provided")
+      exit(1)
+
+    set_token(args.jwt)
+  else:
+    login(args.method)
+
+  try:
+    me = CommaApi(token=get_token()).get('/v1/me')
+    print("Authenticated!")
+    pprint.pprint(me)
+  except UnauthorizedError:
+    print("Got invalid JWT")
+    exit(1)

tools/lib/framereader.py

@@ -6,10 +6,10 @@ import struct
 import subprocess
 import tempfile
 import threading
+from enum import IntEnum
 from functools import wraps
 
 import numpy as np
-from aenum import Enum
 from lru import LRU
 
 import _io
@@ -41,7 +41,7 @@ class DoNothingContextManager:
     pass
 
 
-class FrameType(Enum):
+class FrameType(IntEnum):
   raw = 1
   h265_stream = 2
 

tools/lib/robust_logreader.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import os
+import bz2
+import urllib.parse
+import subprocess
+import tqdm
+import glob
+from tempfile import TemporaryDirectory
+import capnp
+
+from tools.lib.logreader import FileReader, LogReader
+from cereal import log as capnp_log
+
+
+class RobustLogReader(LogReader):
+  def __init__(self, fn, canonicalize=True, only_union_types=False):  # pylint: disable=super-init-not-called
+    data_version = None
+    _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
+    with FileReader(fn) as f:
+      dat = f.read()
+
+    if ext == "":
+      pass
+    elif ext == ".bz2":
+      try:
+        dat = bz2.decompress(dat)
+      except ValueError:
+        print("Failed to decompress, falling back to bzip2recover")
+        with TemporaryDirectory() as directory:
+          # Run bzip2recovery on log
+          with open(os.path.join(directory, 'out.bz2'), 'wb') as f:
+            f.write(dat)
+          subprocess.check_call(["bzip2recover", "out.bz2"], cwd=directory)
+
+          # Decompress and concatenate parts
+          dat = b""
+          for n in sorted(glob.glob(f"{directory}/rec*.bz2")):
+            print(f"Decompressing {n}")
+            with open(n, 'rb') as f:
+              dat += bz2.decompress(f.read())
+    else:
+      raise Exception(f"unknown extension {ext}")
+
+    progress = None
+    while True:
+      try:
+        ents = capnp_log.Event.read_multiple_bytes(dat)
+        self._ents = list(ents)
+        break
+      except capnp.lib.capnp.KjException:
+        if progress is None:
+          progress = tqdm.tqdm(total=len(dat))
+
+        # Cut off bytes at the end until capnp is able to read
+        dat = dat[:-1]
+        progress.update(1)
+
+    self._ts = [x.logMonoTime for x in self._ents]
+    self.data_version = data_version
+    self._only_union_types = only_union_types

tools/lib/route.py

@@ -20,6 +20,7 @@ ECAMERA_FILENAMES = ['ecamera.hevc']
 
 class Route(object):
   def __init__(self, route_name, data_dir=None):
+    self.files = None
     self.route_name = route_name.replace('_', '|')
     if data_dir is not None:
       self._segments = self._get_segments_local(data_dir)
@@ -59,9 +60,10 @@ class Route(object):
   def _get_segments_remote(self):
     api = CommaApi(get_token())
     route_files = api.get('v1/route/' + self.route_name + '/files')
+    self.files = list(chain.from_iterable(route_files.values()))
 
     segments = {}
-    for url in chain.from_iterable(route_files.values()):
+    for url in self.files:
       _, dongle_id, time_str, segment_num, fn = urlparse(url).path.rsplit('/', maxsplit=4)
       segment_name = f'{dongle_id}|{time_str}--{segment_num}'
       if segments.get(segment_name):