dragonpilot beta3

date: 2023-10-09T10:55:55
commit: 91b6e3aecd7170f24bccacb10c515ec281c30295

tools/lib/README.md

@@ -3,8 +3,8 @@
 Route is a class for conveniently accessing all the [logs](/system/loggerd/) from your routes. The LogReader class reads the non-video logs, i.e. rlog.bz2 and qlog.bz2. There's also a matching FrameReader class for reading the videos.
 
 ```python
-from tools.lib.route import Route
-from tools.lib.logreader import LogReader
+from openpilot.tools.lib.route import Route
+from openpilot.tools.lib.logreader import LogReader
 
 r = Route("a2a0ccea32023010|2023-07-27--13-01-19")
 
@@ -37,8 +37,8 @@ for msg in lr:
 `MultiLogIterator` is similar to `LogReader`, but reads multiple logs. 
 
 ```python
-from tools.lib.route import Route
-from tools.lib.logreader import MultiLogIterator
+from openpilot.tools.lib.route import Route
+from openpilot.tools.lib.logreader import MultiLogIterator
 
 # setup a MultiLogIterator to read all the logs in the route
 r = Route("a2a0ccea32023010|2023-07-27--13-01-19")

tools/lib/auth.py

@@ -29,8 +29,8 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import Any, Dict
 from urllib.parse import parse_qs, urlencode
 
-from tools.lib.api import APIError, CommaApi, UnauthorizedError
-from tools.lib.auth_config import set_token, get_token
+from openpilot.tools.lib.api import APIError, CommaApi, UnauthorizedError
+from openpilot.tools.lib.auth_config import set_token, get_token
 
 PORT = 3000
 
@@ -54,7 +54,7 @@ class ClientRedirectHandler(BaseHTTPRequestHandler):
     self.end_headers()
     self.wfile.write(b'Return to the CLI to continue')
 
-  def log_message(self, *args):  # pylint: disable=redefined-builtin
+  def log_message(self, *args):
     pass  # this prevent http server from dumping messages to stdout
 
 

tools/lib/auth_config.py

@@ -1,7 +1,7 @@
 import json
 import os
-from common.file_helpers import mkdirs_exists_ok
-from system.hardware import PC
+from openpilot.common.file_helpers import mkdirs_exists_ok
+from openpilot.system.hardware import PC
 
 
 class MissingAuthConfigError(Exception):
@@ -13,9 +13,6 @@ if PC:
 else:
   CONFIG_DIR = "/tmp/.comma"
 
-mkdirs_exists_ok(CONFIG_DIR)
-
-
 def get_token():
   try:
     with open(os.path.join(CONFIG_DIR, 'auth.json')) as f:
@@ -26,9 +23,13 @@ def get_token():
 
 
 def set_token(token):
+  mkdirs_exists_ok(CONFIG_DIR)
   with open(os.path.join(CONFIG_DIR, 'auth.json'), 'w') as f:
     json.dump({'access_token': token}, f)
 
 
 def clear_token():
-  os.unlink(os.path.join(CONFIG_DIR, 'auth.json'))
+  try:
+    os.unlink(os.path.join(CONFIG_DIR, 'auth.json'))
+  except FileNotFoundError:
+    pass

tools/lib/bootlog.py

@@ -3,9 +3,9 @@ import functools
 import re
 from typing import List, Optional
 
-from tools.lib.auth_config import get_token
-from tools.lib.api import CommaApi
-from tools.lib.helpers import RE, timestamp_to_datetime
+from openpilot.tools.lib.auth_config import get_token
+from openpilot.tools.lib.api import CommaApi
+from openpilot.tools.lib.helpers import RE, timestamp_to_datetime
 
 
 @functools.total_ordering

tools/lib/cache.py

@@ -1,11 +1,11 @@
 import os
 import urllib.parse
-from common.file_helpers import mkdirs_exists_ok
+from openpilot.common.file_helpers import mkdirs_exists_ok
 
-DEFAULT_CACHE_DIR = os.path.expanduser("~/.commacache")
+DEFAULT_CACHE_DIR = os.getenv("CACHE_ROOT", os.path.expanduser("~/.commacache"))
 
-def cache_path_for_file_path(fn, cache_prefix=None):
-  dir_ = os.path.join(DEFAULT_CACHE_DIR, "local")
+def cache_path_for_file_path(fn, cache_dir=DEFAULT_CACHE_DIR):
+  dir_ = os.path.join(cache_dir, "local")
   mkdirs_exists_ok(dir_)
   fn_parsed = urllib.parse.urlparse(fn)
   if fn_parsed.scheme == '':

tools/lib/filereader.py

@@ -1,5 +1,5 @@
 import os
-from tools.lib.url_file import URLFile
+from openpilot.tools.lib.url_file import URLFile
 
 DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/")
 

tools/lib/framereader.py

@@ -1,4 +1,3 @@
-# pylint: skip-file
 import json
 import os
 import pickle
@@ -13,11 +12,11 @@ import numpy as np
 from lru import LRU
 
 import _io
-from tools.lib.cache import cache_path_for_file_path
-from tools.lib.exceptions import DataUnreadableError
-from common.file_helpers import atomic_write_in_dir
+from openpilot.tools.lib.cache import cache_path_for_file_path, DEFAULT_CACHE_DIR
+from openpilot.tools.lib.exceptions import DataUnreadableError
+from openpilot.common.file_helpers import atomic_write_in_dir
 
-from tools.lib.filereader import FileReader
+from openpilot.tools.lib.filereader import FileReader
 
 HEVC_SLICE_B = 0
 HEVC_SLICE_P = 1
@@ -107,8 +106,8 @@ def cache_fn(func):
     if kwargs.pop('no_cache', None):
       cache_path = None
     else:
-      cache_prefix = kwargs.pop('cache_prefix', None)
-      cache_path = cache_path_for_file_path(fn, cache_prefix)
+      cache_dir = kwargs.pop('cache_dir', DEFAULT_CACHE_DIR)
+      cache_path = cache_path_for_file_path(fn, cache_dir)
 
     if cache_path and os.path.exists(cache_path):
       with open(cache_path, "rb") as cache_file:
@@ -141,18 +140,18 @@ def index_stream(fn, typ):
   }
 
 
-def index_videos(camera_paths, cache_prefix=None):
+def index_videos(camera_paths, cache_dir=DEFAULT_CACHE_DIR):
   """Requires that paths in camera_paths are contiguous and of the same type."""
   if len(camera_paths) < 1:
     raise ValueError("must provide at least one video to index")
 
   frame_type = fingerprint_video(camera_paths[0])
   for fn in camera_paths:
-    index_video(fn, frame_type, cache_prefix)
+    index_video(fn, frame_type, cache_dir)
 
 
-def index_video(fn, frame_type=None, cache_prefix=None):
-  cache_path = cache_path_for_file_path(fn, cache_prefix)
+def index_video(fn, frame_type=None, cache_dir=DEFAULT_CACHE_DIR):
+  cache_path = cache_path_for_file_path(fn, cache_dir)
 
   if os.path.exists(cache_path):
     return
@@ -161,16 +160,16 @@ def index_video(fn, frame_type=None, cache_prefix=None):
     frame_type = fingerprint_video(fn[0])
 
   if frame_type == FrameType.h265_stream:
-    index_stream(fn, "hevc", cache_prefix=cache_prefix)
+    index_stream(fn, "hevc", cache_dir=cache_dir)
   else:
     raise NotImplementedError("Only h265 supported")
 
 
-def get_video_index(fn, frame_type, cache_prefix=None):
-  cache_path = cache_path_for_file_path(fn, cache_prefix)
+def get_video_index(fn, frame_type, cache_dir=DEFAULT_CACHE_DIR):
+  cache_path = cache_path_for_file_path(fn, cache_dir)
 
   if not os.path.exists(cache_path):
-    index_video(fn, frame_type, cache_prefix)
+    index_video(fn, frame_type, cache_dir)
 
   if not os.path.exists(cache_path):
     return None
@@ -285,13 +284,13 @@ class BaseFrameReader:
     raise NotImplementedError
 
 
-def FrameReader(fn, cache_prefix=None, readahead=False, readbehind=False, index_data=None):
+def FrameReader(fn, cache_dir=DEFAULT_CACHE_DIR, readahead=False, readbehind=False, index_data=None):
   frame_type = fingerprint_video(fn)
   if frame_type == FrameType.raw:
     return RawFrameReader(fn)
   elif frame_type in (FrameType.h265_stream,):
     if not index_data:
-      index_data = get_video_index(fn, frame_type, cache_prefix)
+      index_data = get_video_index(fn, frame_type, cache_dir)
     return StreamFrameReader(fn, frame_type, index_data, readahead=readahead, readbehind=readbehind)
   else:
     raise NotImplementedError(frame_type)

tools/lib/logreader.py

@@ -8,8 +8,8 @@ import warnings
 
 
 from cereal import log as capnp_log
-from tools.lib.filereader import FileReader
-from tools.lib.route import Route, SegmentName
+from openpilot.tools.lib.filereader import FileReader
+from openpilot.tools.lib.route import Route, SegmentName
 
 # this is an iterator itself, and uses private variables from LogReader
 class MultiLogIterator:

tools/lib/route.py

@@ -5,9 +5,9 @@ from collections import defaultdict
 from itertools import chain
 from typing import Optional
 
-from tools.lib.auth_config import get_token
-from tools.lib.api import CommaApi
-from tools.lib.helpers import RE
+from openpilot.tools.lib.auth_config import get_token
+from openpilot.tools.lib.api import CommaApi
+from openpilot.tools.lib.helpers import RE
 
 QLOG_FILENAMES = ['qlog', 'qlog.bz2']
 QCAMERA_FILENAMES = ['qcamera.ts']

tools/lib/tests/test_caching.py

@@ -1,18 +1,13 @@
 #!/usr/bin/env python3
 import os
-import shutil
 import unittest
-
-os.environ["COMMA_CACHE"] = "/tmp/__test_cache__"
-from tools.lib.url_file import URLFile, CACHE_DIR
+from openpilot.tools.lib.url_file import URLFile
 
 
 class TestFileDownload(unittest.TestCase):
 
   def compare_loads(self, url, start=0, length=None):
     """Compares range between cached and non cached version"""
-    shutil.rmtree(CACHE_DIR)
-
     file_cached = URLFile(url, cache=True)
     file_downloaded = URLFile(url, cache=False)
 

tools/lib/tests/test_readers.py

@@ -5,8 +5,8 @@ import tempfile
 
 from collections import defaultdict
 import numpy as np
-from tools.lib.framereader import FrameReader
-from tools.lib.logreader import LogReader
+from openpilot.tools.lib.framereader import FrameReader
+from openpilot.tools.lib.logreader import LogReader
 
 
 class TestReaders(unittest.TestCase):

tools/lib/tests/test_route_library.py

@@ -2,7 +2,7 @@
 import unittest
 from collections import namedtuple
 
-from tools.lib.route import SegmentName
+from openpilot.tools.lib.route import SegmentName
 
 class TestRouteLibrary(unittest.TestCase):
   def test_segment_name_formats(self):

tools/lib/url_file.py

@@ -1,5 +1,3 @@
-# pylint: skip-file
-
 import os
 import time
 import tempfile
@@ -9,13 +7,12 @@ import pycurl
 from hashlib import sha256
 from io import BytesIO
 from tenacity import retry, wait_random_exponential, stop_after_attempt
-from common.file_helpers import mkdirs_exists_ok, atomic_write_in_dir
+from openpilot.common.file_helpers import mkdirs_exists_ok, atomic_write_in_dir
+from openpilot.system.hardware.hw import Paths
 #  Cache chunk size
 K = 1000
 CHUNK_SIZE = 1000 * K
 
-CACHE_DIR = os.environ.get("COMMA_CACHE", "/tmp/comma_download_cache/")
-
 
 def hash_256(link):
   hsh = str(sha256((link.split("?")[0]).encode('utf-8')).hexdigest())
@@ -40,7 +37,7 @@ class URLFile:
       self._curl = self._tlocal.curl
     except AttributeError:
       self._curl = self._tlocal.curl = pycurl.Curl()
-    mkdirs_exists_ok(CACHE_DIR)
+    mkdirs_exists_ok(Paths.download_cache_root())
 
   def __enter__(self):
     return self
@@ -68,7 +65,7 @@ class URLFile:
   def get_length(self):
     if self._length is not None:
       return self._length
-    file_length_path = os.path.join(CACHE_DIR, hash_256(self._url) + "_length")
+    file_length_path = os.path.join(Paths.download_cache_root(), hash_256(self._url) + "_length")
     if os.path.exists(file_length_path) and not self._force_download:
       with open(file_length_path) as file_length:
           content = file_length.read()
@@ -95,7 +92,7 @@ class URLFile:
       self._pos = position
       chunk_number = self._pos / CHUNK_SIZE
       file_name = hash_256(self._url) + "_" + str(chunk_number)
-      full_path = os.path.join(CACHE_DIR, str(file_name))
+      full_path = os.path.join(Paths.download_cache_root(), str(file_name))
       data = None
       #  If we don't have a file, download it
       if not os.path.exists(full_path):

tools/lib/vidindex/bitstream.c

@@ -1,8 +1,8 @@
+#include "./bitstream.h"
+
 #include <stdbool.h>
 #include <assert.h>
 
-#include "bitstream.h"
-
 static const uint32_t BS_MASKS[33] = {
     0,           0x1L,        0x3L,       0x7L,       0xFL,       0x1FL,
     0x3FL,       0x7FL,       0xFFL,      0x1FFL,     0x3FFL,     0x7FFL,

tools/lib/vidindex/vidindex.c

@@ -9,7 +9,7 @@
 #include <sys/stat.h>
 #include <sys/mman.h>
 
-#include "bitstream.h"
+#include "./bitstream.h"
 
 #define START_CODE 0x000001