FileReader: cache files locally (#2170)

* framework for updates

* File caching

* update .gitignore

* update .gitignore

* Store in home tmp directory

* fix bugs in old code, add tests

* add test

* revert bug

* Fixed some bugs, more testing

* doing some style fixes

* Update caching with lenghts

* doing some style fixes

* doing some style fixes

* doing some style fixes

* subrepositories

* try to fix the tests

* change submodules back

* doing some style fixes

* if we don't cache, we don't cache length either

* fix curls

* make sure tests run in ci

* Cleanup tests

* Use int for cache flag

* fix linter

Co-authored-by: Willem Melching <willem.melching@gmail.com>
old-commit-hash: 05b64257dfac3e240135e9f40eb5ebaa8f97fdcb

tools/lib/tests/test_caching.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+import shutil
+from tools.lib.url_file import URLFile, CACHE_DIR
+
+
+class TestFileDownload(unittest.TestCase):
+
+  def compare_loads(self, url, start=0, length=None):
+    """Compares range between cached and non cached version"""
+    shutil.rmtree(CACHE_DIR)
+
+    file_cached = URLFile(url, cache=True)
+    file_downloaded = URLFile(url, cache=False)
+
+    file_cached.seek(start)
+    file_downloaded.seek(start)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertLessEqual(length + start if length is not None else 0, file_downloaded.get_length())
+
+    response_cached = file_cached.read(ll=length)
+    response_downloaded = file_downloaded.read(ll=length)
+
+    self.assertEqual(response_cached, response_downloaded)
+
+    # Now test with cache in place
+    file_cached = URLFile(url, cache=True)
+    file_cached.seek(start)
+    response_cached = file_cached.read(ll=length)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertEqual(response_cached, response_downloaded)
+
+  def test_small_file(self):
+    # Make sure we don't force cache
+    os.environ["FILEREADER_CACHE"] = "0"
+    small_file_url = "https://raw.githubusercontent.com/commaai/openpilot/master/SAFETY.md"
+    #  If you want large file to be larger than a chunk
+    #  large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/fcamera.hevc"
+
+    #  Load full small file
+    self.compare_loads(small_file_url)
+
+    file_small = URLFile(small_file_url)
+    length = file_small.get_length()
+
+    self.compare_loads(small_file_url, length - 100, 100)
+    self.compare_loads(small_file_url, 50, 100)
+
+    #  Load small file 100 bytes at a time
+    for i in range(length // 100):
+      self.compare_loads(small_file_url, 100 * i, 100)
+
+  def test_large_file(self):
+    large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+    #  Load the end 100 bytes of both files
+    file_large = URLFile(large_file_url)
+    length = file_large.get_length()
+
+    self.compare_loads(large_file_url, length - 100, 100)
+    self.compare_loads(large_file_url)
+
+
+if __name__ == "__main__":
+    unittest.main()

tools/lib/tests/test_readers.py

@@ -8,7 +8,9 @@ import numpy as np
 from tools.lib.framereader import FrameReader
 from tools.lib.logreader import LogReader
 
+
 class TestReaders(unittest.TestCase):
+  @unittest.skip("skip for bandwith reasons")
   def test_logreader(self):
     def _check_data(lr):
       hist = defaultdict(int)
@@ -29,6 +31,7 @@ class TestReaders(unittest.TestCase):
     lr_url = LogReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true")
     _check_data(lr_url)
 
+  @unittest.skip("skip for bandwith reasons")
   def test_framereader(self):
     def _check_data(f):
       self.assertEqual(f.frame_count, 1200)