loggerd: switch logging from raw file format to zstd compressed files (#34549)

* switch logging from raw file format to zstd compressed files

* more zst suffix

* compress bootlog

* remove class RawFile

* Optimize ZstdFileWriter by adding input caching

* use ZSTD_compressStream2

* cleanup

* LOG_COMPRESSION_LEVEL=10

* space

* add zst suffix to LOGS_SIZE_RATE

tools/lib/logreader.py

@@ -38,6 +38,14 @@ def save_log(dest, log_msgs, compress=True):
   with open(dest, "wb") as f:
     f.write(dat)
 
+def decompress_stream(data: bytes):
+  dctx = zstd.ZstdDecompressor()
+  decompressed_data = b""
+
+  with dctx.stream_reader(data) as reader:
+    decompressed_data = reader.read()
+
+  return decompressed_data
 
 class _LogFileReader:
   def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False, dat=None):
@@ -58,7 +66,7 @@ class _LogFileReader:
       dat = bz2.decompress(dat)
     elif ext == ".zst" or dat.startswith(b'\x28\xB5\x2F\xFD'):
       # https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
-      dat = zstd.decompress(dat)
+      dat = decompress_stream(dat)
 
     ents = capnp_log.Event.read_multiple_bytes(dat)
 

tools/longitudinal_maneuvers/generate_report.py

@@ -159,7 +159,7 @@ if __name__ == '__main__':
     lr = LogReader(args.route)
   else:
     segs = [seg for seg in os.listdir(Paths.log_root()) if args.route in seg]
-    lr = LogReader([os.path.join(Paths.log_root(), seg, 'rlog') for seg in segs])
+    lr = LogReader([os.path.join(Paths.log_root(), seg, 'rlog.zst') for seg in segs])
 
   CP = lr.first('carParams')
   ID = lr.first('initData')