uploader: compress with zstd (#32736)

* zstd uploader

* fix that

* fix name of function

* comment

* log failed

* fix comma_api_source for routes with both bz2 and zst rlogs

* TODO

* 10-14 achieves almost no benefit on qlogs in a few cases, but takes 2x the time

* these aren't written out

* regen: specify any list of sources

ooh this is pretty nice

* regen and process replay

* damn, actually we don't need all this (cool tho)

Revert "regen: specify any list of sources"

This reverts commit ceb0b4abed9ad463a9fe98d9b98a05875a52806f.

* just let it auto resolve

* fix athenad/uploader tests

* zst here too

* TODOs

* yes

* Revert "TODOs"

This reverts commit 8c7da1dbd0340c72290b5eb5563b642080ddc131.

* Revert "zst here too"

This reverts commit 23b0023ddfd22c8090be7a7caa09e7026a12aa5c.

* Revert "just let it auto resolve"

This reverts commit f296d62424227ad05facc62abc18a6f81b474e84.

* Revert "regen and process replay"

This reverts commit 0768330e96974a42616d229d159780619d049cd0.

* revert readme

* not in save_log either

* lfg

* Revert "lfg"

This reverts commit 3718559c6c4de7d1f0c80dc9f1a1d335fe679a89.
old-commit-hash: 7dec7c39bec0b7015c9cc12ee78f07a07ed47ee6

system/loggerd/uploader.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import bz2
 import io
 import json
 import os
@@ -9,6 +8,7 @@ import threading
 import time
 import traceback
 import datetime
+import zstd
 from typing import BinaryIO
 from collections.abc import Iterator
 
@@ -26,6 +26,7 @@ UPLOAD_ATTR_NAME = 'user.upload'
 UPLOAD_ATTR_VALUE = b'1'
 
 UPLOAD_QLOG_QCAM_MAX_SIZE = 5 * 1e6  # MB
+LOG_COMPRESSION_LEVEL = 10  # little benefit up to level 15. level ~17 is a small step change
 
 allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1"))
 force_wifi = os.getenv("FORCEWIFI") is not None
@@ -83,7 +84,7 @@ class Uploader:
     self.last_filename = ""
 
     self.immediate_folders = ["crash/", "boot/"]
-    self.immediate_priority = {"qlog": 0, "qlog.bz2": 0, "qcamera.ts": 1}
+    self.immediate_priority = {"qlog": 0, "qlog.zst": 0, "qcamera.ts": 1}
 
   def list_upload_files(self, metered: bool) -> Iterator[tuple[str, str, str]]:
     r = self.params.get("AthenadRecentlyViewedRoutes", encoding="utf8")
@@ -152,8 +153,8 @@ class Uploader:
 
     with open(fn, "rb") as f:
       data: BinaryIO
-      if key.endswith('.bz2') and not fn.endswith('.bz2'):
-        compressed = bz2.compress(f.read())
+      if key.endswith('.zst') and not fn.endswith('.zst'):
+        compressed = zstd.compress(f.read(), LOG_COMPRESSION_LEVEL)
         data = io.BytesIO(compressed)
       else:
         data = f
@@ -218,8 +219,8 @@ class Uploader:
     name, key, fn = d
 
     # qlogs and bootlogs need to be compressed before uploading
-    if key.endswith(('qlog', 'rlog')) or (key.startswith('boot/') and not key.endswith('.bz2')):
-      key += ".bz2"
+    if key.endswith(('qlog', 'rlog')) or (key.startswith('boot/') and not key.endswith('.zst')):
+      key += ".zst"
 
     return self.upload(name, key, fn, network_type, metered)