loggerd: switch logging from raw file format to zstd compressed files (#34549)

* switch logging from raw file format to zstd compressed files

* more zst suffix

* compress bootlog

* remove class RawFile

* Optimize ZstdFileWriter by adding input caching

* use ZSTD_compressStream2

* cleanup

* LOG_COMPRESSION_LEVEL=10

* space

* add zst suffix to LOGS_SIZE_RATE

system/loggerd/tests/test_encoder.py

@@ -106,7 +106,7 @@ class TestEncoder:
 
         # Check encodeIdx
         if encode_idx_name is not None:
-          rlog_path = f"{route_prefix_path}--{i}/rlog"
+          rlog_path = f"{route_prefix_path}--{i}/rlog.zst"
           msgs = [m for m in LogReader(rlog_path) if m.which() == encode_idx_name]
           encode_msgs = [getattr(m, encode_idx_name) for m in msgs]
 

system/loggerd/tests/test_logger.cc

@@ -9,12 +9,13 @@ void verify_segment(const std::string &route_path, int segment, int max_segment,
   SentinelType end_sentinel = segment == max_segment - 1 ? SentinelType::END_OF_ROUTE : SentinelType::END_OF_SEGMENT;
 
   REQUIRE(!util::file_exists(segment_path + "/rlog.lock"));
-  for (const char *fn : {"/rlog", "/qlog"}) {
+  for (const char *fn : {"/rlog.zst", "/qlog.zst"}) {
     const std::string log_file = segment_path + fn;
     std::string log = util::read_file(log_file);
     REQUIRE(!log.empty());
+    std::string decompressed_log = zstd_decompress(log);
     int event_cnt = 0, i = 0;
-    kj::ArrayPtr<const capnp::word> words((capnp::word *)log.data(), log.size() / sizeof(capnp::word));
+    kj::ArrayPtr<const capnp::word> words((capnp::word *)decompressed_log.data(), decompressed_log.size() / sizeof(capnp::word));
     while (words.size() > 0) {
       try {
         capnp::FlatArrayMessageReader reader(words);

system/loggerd/tests/test_loggerd.py

@@ -142,7 +142,7 @@ class TestLoggerd:
     Params().put("RecordFront", "1")
 
     d = DEVICE_CAMERAS[("tici", "ar0231")]
-    expected_files = {"rlog", "qlog", "qcamera.ts", "fcamera.hevc", "dcamera.hevc", "ecamera.hevc"}
+    expected_files = {"rlog.zst", "qlog.zst", "qcamera.ts", "fcamera.hevc", "dcamera.hevc", "ecamera.hevc"}
     streams = [(VisionStreamType.VISION_STREAM_ROAD, (d.fcam.width, d.fcam.height, 2048*2346, 2048, 2048*1216), "roadCameraState"),
                (VisionStreamType.VISION_STREAM_DRIVER, (d.dcam.width, d.dcam.height, 2048*2346, 2048, 2048*1216), "driverCameraState"),
                (VisionStreamType.VISION_STREAM_WIDE_ROAD, (d.ecam.width, d.ecam.height, 2048*2346, 2048, 2048*1216), "wideRoadCameraState")]
@@ -229,7 +229,7 @@ class TestLoggerd:
                random.sample(no_qlog_services, random.randint(2, min(10, len(no_qlog_services))))
     sent_msgs = self._publish_random_messages(services)
 
-    qlog_path = os.path.join(self._get_latest_log_dir(), "qlog")
+    qlog_path = os.path.join(self._get_latest_log_dir(), "qlog.zst")
     lr = list(LogReader(qlog_path))
 
     # check initData and sentinel
@@ -255,7 +255,7 @@ class TestLoggerd:
     services = random.sample(CEREAL_SERVICES, random.randint(5, 10))
     sent_msgs = self._publish_random_messages(services)
 
-    lr = list(LogReader(os.path.join(self._get_latest_log_dir(), "rlog")))
+    lr = list(LogReader(os.path.join(self._get_latest_log_dir(), "rlog.zst")))
 
     # check initData and sentinel
     self._check_init_data(lr)

system/loggerd/tests/test_zstd_writer.cc

@@ -0,0 +1,38 @@
+#include <zstd.h>
+
+#include <catch2/catch.hpp>
+#include <cstring>
+#include <vector>
+
+#include "common/util.h"
+#include "system/loggerd/logger.h"
+#include "system/loggerd/zstd_writer.h"
+
+TEST_CASE("ZstdFileWriter writes and compresses data correctly in loops", "[ZstdFileWriter]") {
+  const std::string filename = "test_zstd_file.zst";
+  const int iterations = 100;
+  const size_t dataSize = 1024;
+
+  std::string totalTestData;
+
+  // Step 1: Write compressed data to file in a loop
+  {
+    ZstdFileWriter writer(filename, LOG_COMPRESSION_LEVEL);
+    for (int i = 0; i < iterations; ++i) {
+      std::string testData = util::random_string(dataSize);
+      totalTestData.append(testData);
+      writer.write((void *)testData.c_str(), testData.size());
+    }
+  }
+
+  // Step 2: Decompress the file and verify the data
+  auto compressedContent = util::read_file(filename);
+  std::string decompressedData = zstd_decompress(compressedContent);
+
+  // Step 3: Verify that the decompressed data matches the original accumulated data
+  REQUIRE(decompressedData.size() == totalTestData.size());
+  REQUIRE(std::memcmp(decompressedData.data(), totalTestData.c_str(), totalTestData.size()) == 0);
+
+  // Clean up the test file
+  std::remove(filename.c_str());
+}