diff --git a/tools/cabana/SConscript b/tools/cabana/SConscript index 8ea59d076..90de21265 100644 --- a/tools/cabana/SConscript +++ b/tools/cabana/SConscript @@ -75,7 +75,7 @@ qt_libs = base_libs cabana_env = qt_env.Clone() -cabana_libs = [cereal, messaging, visionipc, replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil', 'x264', 'z', 'bz2', 'zstd', 'curl', 'yuv', 'usb-1.0'] + qt_libs +cabana_libs = [cereal, messaging, visionipc, replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil', 'x264', 'z', 'zstd', 'curl', 'yuv', 'usb-1.0'] + qt_libs opendbc_path = '-DOPENDBC_FILE_PATH=\'"%s"\'' % (cabana_env.Dir("../../opendbc/dbc").abspath) cabana_env['CXXFLAGS'] += [opendbc_path] diff --git a/tools/cabana/tests/test_cabana.cc b/tools/cabana/tests/test_cabana.cc index d9fcae6f2..4c11bfc8b 100644 --- a/tools/cabana/tests/test_cabana.cc +++ b/tools/cabana/tests/test_cabana.cc @@ -5,7 +5,7 @@ #include "catch2/catch.hpp" #include "tools/cabana/dbc/dbcmanager.h" -const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.bz2"; +const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.zst"; TEST_CASE("DBCFile::generateDBC") { QString fn = QString("%1/%2.dbc").arg(OPENDBC_FILE_PATH, "tesla_can"); diff --git a/tools/replay/SConscript b/tools/replay/SConscript index b39cf6dab..47b25df16 100644 --- a/tools/replay/SConscript +++ b/tools/replay/SConscript @@ -12,7 +12,7 @@ if arch != "Darwin": replay_lib_src.append("qcom_decoder.cc") replay_lib = replay_env.Library("replay", replay_lib_src, LIBS=base_libs, FRAMEWORKS=base_frameworks) Export('replay_lib') -replay_libs = [replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil', 'x264', 'z', 'bz2', 'zstd', 'curl', 'yuv', 'ncurses'] + base_libs +replay_libs = [replay_lib, 'avformat', 'avcodec', 'swresample', 'avutil', 'x264', 'z', 'zstd', 'curl', 'yuv', 'ncurses'] + base_libs replay_env.Program("replay", ["main.cc"], LIBS=replay_libs, FRAMEWORKS=base_frameworks) if GetOption('extras'): diff --git a/tools/replay/logreader.cc b/tools/replay/logreader.cc index 75abb8417..0d9e053ab 100644 --- a/tools/replay/logreader.cc +++ b/tools/replay/logreader.cc @@ -9,9 +9,7 @@ bool LogReader::load(const std::string &url, std::atomic *abort, bool local_cache, int chunk_size, int retries) { std::string data = FileReader(local_cache, chunk_size, retries).read(url, abort); if (!data.empty()) { - if (url.find(".bz2") != std::string::npos || util::starts_with(data, "BZh9")) { - data = decompressBZ2(data, abort); - } else if (url.find(".zst") != std::string::npos || util::starts_with(data, "\x28\xB5\x2F\xFD")) { + if (url.find(".zst") != std::string::npos || util::starts_with(data, "\x28\xB5\x2F\xFD")) { data = decompressZST(data, abort); } } diff --git a/tools/replay/route.cc b/tools/replay/route.cc index ba0082826..663c4b43c 100644 --- a/tools/replay/route.cc +++ b/tools/replay/route.cc @@ -174,9 +174,9 @@ void Route::addFileToSegment(int n, const std::string &file) { auto pos = name.find_last_of("--"); name = pos != std::string::npos ? name.substr(pos + 2) : name; - if (name == "rlog.bz2" || name == "rlog.zst" || name == "rlog") { + if (name == "rlog.zst" || name == "rlog") { segments_[n].rlog = file; - } else if (name == "qlog.bz2" || name == "qlog.zst" || name == "qlog") { + } else if (name == "qlog.zst" || name == "qlog") { segments_[n].qlog = file; } else if (name == "fcamera.hevc") { segments_[n].road_cam = file; diff --git a/tools/replay/tests/test_replay.cc b/tools/replay/tests/test_replay.cc index aed3de59a..f4afc2996 100644 --- a/tools/replay/tests/test_replay.cc +++ b/tools/replay/tests/test_replay.cc @@ -2,14 +2,14 @@ #include "catch2/catch.hpp" #include "tools/replay/replay.h" -const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.bz2"; +const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.zst"; TEST_CASE("LogReader") { SECTION("corrupt log") { FileReader reader(true); std::string corrupt_content = reader.read(TEST_RLOG_URL); corrupt_content.resize(corrupt_content.length() / 2); - corrupt_content = decompressBZ2(corrupt_content); + corrupt_content = decompressZST(corrupt_content); LogReader log; REQUIRE(log.load(corrupt_content.data(), corrupt_content.size())); REQUIRE(log.events.size() > 0); diff --git a/tools/replay/util.cc b/tools/replay/util.cc index 481564322..cc37c19ec 100644 --- a/tools/replay/util.cc +++ b/tools/replay/util.cc @@ -1,6 +1,5 @@ #include "tools/replay/util.h" -#include #include #include @@ -280,47 +279,6 @@ bool httpDownload(const std::string &url, const std::string &file, size_t chunk_ return httpDownload(url, of, chunk_size, size, abort); } -std::string decompressBZ2(const std::string &in, std::atomic *abort) { - return decompressBZ2((std::byte *)in.data(), in.size(), abort); -} - -std::string decompressBZ2(const std::byte *in, size_t in_size, std::atomic *abort) { - if (in_size == 0) return {}; - - bz_stream strm = {}; - int bzerror = BZ2_bzDecompressInit(&strm, 0, 0); - assert(bzerror == BZ_OK); - - strm.next_in = (char *)in; - strm.avail_in = in_size; - std::string out(in_size * 5, '\0'); - do { - strm.next_out = (char *)(&out[strm.total_out_lo32]); - strm.avail_out = out.size() - strm.total_out_lo32; - - const char *prev_write_pos = strm.next_out; - bzerror = BZ2_bzDecompress(&strm); - if (bzerror == BZ_OK && prev_write_pos == strm.next_out) { - // content is corrupt - bzerror = BZ_STREAM_END; - rWarning("decompressBZ2 error: content is corrupt"); - break; - } - - if (bzerror == BZ_OK && strm.avail_in > 0 && strm.avail_out == 0) { - out.resize(out.size() * 2); - } - } while (bzerror == BZ_OK && !(abort && *abort)); - - BZ2_bzDecompressEnd(&strm); - if (bzerror == BZ_STREAM_END && !(abort && *abort)) { - out.resize(strm.total_out_lo32); - out.shrink_to_fit(); - return out; - } - return {}; -} - std::string decompressZST(const std::string &in, std::atomic *abort) { return decompressZST((std::byte *)in.data(), in.size(), abort); } diff --git a/tools/replay/util.h b/tools/replay/util.h index 1f61951d2..fc4d2d54f 100644 --- a/tools/replay/util.h +++ b/tools/replay/util.h @@ -48,8 +48,6 @@ private: std::string sha256(const std::string &str); void precise_nano_sleep(int64_t nanoseconds, std::atomic &interrupt_requested); -std::string decompressBZ2(const std::string &in, std::atomic *abort = nullptr); -std::string decompressBZ2(const std::byte *in, size_t in_size, std::atomic *abort = nullptr); std::string decompressZST(const std::string &in, std::atomic *abort = nullptr); std::string decompressZST(const std::byte *in, size_t in_size, std::atomic *abort = nullptr); std::string getUrlWithoutQuery(const std::string &url);