replay: replace Qt data types with standard C++ data types (#33823)

replace Qt data types with c++ data types

tools/cabana/streams/replaystream.cc

@@ -49,8 +49,8 @@ void ReplayStream::mergeSegments() {
 }
 
 bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) {
-  replay.reset(new Replay(route, {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"},
-                          {}, nullptr, replay_flags, data_dir, this));
+  replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"},
+                          {}, nullptr, replay_flags, data_dir.toStdString(), this));
   replay->setSegmentCacheLimit(settings.max_cached_minutes);
   replay->installEventFilter(event_filter, this);
   QObject::connect(replay.get(), &Replay::seeking, this, &AbstractStream::seeking);
@@ -153,7 +153,7 @@ AbstractStream *OpenReplayWidget::open() {
     route = route.mid(idx + 1);
   }
 
-  bool is_valid_format = Route::parseRoute(route).str.size() > 0;
+  bool is_valid_format = Route::parseRoute(route.toStdString()).str.size() > 0;
   if (!is_valid_format) {
     QMessageBox::warning(nullptr, tr("Warning"), tr("Invalid route format: '%1'").arg(route));
   } else {

tools/cabana/streams/replaystream.h

@@ -20,7 +20,7 @@ public:
   bool eventFilter(const Event *event);
   void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); }
   bool liveStreaming() const override { return false; }
-  inline QString routeName() const override { return replay->route()->name(); }
+  inline QString routeName() const override { return QString::fromStdString(replay->route()->name()); }
   inline QString carFingerprint() const override { return replay->carFingerprint().c_str(); }
   double minSeconds() const override { return replay->minSeconds(); }
   double maxSeconds() const { return replay->maxSeconds(); }

tools/replay/consoleui.cc

@@ -247,7 +247,7 @@ void ConsoleUI::updateProgressBar(uint64_t cur, uint64_t total, bool success) {
 
 void ConsoleUI::updateSummary() {
   const auto &route = replay->route();
-  mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", qPrintable(route->name()), route->segments().size());
+  mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", route->name().c_str(), route->segments().size());
   mvwprintw(w[Win::Stats], 1, 0, "Car Fingerprint: %s", replay->carFingerprint().c_str());
   wrefresh(w[Win::Stats]);
 }

tools/replay/main.cc

@@ -4,10 +4,12 @@
 #include <iostream>
 #include <map>
 #include <string>
+#include <vector>
 
 #include "common/prefix.h"
 #include "tools/replay/consoleui.h"
 #include "tools/replay/replay.h"
+#include "tools/replay/util.h"
 
 const std::string helpText =
 R"(Usage: replay [options]
@@ -32,10 +34,10 @@ Options:
 )";
 
 struct ReplayConfig {
-  QString route;
-  QStringList allow;
-  QStringList block;
-  QString data_dir;
+  std::string route;
+  std::vector<std::string> allow;
+  std::vector<std::string> block;
+  std::string data_dir;
   std::string prefix;
   uint32_t flags = REPLAY_FLAG_NONE;
   int start_seconds = 0;
@@ -84,8 +86,8 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
   int opt, option_index = 0;
   while ((opt = getopt_long(argc, argv, "a:b:c:s:x:d:p:h", cli_options, &option_index)) != -1) {
     switch (opt) {
-      case 'a': config.allow = QString(optarg).split(","); break;
-      case 'b': config.block = QString(optarg).split(","); break;
+      case 'a': config.allow = split(optarg, ','); break;
+      case 'b': config.block = split(optarg, ','); break;
       case 'c': config.cache_segments = std::atoi(optarg); break;
       case 's': config.start_seconds = std::atoi(optarg); break;
       case 'x': config.playback_speed = std::atof(optarg); break;
@@ -106,11 +108,11 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
   }
 
   // Check for a route name (first positional argument)
-  if (config.route.isEmpty() && optind < argc) {
+  if (config.route.empty() && optind < argc) {
     config.route = argv[optind];
   }
 
-  if (config.route.isEmpty()) {
+  if (config.route.empty()) {
     std::cerr << "No route provided. Use --help for usage information.\n";
     return false;
   }

tools/replay/replay.cc

@@ -11,36 +11,43 @@
 
 static void interrupt_sleep_handler(int signal) {}
 
-Replay::Replay(QString route, QStringList allow, QStringList block, SubMaster *sm_,
-               uint32_t flags, QString data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) {
+Replay::Replay(const std::string &route, std::vector<std::string> allow, std::vector<std::string> block, SubMaster *sm_,
+               uint32_t flags, const std::string &data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) {
   // Register signal handler for SIGUSR1
   std::signal(SIGUSR1, interrupt_sleep_handler);
 
   if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) {
-    block << "uiDebug" << "userFlag";
+    block.insert(block.end(), {"uiDebug", "userFlag"});
   }
-  auto event_struct = capnp::Schema::from<cereal::Event>().asStruct();
-  sockets_.resize(event_struct.getUnionFields().size());
+
+  auto event_schema = capnp::Schema::from<cereal::Event>().asStruct();
+  sockets_.resize(event_schema.getUnionFields().size());
+  std::vector<std::string> active_services;
+
   for (const auto &[name, _] : services) {
-    if (!block.contains(name.c_str()) && (allow.empty() || allow.contains(name.c_str()))) {
-      uint16_t which = event_struct.getFieldByName(name).getProto().getDiscriminantValue();
+    bool in_block = std::find(block.begin(), block.end(), name) != block.end();
+    bool in_allow = std::find(allow.begin(), allow.end(), name) != allow.end();
+    if (!in_block && (allow.empty() || in_allow)) {
+      uint16_t which = event_schema.getFieldByName(name).getProto().getDiscriminantValue();
       sockets_[which] = name.c_str();
+      active_services.push_back(name);
     }
   }
-  if (!allow.isEmpty()) {
+
+  if (!allow.empty()) {
     for (int i = 0; i < sockets_.size(); ++i) {
       filters_.push_back(i == cereal::Event::Which::INIT_DATA || i == cereal::Event::Which::CAR_PARAMS || sockets_[i]);
     }
   }
 
-  std::vector<const char *> s;
-  std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(s),
-               [](const char *name) { return name != nullptr; });
-  qDebug() << "services " << s;
-  qDebug() << "loading route " << route;
+  rInfo("active services: %s", join(active_services, ',').c_str());
+  rInfo("loading route %s", route.c_str());
 
   if (sm == nullptr) {
-    pm = std::make_unique<PubMaster>(s);
+    std::vector<const char *> socket_names;
+    std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(socket_names),
+                 [](const char *name) { return name != nullptr; });
+    pm = std::make_unique<PubMaster>(socket_names);
   }
   route_ = std::make_unique<Route>(route, data_dir);
 }
@@ -68,23 +75,23 @@ void Replay::stop() {
 
 bool Replay::load() {
   if (!route_->load()) {
-    qCritical() << "failed to load route" << route_->name()
-                << "from" << (route_->dir().isEmpty() ? "server" : route_->dir());
+    rError("failed to load route %s from %s", route_->name().c_str(),
+           route_->dir().empty() ? "server" : route_->dir().c_str());
     return false;
   }
 
   for (auto &[n, f] : route_->segments()) {
-    bool has_log = !f.rlog.isEmpty() || !f.qlog.isEmpty();
-    bool has_video = !f.road_cam.isEmpty() || !f.qcamera.isEmpty();
+    bool has_log = !f.rlog.empty() || !f.qlog.empty();
+    bool has_video = !f.road_cam.empty() || !f.qcamera.empty();
     if (has_log && (has_video || hasFlag(REPLAY_FLAG_NO_VIPC))) {
       segments_.insert({n, nullptr});
     }
   }
   if (segments_.empty()) {
-    qCritical() << "no valid segments in route" << route_->name();
+    rInfo("no valid segments in route: %s", route_->name().c_str());
     return false;
   }
-  rInfo("load route %s with %zu valid segments", qPrintable(route_->name()), segments_.size());
+  rInfo("load route %s with %zu valid segments", route_->name().c_str(), segments_.size());
   max_seconds_ = (segments_.rbegin()->first + 1) * 60;
   return true;
 }
@@ -167,7 +174,7 @@ void Replay::buildTimeline() {
   const auto &route_segments = route_->segments();
   for (auto it = route_segments.cbegin(); it != route_segments.cend() && !exit_; ++it) {
     std::shared_ptr<LogReader> log(new LogReader());
-    if (!log->load(it->second.qlog.toStdString(), &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue;
+    if (!log->load(it->second.qlog, &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue;
 
     std::vector<std::tuple<double, double, TimelineType>> timeline;
     for (const Event &e : log->events) {

tools/replay/replay.h

@@ -15,7 +15,7 @@
 #include "tools/replay/camera.h"
 #include "tools/replay/route.h"
 
-const QString DEMO_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19";
+#define DEMO_ROUTE "a2a0ccea32023010|2023-07-27--13-01-19"
 
 // one segment uses about 100M of memory
 constexpr int MIN_SEGMENTS_CACHE = 5;
@@ -49,8 +49,8 @@ class Replay : public QObject {
   Q_OBJECT
 
 public:
-  Replay(QString route, QStringList allow, QStringList block, SubMaster *sm = nullptr,
-         uint32_t flags = REPLAY_FLAG_NONE, QString data_dir = "", QObject *parent = 0);
+  Replay(const std::string &route, std::vector<std::string> allow, std::vector<std::string> block, SubMaster *sm = nullptr,
+         uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "", QObject *parent = 0);
   ~Replay();
   bool load();
   RouteLoadError lastRouteError() const { return route_->lastError(); }

tools/replay/route.cc

@@ -4,35 +4,44 @@
 #include <QEventLoop>
 #include <QJsonArray>
 #include <QJsonDocument>
-#include <QRegularExpression>
 #include <QtConcurrent>
 #include <array>
+#include <filesystem>
+#include <regex>
 
 #include "selfdrive/ui/qt/api.h"
 #include "system/hardware/hw.h"
 #include "tools/replay/replay.h"
 #include "tools/replay/util.h"
 
-Route::Route(const QString &route, const QString &data_dir) : data_dir_(data_dir) {
+Route::Route(const std::string &route, const std::string &data_dir) : data_dir_(data_dir) {
   route_ = parseRoute(route);
 }
 
-RouteIdentifier Route::parseRoute(const QString &str) {
+RouteIdentifier Route::parseRoute(const std::string &str) {
   RouteIdentifier identifier = {};
-  QRegularExpression rx(R"(^((?<dongle_id>[a-z0-9]{16})[|_/])?(?<timestamp>.{20})((?<separator>--|/)(?<range>((-?\d+(:(-?\d+)?)?)|(:-?\d+))))?$)");
-  if (auto match = rx.match(str); match.hasMatch()) {
-    identifier.dongle_id = match.captured("dongle_id");
-    identifier.timestamp = match.captured("timestamp");
+  static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?$)");
+  std::smatch match;
+
+  if (std::regex_match(str, match, pattern)) {
+    identifier.dongle_id = match[2].str();
+    identifier.timestamp = match[3].str();
     identifier.str = identifier.dongle_id + "|" + identifier.timestamp;
-    auto range_str = match.captured("range");
-    if (auto separator = match.captured("separator"); separator == "/" && !range_str.isEmpty()) {
-      auto range = range_str.split(":");
-      identifier.begin_segment = identifier.end_segment = range[0].toInt();
-      if (range.size() == 2) {
-        identifier.end_segment = range[1].isEmpty() ? -1 : range[1].toInt();
+
+    const auto separator = match[5].str();
+    const auto range_str = match[6].str();
+    if (!range_str.empty()) {
+      if (separator == "/") {
+        int pos = range_str.find(':');
+        int begin_seg = std::stoi(range_str.substr(0, pos));
+        identifier.begin_segment = identifier.end_segment = begin_seg;
+        if (pos != std::string::npos) {
+          auto end_seg_str = range_str.substr(pos + 1);
+          identifier.end_segment = end_seg_str.empty() ? -1 : std::stoi(end_seg_str);
+        }
+      } else if (separator == "--") {
+        identifier.begin_segment = std::atoi(range_str.c_str());
       }
-    } else if (separator == "--") {
-      identifier.begin_segment = range_str.toInt();
     }
   }
   return identifier;
@@ -40,12 +49,12 @@ RouteIdentifier Route::parseRoute(const QString &str) {
 
 bool Route::load() {
   err_ = RouteLoadError::None;
-  if (route_.str.isEmpty() || (data_dir_.isEmpty() && route_.dongle_id.isEmpty())) {
+  if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) {
     rInfo("invalid route format");
     return false;
   }
-  date_time_ = QDateTime::fromString(route_.timestamp, "yyyy-MM-dd--HH-mm-ss");
-  bool ret = data_dir_.isEmpty() ? loadFromServer() : loadFromLocal();
+  date_time_ = QDateTime::fromString(route_.timestamp.c_str(), "yyyy-MM-dd--HH-mm-ss");
+  bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal();
   if (ret) {
     if (route_.begin_segment == -1) route_.begin_segment = segments_.rbegin()->first;
     if (route_.end_segment == -1) route_.end_segment = segments_.rbegin()->first;
@@ -69,7 +78,7 @@ bool Route::loadFromServer(int retries) {
       result = json;
       loop.exit((int)err);
     });
-    http.sendRequest(CommaApi::BASE_URL + "/v1/route/" + route_.str + "/files");
+    http.sendRequest(CommaApi::BASE_URL + "/v1/route/" + QString::fromStdString(route_.str) + "/files");
     auto err = (QNetworkReply::NetworkError)loop.exec();
     if (err == QNetworkReply::NoError) {
       return loadFromJson(result);
@@ -96,7 +105,7 @@ bool Route::loadFromJson(const QString &json) {
     for (const auto &url : value.toArray()) {
       QString url_str = url.toString();
       if (rx.indexIn(url_str) != -1) {
-        addFileToSegment(rx.cap(1).toInt(), url_str);
+        addFileToSegment(rx.cap(1).toInt(), url_str.toStdString());
       }
     }
   }
@@ -104,23 +113,26 @@ bool Route::loadFromJson(const QString &json) {
 }
 
 bool Route::loadFromLocal() {
-  QDirIterator it(data_dir_, {QString("%1--*").arg(route_.timestamp)}, QDir::Dirs | QDir::NoDotAndDotDot);
-  while (it.hasNext()) {
-    QString segment = it.next();
-    const int seg_num = segment.mid(segment.lastIndexOf("--") + 2).toInt();
-    QDir segment_dir(segment);
-    for (const auto &f : segment_dir.entryList(QDir::Files)) {
-      addFileToSegment(seg_num, segment_dir.absoluteFilePath(f));
+  std::string pattern = route_.timestamp + "--";
+  for (const auto &entry : std::filesystem::directory_iterator(data_dir_)) {
+    if (entry.is_directory() && entry.path().filename().string().find(pattern) == 0) {
+      std::string segment = entry.path().string();
+      int seg_num = std::atoi(segment.substr(segment.rfind("--") + 2).c_str());
+
+      for (const auto &file : std::filesystem::directory_iterator(segment)) {
+        if (file.is_regular_file()) {
+          addFileToSegment(seg_num, file.path().string());
+        }
+      }
     }
   }
   return !segments_.empty();
 }
 
-void Route::addFileToSegment(int n, const QString &file) {
-  QString name = QUrl(file).fileName();
-
-  const int pos = name.lastIndexOf("--");
-  name = pos != -1 ? name.mid(pos + 2) : name;
+void Route::addFileToSegment(int n, const std::string &file) {
+  std::string name = extractFileName(file);
+  auto pos = name.find_last_of("--");
+  name = pos != std::string::npos ? name.substr(pos + 2) : name;
 
   if (name == "rlog.bz2" || name == "rlog.zst" || name == "rlog") {
     segments_[n].rlog = file;
@@ -143,15 +155,15 @@ Segment::Segment(int n, const SegmentFile &files, uint32_t flags, const std::vec
     : seg_num(n), flags(flags), filters_(filters) {
   // [RoadCam, DriverCam, WideRoadCam, log]. fallback to qcamera/qlog
   const std::array file_list = {
-      (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.isEmpty() ? files.qcamera : files.road_cam,
+      (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.empty() ? files.qcamera : files.road_cam,
       flags & REPLAY_FLAG_DCAM ? files.driver_cam : "",
       flags & REPLAY_FLAG_ECAM ? files.wide_road_cam : "",
-      files.rlog.isEmpty() ? files.qlog : files.rlog,
+      files.rlog.empty() ? files.qlog : files.rlog,
   };
   for (int i = 0; i < file_list.size(); ++i) {
-    if (!file_list[i].isEmpty() && (!(flags & REPLAY_FLAG_NO_VIPC) || i >= MAX_CAMERAS)) {
+    if (!file_list[i].empty() && (!(flags & REPLAY_FLAG_NO_VIPC) || i >= MAX_CAMERAS)) {
       ++loading_;
-      synchronizer_.addFuture(QtConcurrent::run(this, &Segment::loadFile, i, file_list[i].toStdString()));
+      synchronizer_.addFuture(QtConcurrent::run(this, &Segment::loadFile, i, file_list[i]));
     }
   }
 }

tools/replay/route.h

@@ -21,42 +21,42 @@ enum class RouteLoadError {
 };
 
 struct RouteIdentifier {
-  QString dongle_id;
-  QString timestamp;
+  std::string dongle_id;
+  std::string timestamp;
   int begin_segment = 0;
   int end_segment = -1;
-  QString str;
+  std::string str;
 };
 
 struct SegmentFile {
-  QString rlog;
-  QString qlog;
-  QString road_cam;
-  QString driver_cam;
-  QString wide_road_cam;
-  QString qcamera;
+  std::string rlog;
+  std::string qlog;
+  std::string road_cam;
+  std::string driver_cam;
+  std::string wide_road_cam;
+  std::string qcamera;
 };
 
 class Route {
 public:
-  Route(const QString &route, const QString &data_dir = {});
+  Route(const std::string &route, const std::string &data_dir = {});
   bool load();
   RouteLoadError lastError() const { return err_; }
-  inline const QString &name() const { return route_.str; }
+  inline const std::string &name() const { return route_.str; }
   inline const QDateTime datetime() const { return date_time_; }
-  inline const QString &dir() const { return data_dir_; }
+  inline const std::string &dir() const { return data_dir_; }
   inline const RouteIdentifier &identifier() const { return route_; }
   inline const std::map<int, SegmentFile> &segments() const { return segments_; }
   inline const SegmentFile &at(int n) { return segments_.at(n); }
-  static RouteIdentifier parseRoute(const QString &str);
+  static RouteIdentifier parseRoute(const std::string &str);
 
 protected:
   bool loadFromLocal();
   bool loadFromServer(int retries = 3);
   bool loadFromJson(const QString &json);
-  void addFileToSegment(int seg_num, const QString &file);
+  void addFileToSegment(int seg_num, const std::string &file);
   RouteIdentifier route_ = {};
-  QString data_dir_;
+  std::string data_dir_;
   std::map<int, SegmentFile> segments_;
   QDateTime date_time_;
   RouteLoadError err_ = RouteLoadError::None;

tools/replay/tests/test_replay.cc

@@ -122,12 +122,12 @@ std::string download_demo_route() {
     assert(remote_route.load());
 
     // Create a local route from remote for testing
-    const std::string route_name = DEMO_ROUTE.mid(17).toStdString();
+    const std::string route_name = std::string(DEMO_ROUTE).substr(17);
     for (int i = 0; i < 2; ++i) {
       std::string log_path = util::string_format("%s/%s--%d/", data_dir.c_str(), route_name.c_str(), i);
       util::create_directories(log_path, 0755);
-      REQUIRE(download_to_file(remote_route.at(i).rlog.toStdString(), log_path + "rlog.bz2"));
-      REQUIRE(download_to_file(remote_route.at(i).qcamera.toStdString(), log_path + "qcamera.ts"));
+      REQUIRE(download_to_file(remote_route.at(i).rlog, log_path + "rlog.bz2"));
+      REQUIRE(download_to_file(remote_route.at(i).qcamera, log_path + "qcamera.ts"));
     }
   }
 
@@ -139,7 +139,7 @@ TEST_CASE("Local route") {
   std::string data_dir = download_demo_route();
 
   auto flags = GENERATE(0, REPLAY_FLAG_QCAMERA);
-  Route route(DEMO_ROUTE, QString::fromStdString(data_dir));
+  Route route(DEMO_ROUTE, data_dir);
   REQUIRE(route.load());
   REQUIRE(route.segments().size() == 2);
   for (int i = 0; i < TEST_REPLAY_SEGMENTS; ++i) {

tools/replay/util.cc

@@ -14,6 +14,7 @@
 #include <map>
 #include <mutex>
 #include <numeric>
+#include <sstream>
 #include <utility>
 #include <zstd.h>
 
@@ -390,6 +391,35 @@ std::string sha256(const std::string &str) {
   return util::hexdump(hash, SHA256_DIGEST_LENGTH);
 }
 
+std::vector<std::string> split(std::string_view source, char delimiter) {
+  std::vector<std::string> fields;
+  size_t last = 0;
+  for (size_t i = 0; i < source.length(); ++i) {
+    if (source[i] == delimiter) {
+      fields.emplace_back(source.substr(last, i - last));
+      last = i + 1;
+    }
+  }
+  fields.emplace_back(source.substr(last));
+  return fields;
+}
+
+std::string join(const std::vector<std::string> &elements, char separator) {
+  std::ostringstream oss;
+  for (size_t i = 0; i < elements.size(); ++i) {
+    if (i != 0) oss << separator;
+    oss << elements[i];
+  }
+  return oss.str();
+}
+
+std::string extractFileName(const std::string &file) {
+  size_t queryPos = file.find_first_of("?");
+  std::string path = (queryPos != std::string::npos) ? file.substr(0, queryPos) : file;
+  size_t lastSlash = path.find_last_of("/\\");
+  return (lastSlash != std::string::npos) ? path.substr(lastSlash + 1) : path;
+}
+
 // MonotonicBuffer
 
 void *MonotonicBuffer::allocate(size_t bytes, size_t alignment) {

tools/replay/util.h

@@ -4,6 +4,8 @@
 #include <deque>
 #include <functional>
 #include <string>
+#include <string_view>
+#include <vector>
 #include "cereal/messaging/messaging.h"
 
 enum CameraType {
@@ -57,3 +59,6 @@ typedef std::function<void(uint64_t cur, uint64_t total, bool success)> Download
 void installDownloadProgressHandler(DownloadProgressHandler);
 bool httpDownload(const std::string &url, const std::string &file, size_t chunk_size = 0, std::atomic<bool> *abort = nullptr);
 std::string formattedDataSize(size_t size);
+std::vector<std::string> split(std::string_view source, char delimiter);
+std::string join(const std::vector<std::string> &elements, char separator);
+std::string extractFileName(const std::string& file);