cabana: split comma API route fetching out of RoutesDialog (#38721)

openpilot/tools/cabana/SConscript

@@ -99,7 +99,7 @@ cabana_env.Command(assets, "assets/assets.qrc", f"rcc $SOURCES -o $TARGET")
 cabana_env.Depends(assets, Glob('/assets/*', exclude=[assets, "assets/assets.o"]))
 
 cabana_srcs = ['mainwin.cc', 'streams/pandastream.cc', 'streams/devicestream.cc', 'streams/livestream.cc', 'streams/abstractstream.cc', 'streams/replaystream.cc', 'binaryview.cc', 'historylog.cc', 'videowidget.cc', 'signalview.cc',
-               'routesdialog.cc', 'dbc/dbc.cc', 'dbc/dbcfile.cc', 'dbc/dbcmanager.cc',
+               'routesdialog.cc', 'routes.cc', 'dbc/dbc.cc', 'dbc/dbcfile.cc', 'dbc/dbcmanager.cc',
                'utils/export.cc', 'utils/util.cc', 'utils/strings.cc', 'utils/elidedlabel.cc',
                'chart/chartswidget.cc', 'chart/chart.cc', 'chart/signalselector.cc', 'chart/tiplabel.cc', 'chart/sparkline.cc',
                'commands.cc', 'messageswidget.cc', 'streamselector.cc', 'settings.cc', 'settingsdialog.cc', 'panda.cc',
@@ -120,8 +120,9 @@ if GetOption('extras'):
     dbc_core_test_env.Object('tests/dbc_core_file', 'dbc/dbcfile.cc'),
     dbc_core_test_env.Object('tests/dbc_core_manager', 'dbc/dbcmanager.cc'),
     dbc_core_test_env.Object('tests/dbc_core_strings', 'utils/strings.cc'),
+    dbc_core_test_env.Object('tests/dbc_core_routes', 'routes.cc'),
   ]
-  dbc_core_test_env.Program('tests/test_dbc_core', dbc_core_test_objects)
+  dbc_core_test_env.Program('tests/test_dbc_core', dbc_core_test_objects, LIBS=[replay_lib, common])
 
 output_json_file = 'openpilot/tools/cabana/dbc/car_fingerprint_to_dbc.json'
 generate_dbc = cabana_env.Command('#' + output_json_file,

openpilot/tools/cabana/routes.cc

@@ -0,0 +1,115 @@
+#include "tools/cabana/routes.h"
+
+#include <chrono>
+#include <cstdlib>
+#include <ctime>
+#include <thread>
+
+#include "json11/json11.hpp"
+#include "tools/replay/py_downloader.h"
+
+namespace routes {
+
+std::pair<bool, int> checkApiResponse(const std::string &result) {
+  if (result.empty()) return {false, 500};
+  std::string err;
+  auto doc = json11::Json::parse(result, err);
+  if (!err.empty()) return {false, 500};
+  if (doc.is_object() && doc["error"].is_string()) {
+    return {false, doc["error"].string_value() == "unauthorized" ? 401 : 500};
+  }
+  return {true, 0};
+}
+
+int64_t nowUnixMs() {
+  return std::chrono::duration_cast<std::chrono::milliseconds>(
+             std::chrono::system_clock::now().time_since_epoch())
+      .count();
+}
+
+int64_t parseIsoToUnixMs(const std::string &s) {
+  std::string bytes = s;
+  if (!bytes.empty() && (bytes.back() == 'Z' || bytes.back() == 'z')) bytes.pop_back();
+  int millis = 0;
+  auto dot = bytes.find('.');
+  if (dot != std::string::npos) {
+    std::string frac = bytes.substr(dot + 1);
+    bytes = bytes.substr(0, dot);
+    while (frac.size() < 3) frac.push_back('0');
+    millis = std::atoi(frac.substr(0, 3).c_str());
+  }
+  std::tm tm{};
+  const char *ret = strptime(bytes.c_str(), "%Y-%m-%dT%H:%M:%S", &tm);
+  if (!ret) ret = strptime(bytes.c_str(), "%Y-%m-%d %H:%M:%S", &tm);
+  if (!ret) return 0;
+  time_t secs = timegm(&tm);
+  if (secs == static_cast<time_t>(-1)) return 0;
+  return static_cast<int64_t>(secs) * 1000 + millis;
+}
+
+std::string formatUnixMs(int64_t ms) {
+  time_t secs = static_cast<time_t>(ms / 1000);
+  std::tm tm{};
+  localtime_r(&secs, &tm);
+  char buf[64];
+  std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tm);
+  return buf;
+}
+
+std::vector<DeviceInfo> parseDevices(const std::string &json) {
+  std::vector<DeviceInfo> devices;
+  std::string err;
+  auto doc = json11::Json::parse(json, err);
+  if (err.empty() && doc.is_array()) {
+    for (const auto &device : doc.array_items()) {
+      devices.push_back({device["dongle_id"].string_value()});
+    }
+  }
+  return devices;
+}
+
+std::vector<RouteInfo> parseRoutes(const std::string &json, bool preserved) {
+  std::vector<RouteInfo> items;
+  std::string err;
+  auto doc = json11::Json::parse(json, err);
+  if (err.empty() && doc.is_array()) {
+    for (const auto &route : doc.array_items()) {
+      RouteInfo info;
+      info.name = route["fullname"].string_value();
+      if (preserved) {
+        info.start_ms = parseIsoToUnixMs(route["start_time"].string_value());
+        info.end_ms = parseIsoToUnixMs(route["end_time"].string_value());
+      } else {
+        info.start_ms = static_cast<int64_t>(route["start_time_utc_millis"].number_value());
+        info.end_ms = static_cast<int64_t>(route["end_time_utc_millis"].number_value());
+      }
+      items.push_back(std::move(info));
+    }
+  }
+  return items;
+}
+
+void fetchDevices(DevicesCallback callback) {
+  std::thread([callback = std::move(callback)]() {
+    std::string result = PyDownloader::getDevices();
+    auto [success, error_code] = checkApiResponse(result);
+    callback(success ? parseDevices(result) : std::vector<DeviceInfo>{}, success, error_code);
+  }).detach();
+}
+
+void fetchRoutes(const std::string &dongle_id, int period_days, RoutesCallback callback) {
+  const bool preserved = period_days == -1;
+  int64_t start_ms = 0, end_ms = 0;
+  if (!preserved) {
+    end_ms = nowUnixMs();
+    start_ms = end_ms - static_cast<int64_t>(period_days) * 24LL * 60LL * 60LL * 1000LL;
+  }
+
+  std::thread([dongle_id, start_ms, end_ms, preserved, callback = std::move(callback)]() {
+    std::string result = PyDownloader::getDeviceRoutes(dongle_id, start_ms, end_ms, preserved);
+    auto [success, error_code] = checkApiResponse(result);
+    callback(success ? parseRoutes(result, preserved) : std::vector<RouteInfo>{}, success, error_code);
+  }).detach();
+}
+
+}  // namespace routes

openpilot/tools/cabana/routes.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace routes {
+
+struct DeviceInfo {
+  std::string dongle_id;
+};
+
+struct RouteInfo {
+  std::string name;
+  int64_t start_ms = 0;
+  int64_t end_ms = 0;
+};
+
+using DevicesCallback = std::function<void(std::vector<DeviceInfo> devices, bool success, int error_code)>;
+using RoutesCallback = std::function<void(std::vector<RouteInfo> routes, bool success, int error_code)>;
+
+// Parse a PyDownloader JSON response into (success, error_code).
+std::pair<bool, int> checkApiResponse(const std::string &result);
+
+int64_t nowUnixMs();
+// Parse ISO-8601 (with optional fractional seconds / Z) to unix ms. Returns 0 on failure.
+int64_t parseIsoToUnixMs(const std::string &s);
+// Local time, "%Y-%m-%d %H:%M:%S".
+std::string formatUnixMs(int64_t ms);
+
+std::vector<DeviceInfo> parseDevices(const std::string &json);
+// preserved routes report ISO-8601 timestamps instead of unix millis
+std::vector<RouteInfo> parseRoutes(const std::string &json, bool preserved);
+
+// Both fetch on a detached thread and invoke the callback from that thread.
+void fetchDevices(DevicesCallback callback);
+// period_days of -1 requests preserved routes
+void fetchRoutes(const std::string &dongle_id, int period_days, RoutesCallback callback);
+
+}  // namespace routes

openpilot/tools/cabana/routesdialog.cc

@@ -1,9 +1,6 @@
 #include "tools/cabana/routesdialog.h"
 
-#include <chrono>
-#include <ctime>
 #include <string>
-#include <thread>
 #include <utility>
 
 #include <QDialogButtonBox>
@@ -12,62 +9,7 @@
 #include <QMessageBox>
 #include <QPainter>
 
-#include "json11/json11.hpp"
 #include "tools/cabana/utils/util.h"
-#include "tools/replay/py_downloader.h"
-
-namespace {
-
-// Parse a PyDownloader JSON response into (success, error_code).
-std::pair<bool, int> checkApiResponse(const std::string &result) {
-  if (result.empty()) return {false, 500};
-  std::string err;
-  auto doc = json11::Json::parse(result, err);
-  if (!err.empty()) return {false, 500};
-  if (doc.is_object() && doc["error"].is_string()) {
-    return {false, doc["error"].string_value() == "unauthorized" ? 401 : 500};
-  }
-  return {true, 0};
-}
-
-int64_t nowUnixMs() {
-  return std::chrono::duration_cast<std::chrono::milliseconds>(
-             std::chrono::system_clock::now().time_since_epoch())
-      .count();
-}
-
-// Parse ISO-8601 (with optional fractional seconds / Z) to unix ms. Returns 0 on failure.
-int64_t parseIsoToUnixMs(const std::string &s) {
-  std::string bytes = s;
-  if (!bytes.empty() && (bytes.back() == 'Z' || bytes.back() == 'z')) bytes.pop_back();
-  int millis = 0;
-  auto dot = bytes.find('.');
-  if (dot != std::string::npos) {
-    std::string frac = bytes.substr(dot + 1);
-    bytes = bytes.substr(0, dot);
-    while (frac.size() < 3) frac.push_back('0');
-    millis = std::atoi(frac.substr(0, 3).c_str());
-  }
-  std::tm tm{};
-  const char *ret = strptime(bytes.c_str(), "%Y-%m-%dT%H:%M:%S", &tm);
-  if (!ret) ret = strptime(bytes.c_str(), "%Y-%m-%d %H:%M:%S", &tm);
-  if (!ret) return 0;
-  tm.tm_isdst = -1;
-  time_t secs = timegm(&tm);
-  if (secs == static_cast<time_t>(-1)) return 0;
-  return static_cast<int64_t>(secs) * 1000 + millis;
-}
-
-std::string formatUnixMs(int64_t ms) {
-  time_t secs = static_cast<time_t>(ms / 1000);
-  std::tm tm{};
-  localtime_r(&secs, &tm);
-  char buf[64];
-  std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tm);
-  return buf;
-}
-
-}  // namespace
 
 // The RouteListWidget class extends QListWidget to display a custom message when empty
 class RouteListWidget : public QListWidget {
@@ -110,26 +52,19 @@ RoutesDialog::RoutesDialog(QWidget *parent) : QDialog(parent) {
   connect(button_box, &QDialogButtonBox::accepted, this, &QDialog::accept);
   connect(button_box, &QDialogButtonBox::rejected, this, &QDialog::reject);
 
-  // Fetch devices
-  std::thread([this, alive = std::weak_ptr<bool>(alive_)]() {
-    std::string result = PyDownloader::getDevices();
-    auto response = checkApiResponse(result);
-    utils::runOnMainThread([this, alive, r = std::move(result), response]() {
-      if (!alive.expired()) parseDeviceList(r, response.first, response.second);
+  routes::fetchDevices([this, alive = std::weak_ptr<bool>(alive_)](std::vector<routes::DeviceInfo> devices, bool success, int error_code) {
+    utils::runOnMainThread([this, alive, devices = std::move(devices), success, error_code]() {
+      if (!alive.expired()) setDeviceList(devices, success, error_code);
     });
-  }).detach();
+  });
 }
 
-void RoutesDialog::parseDeviceList(const std::string &json, bool success, int error_code) {
+void RoutesDialog::setDeviceList(const std::vector<routes::DeviceInfo> &devices, bool success, int error_code) {
   if (success) {
     device_list_->clear();
-    std::string err;
-    auto doc = json11::Json::parse(json, err);
-    if (err.empty() && doc.is_array()) {
-      for (const auto &device : doc.array_items()) {
-        QString dongle_id = QString::fromStdString(device["dongle_id"].string_value());
-        device_list_->addItem(dongle_id, dongle_id);
-      }
+    for (const auto &device : devices) {
+      QString dongle_id = QString::fromStdString(device.dongle_id);
+      device_list_->addItem(dongle_id, dongle_id);
     }
   } else {
     QMessageBox::warning(this, tr("Error"), error_code == 401 ? tr("Unauthorized. Authenticate with openpilot/tools/lib/auth.py") : tr("Network error"));
@@ -144,45 +79,22 @@ void RoutesDialog::fetchRoutes() {
   route_list_->clear();
   route_list_->setEmptyText(tr("Loading..."));
 
-  std::string did = device_list_->currentText().toStdString();
-  int period = period_selector_->currentData().toInt();
-
-  bool preserved = (period == -1);
-  int64_t start_ms = 0, end_ms = 0;
-  if (!preserved) {
-    end_ms = nowUnixMs();
-    start_ms = end_ms - static_cast<int64_t>(period) * 24LL * 60LL * 60LL * 1000LL;
-  }
-
   int request_id = ++fetch_id_;
-  std::thread([this, alive = std::weak_ptr<bool>(alive_), did, start_ms, end_ms, preserved, request_id]() {
-    std::string result = PyDownloader::getDeviceRoutes(did, start_ms, end_ms, preserved);
-    auto response = checkApiResponse(result);
-    utils::runOnMainThread([this, alive, r = std::move(result), response, request_id]() {
-      if (!alive.expired() && fetch_id_ == request_id) parseRouteList(r, response.first, response.second);
+  auto on_routes = [this, alive = std::weak_ptr<bool>(alive_), request_id](std::vector<routes::RouteInfo> list, bool success, int) {
+    utils::runOnMainThread([this, alive, list = std::move(list), success, request_id]() {
+      if (!alive.expired() && fetch_id_ == request_id) setRouteList(list, success);
     });
-  }).detach();
+  };
+  routes::fetchRoutes(device_list_->currentText().toStdString(), period_selector_->currentData().toInt(), std::move(on_routes));
 }
 
-void RoutesDialog::parseRouteList(const std::string &json, bool success, int error_code) {
+void RoutesDialog::setRouteList(const std::vector<routes::RouteInfo> &list, bool success) {
   if (success) {
-    std::string err;
-    auto doc = json11::Json::parse(json, err);
-    if (err.empty() && doc.is_array()) {
-      for (const auto &route : doc.array_items()) {
-        int64_t from_ms = 0, to_ms = 0;
-        if (period_selector_->currentData().toInt() == -1) {
-          from_ms = parseIsoToUnixMs(route["start_time"].string_value());
-          to_ms = parseIsoToUnixMs(route["end_time"].string_value());
-        } else {
-          from_ms = static_cast<int64_t>(route["start_time_utc_millis"].number_value());
-          to_ms = static_cast<int64_t>(route["end_time_utc_millis"].number_value());
-        }
-        const int mins = static_cast<int>((to_ms - from_ms) / 60000);
-        auto item = new QListWidgetItem(QString::fromStdString(formatUnixMs(from_ms) + "    " + std::to_string(mins) + "min"));
-        item->setData(Qt::UserRole, QString::fromStdString(route["fullname"].string_value()));
-        route_list_->addItem(item);
-      }
+    for (const auto &route : list) {
+      const int mins = static_cast<int>((route.end_ms - route.start_ms) / 60000);
+      auto item = new QListWidgetItem(QString::fromStdString(routes::formatUnixMs(route.start_ms) + "    " + std::to_string(mins) + "min"));
+      item->setData(Qt::UserRole, QString::fromStdString(route.name));
+      route_list_->addItem(item);
     }
     if (route_list_->count() > 0) route_list_->setCurrentRow(0);
   } else {

openpilot/tools/cabana/routesdialog.h

@@ -2,10 +2,13 @@
 
 #include <atomic>
 #include <memory>
+#include <vector>
 
 #include <QComboBox>
 #include <QDialog>
 
+#include "tools/cabana/routes.h"
+
 class RouteListWidget;
 
 class RoutesDialog : public QDialog {
@@ -15,8 +18,8 @@ public:
   std::string route();
 
 protected:
-  void parseDeviceList(const std::string &json, bool success, int error_code);
-  void parseRouteList(const std::string &json, bool success, int error_code);
+  void setDeviceList(const std::vector<routes::DeviceInfo> &devices, bool success, int error_code);
+  void setRouteList(const std::vector<routes::RouteInfo> &list, bool success);
   void fetchRoutes();
 
   QComboBox *device_list_;

openpilot/tools/cabana/tests/test_cabana.cc

@@ -7,6 +7,7 @@
 #include "common/tests/native_test.h"
 #include "tools/cabana/dbc/dbcfile.h"
 #include "tools/cabana/dbc/dbcmanager.h"
+#include "tools/cabana/routes.h"
 #include "tools/cabana/utils/strings.h"
 
 const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.bz2";
@@ -240,6 +241,64 @@ void test_signal_tooltip() {
   )");
 }
 
+void test_route_timestamps() {
+  REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05Z") == 1704164645000);
+  REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05") == 1704164645000);
+  REQUIRE(routes::parseIsoToUnixMs("2024-01-02 03:04:05") == 1704164645000);
+  REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05.123Z") == 1704164645123);
+  REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05.4Z") == 1704164645400);
+  REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05.123456Z") == 1704164645123);
+  REQUIRE(routes::parseIsoToUnixMs("") == 0);
+  REQUIRE(routes::parseIsoToUnixMs("not a timestamp") == 0);
+
+  // formatUnixMs is local time
+  const char *tz = getenv("TZ");
+  const std::string prev_tz = tz ? tz : "";
+  setenv("TZ", "UTC", 1);
+  tzset();
+  REQUIRE(routes::formatUnixMs(1704164645123) == "2024-01-02 03:04:05");
+  if (tz) {
+    setenv("TZ", prev_tz.c_str(), 1);
+  } else {
+    unsetenv("TZ");
+  }
+  tzset();
+}
+
+void test_route_api_response() {
+  REQUIRE(routes::checkApiResponse("") == std::make_pair(false, 500));
+  REQUIRE(routes::checkApiResponse("not json") == std::make_pair(false, 500));
+  REQUIRE(routes::checkApiResponse(R"({"error": "unauthorized"})") == std::make_pair(false, 401));
+  REQUIRE(routes::checkApiResponse(R"({"error": "server error"})") == std::make_pair(false, 500));
+  REQUIRE(routes::checkApiResponse("[]") == std::make_pair(true, 0));
+  REQUIRE(routes::checkApiResponse(R"({"dongle_id": "aaaa"})") == std::make_pair(true, 0));
+}
+
+void test_route_json() {
+  auto devices = routes::parseDevices(R"([{"dongle_id": "aaaa"}, {"dongle_id": "bbbb"}])");
+  REQUIRE(devices.size() == 2);
+  REQUIRE(devices[0].dongle_id == "aaaa");
+  REQUIRE(devices[1].dongle_id == "bbbb");
+  REQUIRE(routes::parseDevices("not json").empty());
+  REQUIRE(routes::parseDevices(R"({"error": "unauthorized"})").empty());
+
+  auto list = routes::parseRoutes(
+      R"([{"fullname": "aaaa|2024-01-02--03-04-05", "start_time_utc_millis": 1704164645000, "end_time_utc_millis": 1704165245000}])", false);
+  REQUIRE(list.size() == 1);
+  REQUIRE(list[0].name == "aaaa|2024-01-02--03-04-05");
+  REQUIRE(list[0].start_ms == 1704164645000);
+  REQUIRE(list[0].end_ms == 1704165245000);
+
+  // preserved routes report ISO-8601 timestamps
+  auto preserved = routes::parseRoutes(
+      R"([{"fullname": "aaaa|2024-01-02--03-04-05", "start_time": "2024-01-02T03:04:05Z", "end_time": "2024-01-02T03:14:05Z"}])", true);
+  REQUIRE(preserved.size() == 1);
+  REQUIRE(preserved[0].start_ms == 1704164645000);
+  REQUIRE(preserved[0].end_ms == 1704165245000);
+
+  REQUIRE(routes::parseRoutes("not json", false).empty());
+}
+
 void test_cabana_core() {
   test_format_seconds();
   test_to_hex();
@@ -251,6 +310,9 @@ void test_cabana_core() {
   test_parse_dbc();
   test_parse_opendbc();
   test_dbc_manager();
+  test_route_timestamps();
+  test_route_api_response();
+  test_route_json();
 }
 
 int main() {