From 46f612224cf6953a845ddc6ed7aadaa9f2128e6b Mon Sep 17 00:00:00 2001 From: Trey Moen <50057480+greatgitsby@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:57:47 -0700 Subject: [PATCH] cabana: split comma API route fetching out of RoutesDialog (#38721) --- openpilot/tools/cabana/SConscript | 5 +- openpilot/tools/cabana/routes.cc | 115 ++++++++++++++++++ openpilot/tools/cabana/routes.h | 42 +++++++ openpilot/tools/cabana/routesdialog.cc | 126 +++----------------- openpilot/tools/cabana/routesdialog.h | 7 +- openpilot/tools/cabana/tests/test_cabana.cc | 62 ++++++++++ 6 files changed, 246 insertions(+), 111 deletions(-) create mode 100644 openpilot/tools/cabana/routes.cc create mode 100644 openpilot/tools/cabana/routes.h diff --git a/openpilot/tools/cabana/SConscript b/openpilot/tools/cabana/SConscript index 75e67d99e3..f7db3f23b7 100644 --- a/openpilot/tools/cabana/SConscript +++ b/openpilot/tools/cabana/SConscript @@ -99,7 +99,7 @@ cabana_env.Command(assets, "assets/assets.qrc", f"rcc $SOURCES -o $TARGET") cabana_env.Depends(assets, Glob('/assets/*', exclude=[assets, "assets/assets.o"])) cabana_srcs = ['mainwin.cc', 'streams/pandastream.cc', 'streams/devicestream.cc', 'streams/livestream.cc', 'streams/abstractstream.cc', 'streams/replaystream.cc', 'binaryview.cc', 'historylog.cc', 'videowidget.cc', 'signalview.cc', - 'routesdialog.cc', 'dbc/dbc.cc', 'dbc/dbcfile.cc', 'dbc/dbcmanager.cc', + 'routesdialog.cc', 'routes.cc', 'dbc/dbc.cc', 'dbc/dbcfile.cc', 'dbc/dbcmanager.cc', 'utils/export.cc', 'utils/util.cc', 'utils/strings.cc', 'utils/elidedlabel.cc', 'chart/chartswidget.cc', 'chart/chart.cc', 'chart/signalselector.cc', 'chart/tiplabel.cc', 'chart/sparkline.cc', 'commands.cc', 'messageswidget.cc', 'streamselector.cc', 'settings.cc', 'settingsdialog.cc', 'panda.cc', @@ -120,8 +120,9 @@ if GetOption('extras'): dbc_core_test_env.Object('tests/dbc_core_file', 'dbc/dbcfile.cc'), dbc_core_test_env.Object('tests/dbc_core_manager', 'dbc/dbcmanager.cc'), dbc_core_test_env.Object('tests/dbc_core_strings', 'utils/strings.cc'), + dbc_core_test_env.Object('tests/dbc_core_routes', 'routes.cc'), ] - dbc_core_test_env.Program('tests/test_dbc_core', dbc_core_test_objects) + dbc_core_test_env.Program('tests/test_dbc_core', dbc_core_test_objects, LIBS=[replay_lib, common]) output_json_file = 'openpilot/tools/cabana/dbc/car_fingerprint_to_dbc.json' generate_dbc = cabana_env.Command('#' + output_json_file, diff --git a/openpilot/tools/cabana/routes.cc b/openpilot/tools/cabana/routes.cc new file mode 100644 index 0000000000..50a1eacf8d --- /dev/null +++ b/openpilot/tools/cabana/routes.cc @@ -0,0 +1,115 @@ +#include "tools/cabana/routes.h" + +#include +#include +#include +#include + +#include "json11/json11.hpp" +#include "tools/replay/py_downloader.h" + +namespace routes { + +std::pair checkApiResponse(const std::string &result) { + if (result.empty()) return {false, 500}; + std::string err; + auto doc = json11::Json::parse(result, err); + if (!err.empty()) return {false, 500}; + if (doc.is_object() && doc["error"].is_string()) { + return {false, doc["error"].string_value() == "unauthorized" ? 401 : 500}; + } + return {true, 0}; +} + +int64_t nowUnixMs() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +int64_t parseIsoToUnixMs(const std::string &s) { + std::string bytes = s; + if (!bytes.empty() && (bytes.back() == 'Z' || bytes.back() == 'z')) bytes.pop_back(); + int millis = 0; + auto dot = bytes.find('.'); + if (dot != std::string::npos) { + std::string frac = bytes.substr(dot + 1); + bytes = bytes.substr(0, dot); + while (frac.size() < 3) frac.push_back('0'); + millis = std::atoi(frac.substr(0, 3).c_str()); + } + std::tm tm{}; + const char *ret = strptime(bytes.c_str(), "%Y-%m-%dT%H:%M:%S", &tm); + if (!ret) ret = strptime(bytes.c_str(), "%Y-%m-%d %H:%M:%S", &tm); + if (!ret) return 0; + time_t secs = timegm(&tm); + if (secs == static_cast(-1)) return 0; + return static_cast(secs) * 1000 + millis; +} + +std::string formatUnixMs(int64_t ms) { + time_t secs = static_cast(ms / 1000); + std::tm tm{}; + localtime_r(&secs, &tm); + char buf[64]; + std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tm); + return buf; +} + +std::vector parseDevices(const std::string &json) { + std::vector devices; + std::string err; + auto doc = json11::Json::parse(json, err); + if (err.empty() && doc.is_array()) { + for (const auto &device : doc.array_items()) { + devices.push_back({device["dongle_id"].string_value()}); + } + } + return devices; +} + +std::vector parseRoutes(const std::string &json, bool preserved) { + std::vector items; + std::string err; + auto doc = json11::Json::parse(json, err); + if (err.empty() && doc.is_array()) { + for (const auto &route : doc.array_items()) { + RouteInfo info; + info.name = route["fullname"].string_value(); + if (preserved) { + info.start_ms = parseIsoToUnixMs(route["start_time"].string_value()); + info.end_ms = parseIsoToUnixMs(route["end_time"].string_value()); + } else { + info.start_ms = static_cast(route["start_time_utc_millis"].number_value()); + info.end_ms = static_cast(route["end_time_utc_millis"].number_value()); + } + items.push_back(std::move(info)); + } + } + return items; +} + +void fetchDevices(DevicesCallback callback) { + std::thread([callback = std::move(callback)]() { + std::string result = PyDownloader::getDevices(); + auto [success, error_code] = checkApiResponse(result); + callback(success ? parseDevices(result) : std::vector{}, success, error_code); + }).detach(); +} + +void fetchRoutes(const std::string &dongle_id, int period_days, RoutesCallback callback) { + const bool preserved = period_days == -1; + int64_t start_ms = 0, end_ms = 0; + if (!preserved) { + end_ms = nowUnixMs(); + start_ms = end_ms - static_cast(period_days) * 24LL * 60LL * 60LL * 1000LL; + } + + std::thread([dongle_id, start_ms, end_ms, preserved, callback = std::move(callback)]() { + std::string result = PyDownloader::getDeviceRoutes(dongle_id, start_ms, end_ms, preserved); + auto [success, error_code] = checkApiResponse(result); + callback(success ? parseRoutes(result, preserved) : std::vector{}, success, error_code); + }).detach(); +} + +} // namespace routes diff --git a/openpilot/tools/cabana/routes.h b/openpilot/tools/cabana/routes.h new file mode 100644 index 0000000000..ef6fc8dee0 --- /dev/null +++ b/openpilot/tools/cabana/routes.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace routes { + +struct DeviceInfo { + std::string dongle_id; +}; + +struct RouteInfo { + std::string name; + int64_t start_ms = 0; + int64_t end_ms = 0; +}; + +using DevicesCallback = std::function devices, bool success, int error_code)>; +using RoutesCallback = std::function routes, bool success, int error_code)>; + +// Parse a PyDownloader JSON response into (success, error_code). +std::pair checkApiResponse(const std::string &result); + +int64_t nowUnixMs(); +// Parse ISO-8601 (with optional fractional seconds / Z) to unix ms. Returns 0 on failure. +int64_t parseIsoToUnixMs(const std::string &s); +// Local time, "%Y-%m-%d %H:%M:%S". +std::string formatUnixMs(int64_t ms); + +std::vector parseDevices(const std::string &json); +// preserved routes report ISO-8601 timestamps instead of unix millis +std::vector parseRoutes(const std::string &json, bool preserved); + +// Both fetch on a detached thread and invoke the callback from that thread. +void fetchDevices(DevicesCallback callback); +// period_days of -1 requests preserved routes +void fetchRoutes(const std::string &dongle_id, int period_days, RoutesCallback callback); + +} // namespace routes diff --git a/openpilot/tools/cabana/routesdialog.cc b/openpilot/tools/cabana/routesdialog.cc index 9891104c9f..33b773edb9 100644 --- a/openpilot/tools/cabana/routesdialog.cc +++ b/openpilot/tools/cabana/routesdialog.cc @@ -1,9 +1,6 @@ #include "tools/cabana/routesdialog.h" -#include -#include #include -#include #include #include @@ -12,62 +9,7 @@ #include #include -#include "json11/json11.hpp" #include "tools/cabana/utils/util.h" -#include "tools/replay/py_downloader.h" - -namespace { - -// Parse a PyDownloader JSON response into (success, error_code). -std::pair checkApiResponse(const std::string &result) { - if (result.empty()) return {false, 500}; - std::string err; - auto doc = json11::Json::parse(result, err); - if (!err.empty()) return {false, 500}; - if (doc.is_object() && doc["error"].is_string()) { - return {false, doc["error"].string_value() == "unauthorized" ? 401 : 500}; - } - return {true, 0}; -} - -int64_t nowUnixMs() { - return std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); -} - -// Parse ISO-8601 (with optional fractional seconds / Z) to unix ms. Returns 0 on failure. -int64_t parseIsoToUnixMs(const std::string &s) { - std::string bytes = s; - if (!bytes.empty() && (bytes.back() == 'Z' || bytes.back() == 'z')) bytes.pop_back(); - int millis = 0; - auto dot = bytes.find('.'); - if (dot != std::string::npos) { - std::string frac = bytes.substr(dot + 1); - bytes = bytes.substr(0, dot); - while (frac.size() < 3) frac.push_back('0'); - millis = std::atoi(frac.substr(0, 3).c_str()); - } - std::tm tm{}; - const char *ret = strptime(bytes.c_str(), "%Y-%m-%dT%H:%M:%S", &tm); - if (!ret) ret = strptime(bytes.c_str(), "%Y-%m-%d %H:%M:%S", &tm); - if (!ret) return 0; - tm.tm_isdst = -1; - time_t secs = timegm(&tm); - if (secs == static_cast(-1)) return 0; - return static_cast(secs) * 1000 + millis; -} - -std::string formatUnixMs(int64_t ms) { - time_t secs = static_cast(ms / 1000); - std::tm tm{}; - localtime_r(&secs, &tm); - char buf[64]; - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tm); - return buf; -} - -} // namespace // The RouteListWidget class extends QListWidget to display a custom message when empty class RouteListWidget : public QListWidget { @@ -110,26 +52,19 @@ RoutesDialog::RoutesDialog(QWidget *parent) : QDialog(parent) { connect(button_box, &QDialogButtonBox::accepted, this, &QDialog::accept); connect(button_box, &QDialogButtonBox::rejected, this, &QDialog::reject); - // Fetch devices - std::thread([this, alive = std::weak_ptr(alive_)]() { - std::string result = PyDownloader::getDevices(); - auto response = checkApiResponse(result); - utils::runOnMainThread([this, alive, r = std::move(result), response]() { - if (!alive.expired()) parseDeviceList(r, response.first, response.second); + routes::fetchDevices([this, alive = std::weak_ptr(alive_)](std::vector devices, bool success, int error_code) { + utils::runOnMainThread([this, alive, devices = std::move(devices), success, error_code]() { + if (!alive.expired()) setDeviceList(devices, success, error_code); }); - }).detach(); + }); } -void RoutesDialog::parseDeviceList(const std::string &json, bool success, int error_code) { +void RoutesDialog::setDeviceList(const std::vector &devices, bool success, int error_code) { if (success) { device_list_->clear(); - std::string err; - auto doc = json11::Json::parse(json, err); - if (err.empty() && doc.is_array()) { - for (const auto &device : doc.array_items()) { - QString dongle_id = QString::fromStdString(device["dongle_id"].string_value()); - device_list_->addItem(dongle_id, dongle_id); - } + for (const auto &device : devices) { + QString dongle_id = QString::fromStdString(device.dongle_id); + device_list_->addItem(dongle_id, dongle_id); } } else { QMessageBox::warning(this, tr("Error"), error_code == 401 ? tr("Unauthorized. Authenticate with openpilot/tools/lib/auth.py") : tr("Network error")); @@ -144,45 +79,22 @@ void RoutesDialog::fetchRoutes() { route_list_->clear(); route_list_->setEmptyText(tr("Loading...")); - std::string did = device_list_->currentText().toStdString(); - int period = period_selector_->currentData().toInt(); - - bool preserved = (period == -1); - int64_t start_ms = 0, end_ms = 0; - if (!preserved) { - end_ms = nowUnixMs(); - start_ms = end_ms - static_cast(period) * 24LL * 60LL * 60LL * 1000LL; - } - int request_id = ++fetch_id_; - std::thread([this, alive = std::weak_ptr(alive_), did, start_ms, end_ms, preserved, request_id]() { - std::string result = PyDownloader::getDeviceRoutes(did, start_ms, end_ms, preserved); - auto response = checkApiResponse(result); - utils::runOnMainThread([this, alive, r = std::move(result), response, request_id]() { - if (!alive.expired() && fetch_id_ == request_id) parseRouteList(r, response.first, response.second); + auto on_routes = [this, alive = std::weak_ptr(alive_), request_id](std::vector list, bool success, int) { + utils::runOnMainThread([this, alive, list = std::move(list), success, request_id]() { + if (!alive.expired() && fetch_id_ == request_id) setRouteList(list, success); }); - }).detach(); + }; + routes::fetchRoutes(device_list_->currentText().toStdString(), period_selector_->currentData().toInt(), std::move(on_routes)); } -void RoutesDialog::parseRouteList(const std::string &json, bool success, int error_code) { +void RoutesDialog::setRouteList(const std::vector &list, bool success) { if (success) { - std::string err; - auto doc = json11::Json::parse(json, err); - if (err.empty() && doc.is_array()) { - for (const auto &route : doc.array_items()) { - int64_t from_ms = 0, to_ms = 0; - if (period_selector_->currentData().toInt() == -1) { - from_ms = parseIsoToUnixMs(route["start_time"].string_value()); - to_ms = parseIsoToUnixMs(route["end_time"].string_value()); - } else { - from_ms = static_cast(route["start_time_utc_millis"].number_value()); - to_ms = static_cast(route["end_time_utc_millis"].number_value()); - } - const int mins = static_cast((to_ms - from_ms) / 60000); - auto item = new QListWidgetItem(QString::fromStdString(formatUnixMs(from_ms) + " " + std::to_string(mins) + "min")); - item->setData(Qt::UserRole, QString::fromStdString(route["fullname"].string_value())); - route_list_->addItem(item); - } + for (const auto &route : list) { + const int mins = static_cast((route.end_ms - route.start_ms) / 60000); + auto item = new QListWidgetItem(QString::fromStdString(routes::formatUnixMs(route.start_ms) + " " + std::to_string(mins) + "min")); + item->setData(Qt::UserRole, QString::fromStdString(route.name)); + route_list_->addItem(item); } if (route_list_->count() > 0) route_list_->setCurrentRow(0); } else { diff --git a/openpilot/tools/cabana/routesdialog.h b/openpilot/tools/cabana/routesdialog.h index 44d7f068f9..4983f1b558 100644 --- a/openpilot/tools/cabana/routesdialog.h +++ b/openpilot/tools/cabana/routesdialog.h @@ -2,10 +2,13 @@ #include #include +#include #include #include +#include "tools/cabana/routes.h" + class RouteListWidget; class RoutesDialog : public QDialog { @@ -15,8 +18,8 @@ public: std::string route(); protected: - void parseDeviceList(const std::string &json, bool success, int error_code); - void parseRouteList(const std::string &json, bool success, int error_code); + void setDeviceList(const std::vector &devices, bool success, int error_code); + void setRouteList(const std::vector &list, bool success); void fetchRoutes(); QComboBox *device_list_; diff --git a/openpilot/tools/cabana/tests/test_cabana.cc b/openpilot/tools/cabana/tests/test_cabana.cc index e3b8377e02..5c5fce9864 100644 --- a/openpilot/tools/cabana/tests/test_cabana.cc +++ b/openpilot/tools/cabana/tests/test_cabana.cc @@ -7,6 +7,7 @@ #include "common/tests/native_test.h" #include "tools/cabana/dbc/dbcfile.h" #include "tools/cabana/dbc/dbcmanager.h" +#include "tools/cabana/routes.h" #include "tools/cabana/utils/strings.h" const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.bz2"; @@ -240,6 +241,64 @@ void test_signal_tooltip() { )"); } +void test_route_timestamps() { + REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05Z") == 1704164645000); + REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05") == 1704164645000); + REQUIRE(routes::parseIsoToUnixMs("2024-01-02 03:04:05") == 1704164645000); + REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05.123Z") == 1704164645123); + REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05.4Z") == 1704164645400); + REQUIRE(routes::parseIsoToUnixMs("2024-01-02T03:04:05.123456Z") == 1704164645123); + REQUIRE(routes::parseIsoToUnixMs("") == 0); + REQUIRE(routes::parseIsoToUnixMs("not a timestamp") == 0); + + // formatUnixMs is local time + const char *tz = getenv("TZ"); + const std::string prev_tz = tz ? tz : ""; + setenv("TZ", "UTC", 1); + tzset(); + REQUIRE(routes::formatUnixMs(1704164645123) == "2024-01-02 03:04:05"); + if (tz) { + setenv("TZ", prev_tz.c_str(), 1); + } else { + unsetenv("TZ"); + } + tzset(); +} + +void test_route_api_response() { + REQUIRE(routes::checkApiResponse("") == std::make_pair(false, 500)); + REQUIRE(routes::checkApiResponse("not json") == std::make_pair(false, 500)); + REQUIRE(routes::checkApiResponse(R"({"error": "unauthorized"})") == std::make_pair(false, 401)); + REQUIRE(routes::checkApiResponse(R"({"error": "server error"})") == std::make_pair(false, 500)); + REQUIRE(routes::checkApiResponse("[]") == std::make_pair(true, 0)); + REQUIRE(routes::checkApiResponse(R"({"dongle_id": "aaaa"})") == std::make_pair(true, 0)); +} + +void test_route_json() { + auto devices = routes::parseDevices(R"([{"dongle_id": "aaaa"}, {"dongle_id": "bbbb"}])"); + REQUIRE(devices.size() == 2); + REQUIRE(devices[0].dongle_id == "aaaa"); + REQUIRE(devices[1].dongle_id == "bbbb"); + REQUIRE(routes::parseDevices("not json").empty()); + REQUIRE(routes::parseDevices(R"({"error": "unauthorized"})").empty()); + + auto list = routes::parseRoutes( + R"([{"fullname": "aaaa|2024-01-02--03-04-05", "start_time_utc_millis": 1704164645000, "end_time_utc_millis": 1704165245000}])", false); + REQUIRE(list.size() == 1); + REQUIRE(list[0].name == "aaaa|2024-01-02--03-04-05"); + REQUIRE(list[0].start_ms == 1704164645000); + REQUIRE(list[0].end_ms == 1704165245000); + + // preserved routes report ISO-8601 timestamps + auto preserved = routes::parseRoutes( + R"([{"fullname": "aaaa|2024-01-02--03-04-05", "start_time": "2024-01-02T03:04:05Z", "end_time": "2024-01-02T03:14:05Z"}])", true); + REQUIRE(preserved.size() == 1); + REQUIRE(preserved[0].start_ms == 1704164645000); + REQUIRE(preserved[0].end_ms == 1704165245000); + + REQUIRE(routes::parseRoutes("not json", false).empty()); +} + void test_cabana_core() { test_format_seconds(); test_to_hex(); @@ -251,6 +310,9 @@ void test_cabana_core() { test_parse_dbc(); test_parse_opendbc(); test_dbc_manager(); + test_route_timestamps(); + test_route_api_response(); + test_route_json(); } int main() {