Merge branch 'upstream/openpilot/master' into sync-20250731

# Conflicts:
#	.github/workflows/selfdrive_tests.yaml
#	common/params.h
#	common/params_keys.h
#	common/params_pyx.pyx
#	docs/CARS.md
#	opendbc_repo
#	panda
#	selfdrive/car/tests/test_models.py
#	selfdrive/pandad/pandad.cc
#	selfdrive/pandad/pandad.h
#	selfdrive/selfdrived/selfdrived.py
#	selfdrive/ui/translations/main_ar.ts
#	selfdrive/ui/translations/main_de.ts
#	selfdrive/ui/translations/main_es.ts
#	selfdrive/ui/translations/main_fr.ts
#	selfdrive/ui/translations/main_ja.ts
#	selfdrive/ui/translations/main_ko.ts
#	selfdrive/ui/translations/main_pt-BR.ts
#	selfdrive/ui/translations/main_th.ts
#	selfdrive/ui/translations/main_tr.ts
#	selfdrive/ui/translations/main_zh-CHS.ts
#	selfdrive/ui/translations/main_zh-CHT.ts
#	system/athena/athenad.py
#	system/athena/manage_athenad.py
#	system/manager/manager.py
#	system/sentry.py
#	uv.lock
Sync: `commaai/opendbc:master` into `sunnypilot/opendbc:master`

Sync: `commaai/panda:master` into `sunnypilot/panda:master`

tools/cabana/SConscript

@@ -29,7 +29,8 @@ cabana_lib = cabana_env.Library("cabana_lib", ['mainwin.cc', 'streams/socketcans
                                                'streams/routes.cc', 'dbc/dbc.cc', 'dbc/dbcfile.cc', 'dbc/dbcmanager.cc',
                                                'utils/export.cc', 'utils/util.cc',
                                                'chart/chartswidget.cc', 'chart/chart.cc', 'chart/signalselector.cc', 'chart/tiplabel.cc', 'chart/sparkline.cc',
-                                               'commands.cc', 'messageswidget.cc', 'streamselector.cc', 'settings.cc', 'detailwidget.cc', 'tools/findsimilarbits.cc', 'tools/findsignal.cc', 'tools/routeinfo.cc'], LIBS=cabana_libs, FRAMEWORKS=base_frameworks)
+                                               'commands.cc', 'messageswidget.cc', 'streamselector.cc', 'settings.cc',
+                                               'cameraview.cc', 'detailwidget.cc', 'tools/findsimilarbits.cc', 'tools/findsignal.cc', 'tools/routeinfo.cc'], LIBS=cabana_libs, FRAMEWORKS=base_frameworks)
 cabana_env.Program('cabana', ['cabana.cc', cabana_lib, assets], LIBS=cabana_libs, FRAMEWORKS=base_frameworks)
 
 if GetOption('extras'):

tools/cabana/cameraview.cc

@@ -0,0 +1,261 @@
+#include "tools/cabana/cameraview.h"
+
+#ifdef __APPLE__
+#include <OpenGL/gl3.h>
+#else
+#include <GLES3/gl3.h>
+#endif
+
+#include <QApplication>
+
+namespace {
+
+const char frame_vertex_shader[] =
+#ifdef __APPLE__
+  "#version 330 core\n"
+#else
+  "#version 300 es\n"
+#endif
+  "layout(location = 0) in vec4 aPosition;\n"
+  "layout(location = 1) in vec2 aTexCoord;\n"
+  "uniform mat4 uTransform;\n"
+  "out vec2 vTexCoord;\n"
+  "void main() {\n"
+  "  gl_Position = uTransform * aPosition;\n"
+  "  vTexCoord = aTexCoord;\n"
+  "}\n";
+
+const char frame_fragment_shader[] =
+#ifdef __APPLE__
+  "#version 330 core\n"
+#else
+  "#version 300 es\n"
+  "precision mediump float;\n"
+#endif
+  "uniform sampler2D uTextureY;\n"
+  "uniform sampler2D uTextureUV;\n"
+  "in vec2 vTexCoord;\n"
+  "out vec4 colorOut;\n"
+  "void main() {\n"
+  "  float y = texture(uTextureY, vTexCoord).r;\n"
+  "  vec2 uv = texture(uTextureUV, vTexCoord).rg - 0.5;\n"
+  "  float r = y + 1.402 * uv.y;\n"
+  "  float g = y - 0.344 * uv.x - 0.714 * uv.y;\n"
+  "  float b = y + 1.772 * uv.x;\n"
+  "  colorOut = vec4(r, g, b, 1.0);\n"
+  "}\n";
+
+} // namespace
+
+CameraWidget::CameraWidget(std::string stream_name, VisionStreamType type, QWidget* parent) :
+                          stream_name(stream_name), active_stream_type(type), requested_stream_type(type), QOpenGLWidget(parent) {
+  setAttribute(Qt::WA_OpaquePaintEvent);
+  qRegisterMetaType<std::set<VisionStreamType>>("availableStreams");
+  QObject::connect(this, &CameraWidget::vipcThreadConnected, this, &CameraWidget::vipcConnected, Qt::BlockingQueuedConnection);
+  QObject::connect(this, &CameraWidget::vipcThreadFrameReceived, this, &CameraWidget::vipcFrameReceived, Qt::QueuedConnection);
+  QObject::connect(this, &CameraWidget::vipcAvailableStreamsUpdated, this, &CameraWidget::availableStreamsUpdated, Qt::QueuedConnection);
+  QObject::connect(QApplication::instance(), &QCoreApplication::aboutToQuit, this, &CameraWidget::stopVipcThread);
+}
+
+CameraWidget::~CameraWidget() {
+  makeCurrent();
+  stopVipcThread();
+  if (isValid()) {
+    glDeleteVertexArrays(1, &frame_vao);
+    glDeleteBuffers(1, &frame_vbo);
+    glDeleteBuffers(1, &frame_ibo);
+    glDeleteTextures(2, textures);
+    shader_program_.reset();
+  }
+  doneCurrent();
+}
+
+void CameraWidget::initializeGL() {
+  initializeOpenGLFunctions();
+
+  shader_program_ = std::make_unique<QOpenGLShaderProgram>(context());
+  shader_program_->addShaderFromSourceCode(QOpenGLShader::Vertex, frame_vertex_shader);
+  shader_program_->addShaderFromSourceCode(QOpenGLShader::Fragment, frame_fragment_shader);
+  shader_program_->link();
+
+  GLint frame_pos_loc = shader_program_->attributeLocation("aPosition");
+  GLint frame_texcoord_loc = shader_program_->attributeLocation("aTexCoord");
+
+  auto [x1, x2, y1, y2] = requested_stream_type == VISION_STREAM_DRIVER ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
+  const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3};
+  const float frame_coords[4][4] = {
+    {-1.0, -1.0, x2, y1}, // bl
+    {-1.0,  1.0, x2, y2}, // tl
+    { 1.0,  1.0, x1, y2}, // tr
+    { 1.0, -1.0, x1, y1}, // br
+  };
+
+  glGenVertexArrays(1, &frame_vao);
+  glBindVertexArray(frame_vao);
+  glGenBuffers(1, &frame_vbo);
+  glBindBuffer(GL_ARRAY_BUFFER, frame_vbo);
+  glBufferData(GL_ARRAY_BUFFER, sizeof(frame_coords), frame_coords, GL_STATIC_DRAW);
+  glEnableVertexAttribArray(frame_pos_loc);
+  glVertexAttribPointer(frame_pos_loc, 2, GL_FLOAT, GL_FALSE,
+                        sizeof(frame_coords[0]), (const void *)0);
+  glEnableVertexAttribArray(frame_texcoord_loc);
+  glVertexAttribPointer(frame_texcoord_loc, 2, GL_FLOAT, GL_FALSE,
+                        sizeof(frame_coords[0]), (const void *)(sizeof(float) * 2));
+  glGenBuffers(1, &frame_ibo);
+  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, frame_ibo);
+  glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(frame_indicies), frame_indicies, GL_STATIC_DRAW);
+  glBindBuffer(GL_ARRAY_BUFFER, 0);
+  glBindVertexArray(0);
+
+  glGenTextures(2, textures);
+
+  shader_program_->bind();
+  shader_program_->setUniformValue("uTextureY", 0);
+  shader_program_->setUniformValue("uTextureUV", 1);
+  shader_program_->release();
+}
+
+void CameraWidget::showEvent(QShowEvent *event) {
+  if (!vipc_thread) {
+    clearFrames();
+    vipc_thread = new QThread();
+    connect(vipc_thread, &QThread::started, [=]() { vipcThread(); });
+    connect(vipc_thread, &QThread::finished, vipc_thread, &QObject::deleteLater);
+    vipc_thread->start();
+  }
+}
+
+void CameraWidget::stopVipcThread() {
+  makeCurrent();
+  if (vipc_thread) {
+    vipc_thread->requestInterruption();
+    vipc_thread->quit();
+    vipc_thread->wait();
+    vipc_thread = nullptr;
+  }
+}
+
+void CameraWidget::availableStreamsUpdated(std::set<VisionStreamType> streams) {
+  available_streams = streams;
+}
+
+void CameraWidget::paintGL() {
+  glClearColor(bg.redF(), bg.greenF(), bg.blueF(), bg.alphaF());
+  glClear(GL_STENCIL_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
+
+  std::lock_guard lk(frame_lock);
+  if (!current_frame_) return;
+
+  // Scale for aspect ratio
+  float widget_ratio = (float)width() / height();
+  float frame_ratio = (float)stream_width / stream_height;
+  float scale_x = std::min(frame_ratio / widget_ratio, 1.0f);
+  float scale_y = std::min(widget_ratio / frame_ratio, 1.0f);
+
+  glViewport(0, 0, width() * devicePixelRatio(), height() * devicePixelRatio());
+
+  shader_program_->bind();
+  QMatrix4x4 transform;
+  transform.scale(scale_x, scale_y, 1.0f);
+  shader_program_->setUniformValue("uTransform", transform);
+
+  glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
+
+  glPixelStorei(GL_UNPACK_ROW_LENGTH, stream_stride);
+  glActiveTexture(GL_TEXTURE0);
+  glBindTexture(GL_TEXTURE_2D, textures[0]);
+  glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, stream_width, stream_height, GL_RED, GL_UNSIGNED_BYTE, current_frame_->y);
+
+  glPixelStorei(GL_UNPACK_ROW_LENGTH, stream_stride/2);
+  glActiveTexture(GL_TEXTURE1);
+  glBindTexture(GL_TEXTURE_2D, textures[1]);
+  glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, stream_width/2, stream_height/2, GL_RG, GL_UNSIGNED_BYTE, current_frame_->uv);
+
+  glBindVertexArray(frame_vao);
+  glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, nullptr);
+  glBindVertexArray(0);
+
+  // Reset both texture units
+  glActiveTexture(GL_TEXTURE1);
+  glBindTexture(GL_TEXTURE_2D, 0);
+  glActiveTexture(GL_TEXTURE0);
+  glBindTexture(GL_TEXTURE_2D, 0);
+  glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
+  glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
+
+  shader_program_->release();
+}
+
+void CameraWidget::vipcConnected(VisionIpcClient *vipc_client) {
+  makeCurrent();
+  stream_width = vipc_client->buffers[0].width;
+  stream_height = vipc_client->buffers[0].height;
+  stream_stride = vipc_client->buffers[0].stride;
+
+  glBindTexture(GL_TEXTURE_2D, textures[0]);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+  glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, stream_width, stream_height, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr);
+  assert(glGetError() == GL_NO_ERROR);
+
+  glBindTexture(GL_TEXTURE_2D, textures[1]);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+  glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  glTexImage2D(GL_TEXTURE_2D, 0, GL_RG8, stream_width/2, stream_height/2, 0, GL_RG, GL_UNSIGNED_BYTE, nullptr);
+  assert(glGetError() == GL_NO_ERROR);
+}
+
+void CameraWidget::vipcFrameReceived() {
+  update();
+}
+
+void CameraWidget::vipcThread() {
+  VisionStreamType cur_stream = requested_stream_type;
+  std::unique_ptr<VisionIpcClient> vipc_client;
+  VisionIpcBufExtra frame_meta = {};
+
+  while (!QThread::currentThread()->isInterruptionRequested()) {
+    if (!vipc_client || cur_stream != requested_stream_type) {
+      clearFrames();
+      qDebug().nospace() << "connecting to stream " << requested_stream_type << ", was connected to " << cur_stream;
+      cur_stream = requested_stream_type;
+      vipc_client.reset(new VisionIpcClient(stream_name, cur_stream, false));
+    }
+    active_stream_type = cur_stream;
+
+    if (!vipc_client->connected) {
+      clearFrames();
+      auto streams = VisionIpcClient::getAvailableStreams(stream_name, false);
+      if (streams.empty()) {
+        QThread::msleep(100);
+        continue;
+      }
+      emit vipcAvailableStreamsUpdated(streams);
+
+      if (!vipc_client->connect(false)) {
+        QThread::msleep(100);
+        continue;
+      }
+      emit vipcThreadConnected(vipc_client.get());
+    }
+
+    if (VisionBuf *buf = vipc_client->recv(&frame_meta, 100)) {
+      {
+        std::lock_guard lk(frame_lock);
+        current_frame_ = buf;
+        frame_meta_ = frame_meta;
+      }
+      emit vipcThreadFrameReceived();
+    }
+  }
+}
+
+void CameraWidget::clearFrames() {
+  std::lock_guard lk(frame_lock);
+  current_frame_ = nullptr;
+  available_streams.clear();
+}

tools/cabana/cameraview.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <set>
+#include <string>
+#include <utility>
+
+#include <QOpenGLFunctions>
+#include <QOpenGLShaderProgram>
+#include <QOpenGLWidget>
+#include <QThread>
+
+#include "msgq/visionipc/visionipc_client.h"
+
+class CameraWidget : public QOpenGLWidget, protected QOpenGLFunctions {
+  Q_OBJECT
+
+public:
+  using QOpenGLWidget::QOpenGLWidget;
+  explicit CameraWidget(std::string stream_name, VisionStreamType stream_type, QWidget* parent = nullptr);
+  ~CameraWidget();
+  void setStreamType(VisionStreamType type) { requested_stream_type = type; }
+  VisionStreamType getStreamType() { return active_stream_type; }
+  void stopVipcThread();
+
+signals:
+  void clicked();
+  void vipcThreadConnected(VisionIpcClient *);
+  void vipcThreadFrameReceived();
+  void vipcAvailableStreamsUpdated(std::set<VisionStreamType>);
+
+protected:
+  void paintGL() override;
+  void initializeGL() override;
+  void showEvent(QShowEvent *event) override;
+  void mouseReleaseEvent(QMouseEvent *event) override { emit clicked(); }
+  void vipcThread();
+  void clearFrames();
+
+  GLuint frame_vao, frame_vbo, frame_ibo;
+  GLuint textures[2];
+  std::unique_ptr<QOpenGLShaderProgram> shader_program_;
+  QColor bg = Qt::black;
+
+  std::string stream_name;
+  int stream_width = 0;
+  int stream_height = 0;
+  int stream_stride = 0;
+  std::atomic<VisionStreamType> active_stream_type;
+  std::atomic<VisionStreamType> requested_stream_type;
+  std::set<VisionStreamType> available_streams;
+  QThread *vipc_thread = nullptr;
+  std::recursive_mutex frame_lock;
+  VisionBuf* current_frame_ = nullptr;
+  VisionIpcBufExtra frame_meta_ = {};
+
+protected slots:
+  void vipcConnected(VisionIpcClient *vipc_client);
+  void vipcFrameReceived();
+  void availableStreamsUpdated(std::set<VisionStreamType> streams);
+};
+
+Q_DECLARE_METATYPE(std::set<VisionStreamType>);

tools/cabana/chart/chart.cc

@@ -45,7 +45,7 @@ ChartView::ChartView(const std::pair<double, double> &x_range, ChartsWidget *par
   createToolButtons();
   setRubberBand(QChartView::HorizontalRubberBand);
   setMouseTracking(true);
-  setTheme(settings.theme == DARK_THEME ? QChart::QChart::ChartThemeDark : QChart::ChartThemeLight);
+  setTheme(utils::isDarkTheme() ? QChart::QChart::ChartThemeDark : QChart::ChartThemeLight);
   signal_value_font.setPointSize(9);
 
   QObject::connect(axis_y, &QValueAxis::rangeChanged, this, &ChartView::resetChartCache);
@@ -747,7 +747,7 @@ void ChartView::drawTimeline(QPainter *painter) {
   QRectF time_str_rect(QPointF(x - time_str_size.width() / 2.0, plot_area.bottom() + AXIS_X_TOP_MARGIN), time_str_size);
   QPainterPath path;
   path.addRoundedRect(time_str_rect, 3, 3);
-  painter->fillPath(path, settings.theme == DARK_THEME ? Qt::darkGray : Qt::gray);
+  painter->fillPath(path, utils::isDarkTheme() ? Qt::darkGray : Qt::gray);
   painter->setPen(palette().color(QPalette::BrightText));
   painter->setFont(axis_x->labelsFont());
   painter->drawText(time_str_rect, Qt::AlignCenter, time_str);

tools/cabana/chart/chartswidget.cc

@@ -254,7 +254,7 @@ void ChartsWidget::settingChanged() {
   if (std::exchange(current_theme, settings.theme) != current_theme) {
     undo_zoom_action->setIcon(utils::icon("arrow-counterclockwise"));
     redo_zoom_action->setIcon(utils::icon("arrow-clockwise"));
-    auto theme = settings.theme == DARK_THEME ? QChart::QChart::ChartThemeDark : QChart::ChartThemeLight;
+    auto theme = utils::isDarkTheme() ? QChart::QChart::ChartThemeDark : QChart::ChartThemeLight;
     for (auto c : charts) {
       c->setTheme(theme);
     }

tools/cabana/chart/sparkline.cc

@@ -4,51 +4,97 @@
 #include <limits>
 #include <QPainter>
 
-void Sparkline::update(const MessageId &msg_id, const cabana::Signal *sig, double last_msg_ts, int range, QSize size) {
-  points.clear();
-  double value = 0;
-  auto [first, last] = can->eventsInRange(msg_id, std::make_pair(last_msg_ts -range, last_msg_ts));
-  for (auto it = first; it != last; ++it) {
-    if (sig->getValue((*it)->dat, (*it)->size, &value)) {
-      points.emplace_back(((*it)->mono_time - (*first)->mono_time) / 1e9, value);
-    }
-  }
-
-  if (points.empty() || size.isEmpty()) {
+void Sparkline::update(const cabana::Signal *sig, CanEventIter first, CanEventIter last, int range, QSize size) {
+  if (first == last || size.isEmpty()) {
     pixmap = QPixmap();
     return;
   }
 
-  const auto [min, max] = std::minmax_element(points.begin(), points.end(),
-                                              [](auto &l, auto &r) { return l.y() < r.y(); });
-  min_val = min->y() == max->y() ? min->y() - 1 : min->y();
-  max_val = min->y() == max->y() ? max->y() + 1 : max->y();
-  freq_ = points.size() / std::max(points.back().x() - points.front().x(), 1.0);
+  points_.clear();
+  min_val = std::numeric_limits<double>::max();
+  max_val = std::numeric_limits<double>::lowest();
+  points_.reserve(std::distance(first, last));
+
+  uint64_t start_time = (*first)->mono_time;
+  double value = 0.0;
+  for (auto it = first; it != last; ++it) {
+    if (sig->getValue((*it)->dat, (*it)->size, &value)) {
+      min_val = std::min(min_val, value);
+      max_val = std::max(max_val, value);
+      points_.emplace_back(((*it)->mono_time - start_time) / 1e9, value);
+    }
+  }
+
+  if (points_.empty()) {
+    pixmap = QPixmap();
+    return;
+  }
+
+  freq_ = points_.size() / std::max(points_.back().x() - points_.front().x(), 1.0);
   render(sig->color, range, size);
 }
 
 void Sparkline::render(const QColor &color, int range, QSize size) {
-  const double xscale = (size.width() - 1) / (double)range;
-  const double yscale = (size.height() - 3) / (max_val - min_val);
-  for (auto &v : points) {
-    v = QPoint(v.x() * xscale, 1 + std::abs(v.y() - max_val) * yscale);
+  // Adjust for flat lines
+  bool is_flat_line = min_val == max_val;
+  if (is_flat_line) {
+    min_val -= 1.0;
+    max_val += 1.0;
   }
 
+  // Calculate scaling
+  const double xscale = (size.width() - 1) / (double)range;
+  const double yscale = (size.height() - 3) / (max_val - min_val);
+  bool draw_individual_points = (points_.back().x() * xscale / points_.size()) > 8.0;
+
+  // Transform or downsample points
+  render_points_.reserve(points_.size());
+  render_points_.clear();
+  if (draw_individual_points) {
+    for (const auto &p : points_) {
+      render_points_.emplace_back(p.x() * xscale, 1.0 + (max_val - p.y()) * yscale);
+    }
+  } else if (is_flat_line) {
+    double y = size.height() / 2.0;
+    render_points_.emplace_back(0.0, y);
+    render_points_.emplace_back(points_.back().x() * xscale, y);
+  } else {
+    double prev_y = points_.front().y();
+    render_points_.emplace_back(points_.front().x() * xscale, 1.0 + (max_val - prev_y) * yscale);
+    bool in_flat = false;
+
+    for (size_t i = 1; i < points_.size(); ++i) {
+      const auto &p = points_[i];
+      double y = p.y();
+      if (std::abs(y - prev_y) < 1e-6) {
+        in_flat = true;
+      } else {
+        if (in_flat) render_points_.emplace_back(points_[i - 1].x() * xscale, 1.0 + (max_val - prev_y) * yscale);
+        render_points_.emplace_back(p.x() * xscale, 1.0 + (max_val - y) * yscale);
+        in_flat = false;
+      }
+      prev_y = y;
+    }
+    if (in_flat) render_points_.emplace_back(points_.back().x() * xscale, 1.0 + (max_val - prev_y) * yscale);
+  }
+
+  // Render to pixmap
   qreal dpr = qApp->devicePixelRatio();
-  size *= dpr;
-  if (size != pixmap.size()) {
-    pixmap = QPixmap(size);
+  const QSize pixmap_size = size * dpr;
+  if (pixmap.size() != pixmap_size) {
+    pixmap = QPixmap(pixmap_size);
   }
   pixmap.setDevicePixelRatio(dpr);
   pixmap.fill(Qt::transparent);
   QPainter painter(&pixmap);
-  painter.setRenderHint(QPainter::Antialiasing, points.size() < 500);
+  painter.setRenderHint(QPainter::Antialiasing, render_points_.size() <= 500);
   painter.setPen(color);
-  painter.drawPolyline(points.data(), points.size());
+  painter.drawPolyline(render_points_.data(), render_points_.size());
+
   painter.setPen(QPen(color, 3));
-  if ((points.back().x() - points.front().x()) / points.size() > 8) {
-    painter.drawPoints(points.data(), points.size());
+  if (draw_individual_points) {
+    painter.drawPoints(render_points_.data(), render_points_.size());
   } else {
-    painter.drawPoint(points.back());
+    painter.drawPoint(render_points_.back());
   }
 }

tools/cabana/chart/sparkline.h

@@ -9,7 +9,7 @@
 
 class Sparkline {
 public:
-  void update(const MessageId &msg_id, const cabana::Signal *sig, double last_msg_ts, int range, QSize size);
+  void update(const cabana::Signal *sig, CanEventIter first, CanEventIter last, int range, QSize size);
   inline double freq() const { return freq_; }
   bool isEmpty() const { return pixmap.isNull(); }
 
@@ -20,6 +20,7 @@ public:
 private:
   void render(const QColor &color, int range, QSize size);
 
-  std::vector<QPointF> points;
+  std::vector<QPointF> points_;
+  std::vector<QPointF> render_points_;
   double freq_ = 0;
 };

tools/cabana/chart/tiplabel.cc

@@ -7,6 +7,7 @@
 #include <QToolTip>
 
 #include "tools/cabana/settings.h"
+#include "tools/cabana/utils/util.h"
 
 TipLabel::TipLabel(QWidget *parent) : QLabel(parent, Qt::ToolTip | Qt::FramelessWindowHint) {
   setAttribute(Qt::WA_ShowWithoutActivating);
@@ -19,7 +20,7 @@ TipLabel::TipLabel(QWidget *parent) : QLabel(parent, Qt::ToolTip | Qt::Frameless
   font.setPointSizeF(8.34563465);
   setFont(font);
   auto palette = QToolTip::palette();
-  if (settings.theme != DARK_THEME) {
+  if (!utils::isDarkTheme()) {
     palette.setColor(QPalette::ToolTipBase, QApplication::palette().color(QPalette::Base));
     palette.setColor(QPalette::ToolTipText, QRgb(0x404044));  // same color as chart label brush
   }

tools/cabana/dbc/dbc.cc

@@ -109,7 +109,7 @@ void cabana::Msg::update() {
 
       mask[i] |= ((1ULL << sz) - 1) << shift;
 
-      bits -= size;
+      bits -= sz;
       i = sig->is_little_endian ? i - 1 : i + 1;
     }
   }

tools/cabana/historylog.cc

@@ -153,7 +153,7 @@ void HeaderView::paintSection(QPainter *painter, const QRect &rect, int logicalI
     painter->fillRect(rect, bg_role.value<QBrush>());
   }
   QString text = model()->headerData(logicalIndex, Qt::Horizontal, Qt::DisplayRole).toString();
-  painter->setPen(palette().color(settings.theme == DARK_THEME ? QPalette::BrightText : QPalette::Text));
+  painter->setPen(palette().color(utils::isDarkTheme() ? QPalette::BrightText : QPalette::Text));
   painter->drawText(rect.adjusted(5, 3, -5, -3), defaultAlignment(), text.replace(QChar('_'), ' '));
 }
 

tools/cabana/signalview.cc

@@ -634,11 +634,12 @@ void SignalView::updateState(const std::set<MessageId> *msgs) {
     QSize size(available_width - value_width,
                delegate->button_size.height() - style()->pixelMetric(QStyle::PM_FocusFrameVMargin) * 2);
 
+    auto [first, last] = can->eventsInRange(model->msg_id, std::make_pair(last_msg.ts -settings.sparkline_range, last_msg.ts));
     QFutureSynchronizer<void> synchronizer;
     for (int i = first_visible.row(); i <= last_visible.row(); ++i) {
       auto item = model->getItem(model->index(i, 1));
       synchronizer.addFuture(QtConcurrent::run(
-          &item->sparkline, &Sparkline::update, model->msg_id, item->sig, last_msg.ts, settings.sparkline_range, size));
+          &item->sparkline, &Sparkline::update, item->sig, first, last, settings.sparkline_range, size));
     }
     synchronizer.waitForFinished();
   }

tools/cabana/utils/util.cc

@@ -191,8 +191,15 @@ DoubleValidator::DoubleValidator(QObject *parent) : QDoubleValidator(parent) {
 }
 
 namespace utils {
+
+bool isDarkTheme() {
+  QColor windowColor = QApplication::palette().color(QPalette::Window);
+  return windowColor.lightness() < 128;
+}
+
 QPixmap icon(const QString &id) {
-  bool dark_theme = settings.theme == DARK_THEME;
+  bool dark_theme = isDarkTheme();
+
   QPixmap pm;
   QString key = "bootstrap_" % id % (dark_theme ? "1" : "0");
   if (!QPixmapCache::find(key, &pm)) {

tools/cabana/utils/util.h

@@ -98,7 +98,9 @@ public:
 };
 
 namespace utils {
+
 QPixmap icon(const QString &id);
+bool isDarkTheme();
 void setTheme(int theme);
 QString formatSeconds(double sec, bool include_milliseconds = false, bool absolute_time = false);
 inline void drawStaticText(QPainter *p, const QRect &r, const QStaticText &text) {

tools/cabana/videowidget.cc

@@ -261,14 +261,23 @@ void Slider::paintEvent(QPaintEvent *ev) {
 
   QStyleOptionSlider opt;
   initStyleOption(&opt);
-  QRect r = style()->subControlRect(QStyle::CC_Slider, &opt, QStyle::SC_SliderGroove, this);
-  p.fillRect(r, timeline_colors[(int)TimelineType::None]);
+  QRect handle_rect = style()->subControlRect(QStyle::CC_Slider, &opt, QStyle::SC_SliderHandle, this);
+  QRect groove_rect = style()->subControlRect(QStyle::CC_Slider, &opt, QStyle::SC_SliderGroove, this);
+
+  // Adjust groove height to match handle height
+  int handle_height = handle_rect.height();
+  groove_rect.setHeight(handle_height * 0.5);
+  groove_rect.moveCenter(QPoint(groove_rect.center().x(), rect().center().y()));
+
+  p.fillRect(groove_rect, timeline_colors[(int)TimelineType::None]);
 
   double min = minimum() / factor;
   double max = maximum() / factor;
 
   auto fillRange = [&](double begin, double end, const QColor &color) {
     if (begin > max || end < min) return;
+
+    QRect r = groove_rect;
     r.setLeft(((std::max(min, begin) - min) / (max - min)) * width());
     r.setRight(((std::min(max, end) - min) / (max - min)) * width());
     p.fillRect(r, color);

tools/cabana/videowidget.h

@@ -11,7 +11,7 @@
 #include <QToolBar>
 #include <QTabBar>
 
-#include "selfdrive/ui/qt/widgets/cameraview.h"
+#include "tools/cabana/cameraview.h"
 #include "tools/cabana/utils/util.h"
 #include "tools/replay/logreader.h"
 #include "tools/cabana/streams/replaystream.h"

tools/camerastream/compressed_vipc.py

@@ -62,7 +62,7 @@ def decoder(addr, vipc_server, vst, nvidia, W, H, debug=False):
           print("waiting for iframe")
         continue
       time_q.append(time.monotonic())
-      network_latency = (int(time.time()*1e9) - evta.unixTimestampNanos)/1e6
+      network_latency = (int(time.time()*1e9) - evta.unixTimestampNanos)/1e6  # noqa: TID251
       frame_latency = ((evta.idx.timestampEof/1e9) - (evta.idx.timestampSof/1e9))*1000
       process_latency = ((evt.logMonoTime/1e9) - (evta.idx.timestampEof/1e9))*1000
 

tools/car_porting/examples/find_segments_with_message.ipynb

@@ -76,7 +76,7 @@
     "  if platform not in database:\n",
     "    print(f\"No segments available for {platform}\")\n",
     "    continue\n",
-    "  \n",
+    "\n",
     "  all_segments = database[platform]\n",
     "  NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n",
     "  TEST_SEGMENTS.extend(random.sample(all_segments, NUM_SEGMENTS))\n",
@@ -147,7 +147,7 @@
     }
    ],
    "source": [
-    "from openpilot.tools.lib.logreader import LogReader\n",
+    "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n",
     "from tqdm.notebook import tqdm, tnrange\n",
     "\n",
     "# Example search for CAN ignition messages\n",
@@ -169,7 +169,7 @@
     "progress_bar = tnrange(len(TEST_SEGMENTS), desc=\"segments searched\")\n",
     "\n",
     "for segment in TEST_SEGMENTS:\n",
-    "  lr = LogReader(segment)\n",
+    "  lr = LogReader(segment, sources=[comma_car_segments_source])\n",
     "  CP = lr.first(\"carParams\")\n",
     "  if CP is None:\n",
     "    progress_bar.update()\n",

tools/car_porting/examples/ford_vin_fingerprint.ipynb

@@ -20,7 +20,7 @@
    "source": [
     "\"\"\"In this example, we use the public comma car segments database to check if vin fingerprinting is feasible for ford.\"\"\"\n",
     "\n",
-    "from openpilot.tools.lib.logreader import LogReader\n",
+    "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n",
     "from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database\n",
     "from opendbc.car.ford.values import CAR\n",
     "\n",
@@ -100,7 +100,7 @@
     "  if platform not in database:\n",
     "    print(f\"Skipping platform: {platform}, no data available\")\n",
     "    continue\n",
-    "  \n",
+    "\n",
     "  all_segments = database[platform]\n",
     "\n",
     "  NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n",
@@ -110,7 +110,7 @@
     "  segments = random.sample(all_segments, NUM_SEGMENTS)\n",
     "\n",
     "  for segment in segments:\n",
-    "    lr = LogReader(segment)\n",
+    "    lr = LogReader(segment, sources=[comma_car_segments_source])\n",
     "    CP = lr.first(\"carParams\")\n",
     "    if \"FORD\" not in CP.carFingerprint:\n",
     "      print(segment, CP.carFingerprint)\n",

tools/car_porting/examples/hkg_canfd_gear_message.ipynb

@@ -72,7 +72,7 @@
     "  #if platform not in database:\n",
     "  #  print(f\"Skipping platform: {platform}, no data available\")\n",
     "  #  continue\n",
-    "  \n",
+    "\n",
     "  all_segments = database[platform]\n",
     "\n",
     "  NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n",
@@ -198,12 +198,12 @@
     "from opendbc.car.hyundai.hyundaicanfd import CanBus\n",
     "\n",
     "from openpilot.selfdrive.pandad import can_capnp_to_list\n",
-    "from openpilot.tools.lib.logreader import LogReader\n",
+    "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n",
     "\n",
     "message_names = [\"GEAR_SHIFTER\", \"ACCELERATOR\", \"GEAR\", \"GEAR_ALT\", \"GEAR_ALT_2\"]\n",
     "\n",
     "for segment in TEST_SEGMENTS:\n",
-    "  lr = LogReader(segment)\n",
+    "  lr = LogReader(segment, sources=[comma_car_segments_source])\n",
     "  CP = lr.first(\"carParams\")\n",
     "  if CP is None:\n",
     "    continue\n",

tools/lib/api.py

@@ -2,6 +2,8 @@ import os
 import requests
 API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
 
+# TODO: this should be merged into common.api
+
 class CommaApi:
   def __init__(self, token=None):
     self.session = requests.Session()

tools/lib/comma_car_segments.py

@@ -14,7 +14,8 @@ def get_comma_car_segments_database():
 
   ret = {}
   for platform in database:
-    ret[MIGRATION.get(platform, platform)] = database[platform]
+    # TODO: remove this when commaCarSegments is updated to remove selector
+    ret[MIGRATION.get(platform, platform)] = [s.rstrip('/s') for s in database[platform]]
 
   return ret
 
@@ -86,5 +87,5 @@ def get_repo_url(path):
     return get_repo_raw_url(path)
 
 
-def get_url(route, segment, file="rlog.bz2"):
+def get_url(route, segment, file="rlog.zst"):
   return get_repo_url(f"segments/{route.replace('|', '/')}/{segment}/{file}")

tools/lib/filereader.py

@@ -1,6 +1,8 @@
 import os
 import posixpath
 import socket
+from functools import cache
+from openpilot.common.retry import retry
 from urllib.parse import urlparse
 
 from openpilot.tools.lib.url_file import URLFile
@@ -8,14 +10,16 @@ from openpilot.tools.lib.url_file import URLFile
 DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/")
 
 
-def internal_source_available(url=DATA_ENDPOINT):
+@cache
+@retry(delay=0.0)
+def internal_source_available(url: str) -> bool:
   if os.path.isdir(url):
     return True
 
   try:
     hostname = urlparse(url).hostname
     port = urlparse(url).port or 80
-    with socket.socket(socket.AF_INET,socket.SOCK_STREAM) as s:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
       s.settimeout(0.5)
       s.connect((hostname, port))
     return True
@@ -30,6 +34,7 @@ def resolve_name(fn):
   return fn
 
 
+@cache
 def file_exists(fn):
   fn = resolve_name(fn)
   if fn.startswith(("http://", "https://")):

tools/lib/framereader.py

@@ -56,6 +56,7 @@ def decompress_video_data(rawdat, w, h, pix_fmt="rgb24", vid_fmt='hevc') -> np.n
           "-"]
   dat = subprocess.check_output(args, input=rawdat)
 
+  ret: np.ndarray
   if pix_fmt == "rgb24":
     ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, h, w, 3)
   elif pix_fmt in ["nv12", "yuv420p"]:

tools/lib/helpers.py

@@ -9,7 +9,7 @@ class RE:
 
   INDEX = r'-?[0-9]+'
   SLICE = fr'(?P<start>{INDEX})?:?(?P<end>{INDEX})?:?(?P<step>{INDEX})?'
-  SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P<slice>({SLICE})))?(?:/(?P<selector>([qras])))?'
+  SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P<slice>({SLICE})))?(?:/(?P<selector>([qra])))?'
 
   BOOTLOG_NAME = ROUTE_NAME
 

tools/lib/logreader.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 import bz2
-from functools import cache, partial
+from functools import partial
 import multiprocessing
 import capnp
 import enum
@@ -13,14 +13,16 @@ import warnings
 import zstandard as zstd
 
 from collections.abc import Callable, Iterable, Iterator
+from typing import cast
 from urllib.parse import parse_qs, urlparse
 
 from cereal import log as capnp_log
 from openpilot.common.swaglog import cloudlog
 from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url
 from openpilot.tools.lib.openpilotci import get_url
-from openpilot.tools.lib.filereader import FileReader, file_exists, internal_source_available
-from openpilot.tools.lib.route import Route, SegmentRange
+from openpilot.tools.lib.filereader import DATA_ENDPOINT, FileReader, file_exists, internal_source_available
+from openpilot.tools.lib.route import QCAMERA_FILENAMES, CAMERA_FILENAMES, DCAMERA_FILENAMES, \
+  ECAMERA_FILENAMES, BOOTLOG_FILENAMES, Route, SegmentRange
 from openpilot.tools.lib.log_time_series import msgs_to_time_series
 
 LogMessage = type[capnp._DynamicStructReader]
@@ -96,14 +98,23 @@ class _LogFileReader:
 class ReadMode(enum.StrEnum):
   RLOG = "r"  # only read rlogs
   QLOG = "q"  # only read qlogs
-  SANITIZED = "s"  # read from the commaCarSegments database
   AUTO = "a"  # default to rlogs, fallback to qlogs
   AUTO_INTERACTIVE = "i"  # default to rlogs, fallback to qlogs with a prompt from the user
 
 
+class FileName(enum.Enum):
+  #TODO use the ones from route.py
+  RLOG = ("rlog.zst", "rlog.bz2")
+  QLOG = ("qlog.zst", "qlog.bz2")
+  QCAMERA = QCAMERA_FILENAMES
+  FCAMERA = CAMERA_FILENAMES
+  ECAMERA = ECAMERA_FILENAMES
+  DCAMERA = DCAMERA_FILENAMES
+  BOOTLOG = BOOTLOG_FILENAMES
+
+
 LogPath = str | None
-ValidFileCallable = Callable[[LogPath], bool]
-Source = Callable[[SegmentRange, ReadMode], list[LogPath]]
+Source = Callable[[SegmentRange, FileName], list[LogPath]]
 
 InternalUnavailableException = Exception("Internal source not available")
 
@@ -112,139 +123,129 @@ class LogsUnavailable(Exception):
   pass
 
 
-@cache
-def default_valid_file(fn: LogPath) -> bool:
-  return fn is not None and file_exists(fn)
-
-
-def auto_strategy(rlog_paths: list[LogPath], qlog_paths: list[LogPath], interactive: bool, valid_file: ValidFileCallable) -> list[LogPath]:
-  # auto select logs based on availability
-  missing_rlogs = [rlog is None or not valid_file(rlog) for rlog in rlog_paths].count(True)
-  if missing_rlogs != 0:
-    if interactive:
-      if input(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y":
-        return rlog_paths
-    else:
-      cloudlog.warning(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, falling back to qlogs for those segments...")
-
-    return [rlog if valid_file(rlog) else (qlog if valid_file(qlog) else None)
-            for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)]
-  return rlog_paths
-
-
-def apply_strategy(mode: ReadMode, rlog_paths: list[LogPath], qlog_paths: list[LogPath], valid_file: ValidFileCallable = default_valid_file) -> list[LogPath]:
-  if mode == ReadMode.RLOG:
-    return rlog_paths
-  elif mode == ReadMode.QLOG:
-    return qlog_paths
-  elif mode == ReadMode.AUTO:
-    return auto_strategy(rlog_paths, qlog_paths, False, valid_file)
-  elif mode == ReadMode.AUTO_INTERACTIVE:
-    return auto_strategy(rlog_paths, qlog_paths, True, valid_file)
-  raise ValueError(f"invalid mode: {mode}")
-
-
-def comma_api_source(sr: SegmentRange, mode: ReadMode) -> list[LogPath]:
+def comma_api_source(sr: SegmentRange, fns: FileName) -> list[LogPath]:
   route = Route(sr.route_name)
 
-  rlog_paths = [route.log_paths()[seg] for seg in sr.seg_idxs]
-  qlog_paths = [route.qlog_paths()[seg] for seg in sr.seg_idxs]
-
   # comma api will have already checked if the file exists
-  def valid_file(fn):
-    return fn is not None
-
-  return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file)
+  if fns == FileName.RLOG:
+    return [route.log_paths()[seg] for seg in sr.seg_idxs]
+  else:
+    return [route.qlog_paths()[seg] for seg in sr.seg_idxs]
 
 
-def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> list[LogPath]:
-  if not internal_source_available():
+def internal_source(sr: SegmentRange, fns: FileName, endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]:
+  if not internal_source_available(endpoint_url):
     raise InternalUnavailableException
 
   def get_internal_url(sr: SegmentRange, seg, file):
-    return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}"
+    return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}"
 
-  # TODO: list instead of using static URLs to support routes with multiple file extensions
-  rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs]
-  qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in sr.seg_idxs]
-
-  return apply_strategy(mode, rlog_paths, qlog_paths)
+  return eval_source([[get_internal_url(sr, seg, fn) for fn in fns.value] for seg in sr.seg_idxs])
 
 
-def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") -> list[LogPath]:
-  return internal_source(sr, mode, file_ext)
+def openpilotci_source(sr: SegmentRange, fns: FileName) -> list[LogPath]:
+  return eval_source([[get_url(sr.route_name, seg, fn) for fn in fns.value] for seg in sr.seg_idxs])
 
 
-def openpilotci_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> list[LogPath]:
-  rlog_paths = [get_url(sr.route_name, seg, f"rlog.{file_ext}") for seg in sr.seg_idxs]
-  qlog_paths = [get_url(sr.route_name, seg, f"qlog.{file_ext}") for seg in sr.seg_idxs]
-
-  return apply_strategy(mode, rlog_paths, qlog_paths)
+def comma_car_segments_source(sr: SegmentRange, fns: FileName) -> list[LogPath]:
+  return eval_source([get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs])
 
 
-def openpilotci_source_zst(sr: SegmentRange, mode: ReadMode) -> list[LogPath]:
-  return openpilotci_source(sr, mode, "zst")
-
-
-def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG) -> list[LogPath]:
-  return [get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs]
-
-
-def testing_closet_source(sr: SegmentRange, mode=ReadMode.RLOG) -> list[LogPath]:
-  if not internal_source_available('http://testing.comma.life'):
-    raise InternalUnavailableException
-  return [f"http://testing.comma.life/download/{sr.route_name.replace('|', '/')}/{seg}/rlog" for seg in sr.seg_idxs]
-
-
-def direct_source(file_or_url: str) -> list[LogPath]:
+def direct_source(file_or_url: str) -> list[str]:
   return [file_or_url]
 
 
-def get_invalid_files(files):
-  for f in files:
-    if f is None or not file_exists(f):
-      yield f
+def eval_source(files: list[list[str] | str]) -> list[LogPath]:
+  # Returns valid file URLs given a list of possible file URLs for each segment (e.g. rlog.bz2, rlog.zst)
+  valid_files: list[LogPath] = []
+
+  for urls in files:
+    if isinstance(urls, str):
+      urls = [urls]
+
+    for url in urls:
+      if file_exists(url):
+        valid_files.append(url)
+        break
+    else:
+      valid_files.append(None)
+
+  return valid_files
 
 
-def check_source(source: Source, *args) -> list[LogPath]:
-  files = source(*args)
-  assert len(files) > 0, "No files on source"
-  assert next(get_invalid_files(files), False) is False, "Some files are invalid"
-  return files
-
-
-def auto_source(sr: SegmentRange, mode=ReadMode.RLOG, sources: list[Source] = None) -> list[LogPath]:
-  if mode == ReadMode.SANITIZED:
-    return comma_car_segments_source(sr, mode)
-
-  if sources is None:
-    sources = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst,
-               comma_api_source, comma_car_segments_source, testing_closet_source]
+def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) -> list[str]:
   exceptions = {}
 
-  # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source
-  if mode in [ReadMode.AUTO, ReadMode.AUTO_INTERACTIVE]:
+  sr = SegmentRange(identifier)
+  mode = default_mode if sr.selector is None else ReadMode(sr.selector)
+
+  if mode == ReadMode.QLOG:
+    try_fns = [FileName.QLOG]
+  else:
+    try_fns = [FileName.RLOG]
+
+  # If selector allows it, fallback to qlogs
+  if mode in (ReadMode.AUTO, ReadMode.AUTO_INTERACTIVE):
+    try_fns.append(FileName.QLOG)
+
+  # Build a dict of valid files as we evaluate each source. May contain mix of rlogs, qlogs, and None.
+  # This function only returns when we've sourced all files, or throws an exception
+  valid_files: dict[int, LogPath] = {}
+  for fn in try_fns:
     for source in sources:
       try:
-        return check_source(source, sr, ReadMode.RLOG)
-      except Exception:
-        pass
+        files = source(sr, fn)
 
-  # Automatically determine viable source
-  for source in sources:
-    try:
-      return check_source(source, sr, mode)
-    except Exception as e:
-      exceptions[source.__name__] = e
+        # Check every source returns an expected number of files
+        assert len(files) == len(valid_files) or len(valid_files) == 0, f"Source {source.__name__} returned unexpected number of files"
 
-  raise LogsUnavailable("auto_source could not find any valid source, exceptions for sources:\n  - " +
-                        "\n  - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()]))
+        # Build a dict of valid files
+        for idx, f in enumerate(files):
+          if valid_files.get(idx) is None:
+            valid_files[idx] = f
+
+        # We've found all files, return them
+        if all(f is not None for f in valid_files.values()):
+          return cast(list[str], list(valid_files.values()))
+
+      except Exception as e:
+        exceptions[source.__name__] = e
+
+    if fn == try_fns[0]:
+      missing_logs = list(valid_files.values()).count(None)
+      if mode == ReadMode.AUTO:
+        cloudlog.warning(f"{missing_logs}/{len(valid_files)} rlogs were not found, falling back to qlogs for those segments...")
+      elif mode == ReadMode.AUTO_INTERACTIVE:
+        if input(f"{missing_logs}/{len(valid_files)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/N) ").lower() != "y":
+          break
+
+  missing_logs = list(valid_files.values()).count(None)
+  raise LogsUnavailable(f"{missing_logs}/{len(valid_files)} logs were not found, please ensure all logs " +
+                        "are uploaded. You can fall back to qlogs with '/a' selector at the end of the route name.\n\n" +
+                        "Exceptions for sources:\n  - " + "\n  - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()]))
 
 
 def parse_indirect(identifier: str) -> str:
   if "useradmin.comma.ai" in identifier:
     query = parse_qs(urlparse(identifier).query)
-    return query["onebox"][0]
+    identifier = query["onebox"][0]
+  elif "connect.comma.ai" in identifier:
+    path = urlparse(identifier).path.strip("/").split("/")
+    path = ['/'.join(path[:2]), *path[2:]]  # recombine log id
+
+    identifier = path[0]
+    if len(path) > 2:
+      # convert url with seconds to segments
+      start, end = int(path[1]) // 60, int(path[2]) // 60 + 1
+      identifier = f"{identifier}/{start}:{end}"
+
+      # add selector if it exists
+      if len(path) > 3:
+        identifier += f"/{path[3]}"
+    else:
+      # add selector if it exists
+      identifier = "/".join(path)
+
   return identifier
 
 
@@ -255,7 +256,7 @@ def parse_direct(identifier: str):
 
 
 class LogReader:
-  def _parse_identifier(self, identifier: str) -> list[LogPath]:
+  def _parse_identifier(self, identifier: str) -> list[str]:
     # useradmin, etc.
     identifier = parse_indirect(identifier)
 
@@ -264,20 +265,16 @@ class LogReader:
     if direct_parsed is not None:
       return direct_source(identifier)
 
-    sr = SegmentRange(identifier)
-    mode = self.default_mode if sr.selector is None else ReadMode(sr.selector)
-
-    identifiers = self.source(sr, mode)
-
-    invalid_count = len(list(get_invalid_files(identifiers)))
-    assert invalid_count == 0, (f"{invalid_count}/{len(identifiers)} invalid log(s) found, please ensure all logs " +
-                                "are uploaded or auto fallback to qlogs with '/a' selector at the end of the route name.")
+    identifiers = auto_source(identifier, self.sources, self.default_mode)
     return identifiers
 
   def __init__(self, identifier: str | list[str], default_mode: ReadMode = ReadMode.RLOG,
-               source: Source = auto_source, sort_by_time=False, only_union_types=False):
+               sources: list[Source] = None, sort_by_time=False, only_union_types=False):
+    if sources is None:
+      sources = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source]
+
     self.default_mode = default_mode
-    self.source = source
+    self.sources = sources
     self.identifier = identifier
     if isinstance(identifier, str):
       self.identifier = [identifier]

tools/lib/route.py

@@ -10,12 +10,13 @@ from openpilot.tools.lib.auth_config import get_token
 from openpilot.tools.lib.api import APIError, CommaApi
 from openpilot.tools.lib.helpers import RE
 
-QLOG_FILENAMES = ['qlog', 'qlog.bz2', 'qlog.zst']
-QCAMERA_FILENAMES = ['qcamera.ts']
-LOG_FILENAMES = ['rlog', 'rlog.bz2', 'raw_log.bz2', 'rlog.zst']
-CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc']
-DCAMERA_FILENAMES = ['dcamera.hevc']
-ECAMERA_FILENAMES = ['ecamera.hevc']
+QLOG_FILENAMES = ('qlog.bz2', 'qlog.zst', 'qlog')
+QCAMERA_FILENAMES = ('qcamera.ts',)
+LOG_FILENAMES = ('rlog.bz2', 'raw_log.bz2', 'rlog.zst', 'rlog')
+CAMERA_FILENAMES = ('fcamera.hevc', 'video.hevc')
+DCAMERA_FILENAMES = ('dcamera.hevc',)
+ECAMERA_FILENAMES = ('ecamera.hevc',)
+BOOTLOG_FILENAMES = ('bootlog.zst', 'bootlog.bz2', 'bootlog')
 
 
 class Route:
@@ -215,12 +216,20 @@ class RouteName:
   @property
   def dongle_id(self) -> str: return self._dongle_id
 
+  @property
+  def log_id(self) -> str: return self._time_str
+
   @property
   def time_str(self) -> str: return self._time_str
 
+  @property
+  def azure_prefix(self):
+    return f'{self.dongle_id}/{self.log_id}'
+
   def __str__(self) -> str: return self._canonical_name
 
 
+
 class SegmentName:
   # TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances
   # of this class instead of manually constructing a segment name (use canonical_name prop instead)
@@ -241,12 +250,23 @@ class SegmentName:
   @property
   def canonical_name(self) -> str: return self._canonical_name
 
+  #TODO should only use one name
+  @property
+  def data_name(self) -> str: return f"{self._route_name.canonical_name}/{self._num}"
+
+  @property
+  def azure_prefix(self):
+    return f'{self.dongle_id}/{self.log_id}/{self._num}'
+
   @property
   def dongle_id(self) -> str: return self._route_name.dongle_id
 
   @property
   def time_str(self) -> str: return self._route_name.time_str
 
+  @property
+  def log_id(self) -> str: return self._route_name.time_str
+
   @property
   def segment_num(self) -> int: return self._num
 
@@ -258,6 +278,29 @@ class SegmentName:
 
   def __str__(self) -> str: return self._canonical_name
 
+  @staticmethod
+  def from_file_name(file_name):
+    # ??????/xxxxxxxxxxxxxxxx|1111-11-11-11--11-11-11/1/rlog.bz2
+    dongle_id, route_name, segment_num = file_name.replace('|','/').split('/')[-4:-1]
+    return SegmentName(dongle_id + "|" + route_name + "--" + segment_num)
+
+  @staticmethod
+  def from_device_key(dongle_id, key):
+    # 2018-05-07--18-56-13--5/rlog.bz2
+    segment_name = key.split('/')[0]
+    return SegmentName(dongle_id + "|" + segment_name)
+
+  @staticmethod
+  def from_file_key(key):
+    # 38c52c217150700f/2018-05-07--18-56-13/5/rlog.bz2
+    az_prefix = '/'.join(key.split('/')[:3])
+    return SegmentName.from_azure_prefix(az_prefix)
+
+  @staticmethod
+  def from_azure_prefix(prefix):
+    # xxxxxxxx/1111-11-11-11--11-11-11/0
+    dongle_id, route_name, segment_num = prefix.split("/")
+    return SegmentName(dongle_id + "|" + route_name + "--" + segment_num)
 
 @cache
 def get_max_seg_number_cached(sr: 'SegmentRange') -> int:
@@ -320,3 +363,4 @@ class SegmentRange:
 
   def __repr__(self) -> str:
     return self.__str__()
+

tools/lib/tests/test_logreader.py

@@ -10,7 +10,7 @@ import requests
 from parameterized import parameterized
 
 from cereal import log as capnp_log
-from openpilot.tools.lib.logreader import LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException
+from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException
 from openpilot.tools.lib.route import SegmentRange
 from openpilot.tools.lib.url_file import URLFileException
 
@@ -193,17 +193,17 @@ class TestLogReader:
 
     with subtests.test("interactive_yes"):
       mocker.patch("sys.stdin", new=io.StringIO("y\n"))
-      lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source)
+      lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source])
       log_len = len(list(lr))
       assert qlog_len == log_len
 
     with subtests.test("interactive_no"):
       mocker.patch("sys.stdin", new=io.StringIO("n\n"))
-      with pytest.raises(AssertionError):
-        lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source)
+      with pytest.raises(LogsUnavailable):
+        lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source])
 
     with subtests.test("non_interactive"):
-      lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, source=comma_api_source)
+      lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, sources=[comma_api_source])
       log_len = len(list(lr))
       assert qlog_len == log_len
 

tools/lib/url_file.py

@@ -131,13 +131,13 @@ class URLFile:
       download_range = True
 
     if self._debug:
-      t1 = time.time()
+      t1 = time.monotonic()
 
     response = self._request('GET', self._url, headers=headers)
     ret = response.data
 
     if self._debug:
-      t2 = time.time()
+      t2 = time.monotonic()
       if t2 - t1 > 0.1:
         print(f"get {self._url} {headers!r} {t2 - t1:.3f} slow")
 

tools/op.sh

@@ -231,7 +231,7 @@ function op_setup() {
 
   echo "Getting git submodules..."
   st="$(date +%s)"
-  if ! git submodule update --filter=blob:none --jobs 4 --init --recursive; then
+  if ! git submodule update --jobs 4 --init --recursive; then
     echo -e " ↳ [${RED}✗${NC}] Getting git submodules failed!"
     loge "ERROR_GIT_SUBMODULES"
     return 1
@@ -287,6 +287,11 @@ function op_adb() {
   op_run_command tools/scripts/adb_ssh.sh
 }
 
+function op_ssh() {
+  op_before_cmd
+  op_run_command tools/scripts/ssh.py "$@"
+}
+
 function op_check() {
   VERBOSE=1
   op_before_cmd
@@ -412,6 +417,7 @@ function op_default() {
   echo -e "  ${BOLD}cabana${NC}       Run Cabana"
   echo -e "  ${BOLD}clip${NC}         Run clip (linux only)"
   echo -e "  ${BOLD}adb${NC}          Run adb shell"
+  echo -e "  ${BOLD}ssh${NC}          comma prime SSH helper"
   echo ""
   echo -e "${BOLD}${UNDERLINE}Commands [Testing]:${NC}"
   echo -e "  ${BOLD}sim${NC}          Run openpilot in a simulator"
@@ -471,6 +477,7 @@ function _op() {
     restart )       shift 1; op_restart "$@" ;;
     post-commit )   shift 1; op_install_post_commit "$@" ;;
     adb )           shift 1; op_adb "$@" ;;
+    ssh )           shift 1; op_ssh "$@" ;;
     * ) op_default "$@" ;;
   esac
 }

tools/replay/framereader.cc

@@ -80,7 +80,13 @@ bool FrameReader::loadFromFile(CameraType type, const std::string &file, bool no
   }
   input_ctx->probesize = 10 * 1024 * 1024;  // 10MB
 
-  decoder_ = decoder_manager.acquire(type, input_ctx->streams[0]->codecpar, !no_hw_decoder);
+  video_stream_idx_ = av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);
+  if (video_stream_idx_ < 0) {
+    rError("No video stream found in file");
+    return false;
+  }
+
+  decoder_ = decoder_manager.acquire(type, input_ctx->streams[video_stream_idx_]->codecpar, !no_hw_decoder);
   if (!decoder_) {
     return false;
   }
@@ -90,7 +96,9 @@ bool FrameReader::loadFromFile(CameraType type, const std::string &file, bool no
   AVPacket pkt;
   packets_info.reserve(60 * 20);  // 20fps, one minute
   while (!(abort && *abort) && av_read_frame(input_ctx, &pkt) == 0) {
-    packets_info.emplace_back(PacketInfo{.flags = pkt.flags, .pos = pkt.pos});
+    if (pkt.stream_index == video_stream_idx_) {
+      packets_info.emplace_back(PacketInfo{.flags = pkt.flags, .pos = pkt.pos});
+    }
     av_packet_unref(&pkt);
   }
   avio_seek(input_ctx->pb, 0, SEEK_SET);
@@ -168,17 +176,17 @@ bool VideoDecoder::initHardwareDecoder(AVHWDeviceType hw_device_type) {
 }
 
 bool VideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
-  int from_idx = idx;
+  int current_idx = idx;
   if (idx != reader->prev_idx + 1) {
     // seeking to the nearest key frame
     for (int i = idx; i >= 0; --i) {
       if (reader->packets_info[i].flags & AV_PKT_FLAG_KEY) {
-        from_idx = i;
+        current_idx = i;
         break;
       }
     }
 
-    auto pos = reader->packets_info[from_idx].pos;
+    auto pos = reader->packets_info[current_idx].pos;
     int ret = avformat_seek_file(reader->input_ctx, 0, pos, pos, pos, AVSEEK_FLAG_BYTE);
     if (ret < 0) {
       rError("Failed to seek to byte position %lld: %d", pos, AVERROR(ret));
@@ -188,18 +196,27 @@ bool VideoDecoder::decode(FrameReader *reader, int idx, VisionBuf *buf) {
   }
   reader->prev_idx = idx;
 
-  bool result = false;
   AVPacket pkt;
-  for (int i = from_idx; i <= idx; ++i) {
-    if (av_read_frame(reader->input_ctx, &pkt) == 0) {
-      AVFrame *f = decodeFrame(&pkt);
-      if (f && i == idx) {
-        result = copyBuffer(f, buf);
-      }
+  while (av_read_frame(reader->input_ctx, &pkt) >= 0) {
+    // Skip non-video packets
+    if (pkt.stream_index != reader->video_stream_idx_) {
       av_packet_unref(&pkt);
+      continue;
+    }
+
+    AVFrame *frame = decodeFrame(&pkt);
+    av_packet_unref(&pkt);
+    if (!frame) {
+      rError("Failed to decode frame at index %d", current_idx);
+      return false;
+    }
+
+    if (current_idx++ == idx) {
+      return copyBuffer(frame, buf);
     }
   }
-  return result;
+  rError("Failed to find frame at index %d", idx);
+  return false;
 }
 
 AVFrame *VideoDecoder::decodeFrame(AVPacket *pkt) {

tools/replay/framereader.h

@@ -28,6 +28,7 @@ public:
 
   VideoDecoder *decoder_ = nullptr;
   AVFormatContext *input_ctx = nullptr;
+  int video_stream_idx_ = -1;
   int prev_idx = -1;
   struct PacketInfo {
     int flags;

tools/replay/unlog_ci_segment.py

@@ -42,7 +42,7 @@ def replay(route, segment, loop):
     msg = msgs[i].as_builder()
     next_msg = msgs[i + 1]
 
-    start_time = time.time()
+    start_time = time.monotonic()
     w = msg.which()
 
     if w == 'roadCameraState':
@@ -63,7 +63,7 @@ def replay(route, segment, loop):
       socks[w] = None
 
     lag += (next_msg.logMonoTime - msg.logMonoTime) / 1e9
-    lag -= time.time() - start_time
+    lag -= time.monotonic() - start_time
 
     dt = max(lag, 0.0)
     lag -= dt

tools/scripts/ssh.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+import os
+import sys
+import argparse
+import re
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.tools.lib.auth_config import get_token
+from openpilot.tools.lib.api import CommaApi
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="A helper for connecting to devices over the comma prime SSH proxy.\
+      Adding your SSH key to your SSH config is recommended for more convenient use; see https://docs.comma.ai/how-to/connect-to-comma/.")
+  parser.add_argument("device", help="device name or dongle id")
+  parser.add_argument("--host", help="ssh jump server host", default="ssh.comma.ai")
+  parser.add_argument("--port", help="ssh jump server port", default=22, type=int)
+  parser.add_argument("--key", help="ssh key", default=os.path.join(BASEDIR, "system/hardware/tici/id_rsa"))
+  parser.add_argument("--debug", help="enable debug output", action="store_true")
+  args = parser.parse_args()
+
+  r = CommaApi(get_token()).get("v1/me/devices")
+  devices = {x['dongle_id']: x['alias'] for x in r}
+
+  if not re.match("[0-9a-zA-Z]{16}", args.device):
+    user_input = args.device.replace(" ", "").lower()
+    matches = { k: v for k, v in devices.items() if isinstance(v, str) and user_input in v.replace(" ", "").lower() }
+    if len(matches) == 1:
+      dongle_id = list(matches.keys())[0]
+    else:
+      print(f"failed to look up dongle id for \"{args.device}\"", file=sys.stderr)
+      if len(matches) > 1:
+        print("found multiple matches:", file=sys.stderr)
+        for k, v in matches.items():
+          print(f"  \"{v}\" ({k})", file=sys.stderr)
+      exit(1)
+  else:
+    dongle_id = args.device
+
+  name = dongle_id
+  if dongle_id in devices:
+    name = f"{devices[dongle_id]} ({dongle_id})"
+  print(f"connecting to {name} through {args.host}:{args.port} ...")
+
+  command = [
+    "ssh",
+    "-i", args.key,
+    "-o", f"ProxyCommand=ssh -i {args.key} -W %h:%p -p %p %h@{args.host}",
+    "-p", str(args.port),
+  ]
+  if args.debug:
+    command += ["-v"]
+  command += [
+    f"comma@{dongle_id}",
+  ]
+  if args.debug:
+    print(" ".join([f"'{c}'" if " " in c else c for c in command]))
+  os.execvp(command[0], command)

tools/sim/lib/simulated_sensors.py

@@ -53,7 +53,7 @@ class SimulatedSensors:
     for _ in range(10):
       dat = messaging.new_message('gpsLocationExternal', valid=True)
       dat.gpsLocationExternal = {
-        "unixTimestampMillis": int(time.time() * 1000),
+        "unixTimestampMillis": int(time.time() * 1000),  # noqa: TID251
         "flags": 1,  # valid fix
         "horizontalAccuracy": 1.0,
         "verticalAccuracy": 1.0,
@@ -109,7 +109,7 @@ class SimulatedSensors:
       self.camerad.cam_send_yuv_wide_road(yuv)
 
   def update(self, simulator_state: 'SimulatorState', world: 'World'):
-    now = time.time()
+    now = time.monotonic()
     self.send_imu_message(simulator_state)
     self.send_gps_message(simulator_state)