Cabana: show video thumbnail over slider (#26689)

* show thumbnail

* copy segments vector

* cleanup

cleanup mouseMoveEvent

cleanup include

* move lambda to memeber function

* reduce lock time

* private functions

* split to small functions

tools/cabana/canmessages.h

@@ -31,7 +31,7 @@ public:
   QList<QPointF> findSignalValues(const QString&id, const Signal* signal, double value, FindFlags flag, int max_count);
   bool eventFilter(const Event *event);
 
-  inline QString route() const { return replay->route()->name(); }
+  inline QString routeName() const { return replay->route()->name(); }
   inline QString carFingerprint() const { return replay->carFingerprint().c_str(); }
   inline double totalSeconds() const { return replay->totalSeconds(); }
   inline double routeStartTime() const { return replay->routeStartTime() / (double)1e9; }
@@ -39,6 +39,7 @@ public:
   const std::deque<CanData> messages(const QString &id);
   inline const CanData &lastMessage(const QString &id) { return can_msgs[id]; }
 
+  inline const Route* route() const { return replay->route(); }
   inline const std::vector<Event *> *events() const { return replay->events(); }
   inline void setSpeed(float speed) { replay->setSpeed(speed); }
   inline bool isPaused() const { return replay->isPaused(); }

tools/cabana/mainwin.cc

@@ -71,7 +71,7 @@ MainWindow::MainWindow() : QMainWindow() {
   right_hlayout->addWidget(fingerprint_label, 0, Qt::AlignLeft);
 
   // TODO: click to select another route.
-  right_hlayout->addWidget(new QLabel(can->route()), 0, Qt::AlignRight);
+  right_hlayout->addWidget(new QLabel(can->routeName()), 0, Qt::AlignRight);
   r_layout->addLayout(right_hlayout);
 
   video_widget = new VideoWidget(this);

tools/cabana/videowidget.cc

@@ -1,13 +1,17 @@
 #include "tools/cabana/videowidget.h"
 
+#include <QBuffer>
 #include <QButtonGroup>
 #include <QDateTime>
 #include <QHBoxLayout>
 #include <QMouseEvent>
 #include <QPainter>
+#include <QPixmap>
 #include <QStyleOptionSlider>
 #include <QTimer>
+#include <QToolTip>
 #include <QVBoxLayout>
+#include <QtConcurrent>
 
 inline QString formatTime(int seconds) {
   return QDateTime::fromTime_t(seconds).toString(seconds > 60 * 60 ? "hh:mm:ss" : "mm:ss");
@@ -92,7 +96,50 @@ Slider::Slider(QWidget *parent) : QSlider(Qt::Horizontal, parent) {
     timeline = can->getTimeline();
     update();
   });
+  setMouseTracking(true);
+
   QObject::connect(can, SIGNAL(streamStarted()), timer, SLOT(start()));
+  QObject::connect(can, &CANMessages::streamStarted, this, &Slider::streamStarted);
+}
+
+void Slider::streamStarted() {
+  abort_load_thumbnail = true;
+  thumnail_future.waitForFinished();
+  abort_load_thumbnail = false;
+  thumbnails.clear();
+  thumnail_future = QtConcurrent::run(this, &Slider::loadThumbnails);
+}
+
+void Slider::loadThumbnails() {
+  const auto segments = can->route()->segments();
+  for (auto it = segments.rbegin(); it != segments.rend() && !abort_load_thumbnail; ++it) {
+    std::string qlog = it->second.qlog.toStdString();
+    if (!qlog.empty()) {
+      LogReader log;
+      if (log.load(qlog, &abort_load_thumbnail, {cereal::Event::Which::THUMBNAIL}, true, 0, 3)) {
+        for (auto ev = log.events.cbegin(); ev != log.events.cend() && !abort_load_thumbnail; ++ev) {
+          auto thumb = (*ev)->event.getThumbnail();
+          QString str = getThumbnailString(thumb.getThumbnail());
+          std::lock_guard lk(thumbnail_lock);
+          thumbnails[thumb.getTimestampEof()] = str;
+        }
+      }
+    }
+  }
+}
+
+QString Slider::getThumbnailString(const capnp::Data::Reader &data) {
+  QPixmap thumb;
+  if (thumb.loadFromData(data.begin(), data.size(), "jpeg")) {
+    thumb = thumb.scaled({thumb.width()/3, thumb.height()/3}, Qt::KeepAspectRatio);
+    thumbnail_size = thumb.size();
+    QByteArray bytes;
+    QBuffer buffer(&bytes);
+    buffer.open(QIODevice::WriteOnly);
+    thumb.save(&buffer, "png");
+    return  QString("<img src='data:image/png;base64, %0'>").arg(QString(bytes.toBase64()));
+  }
+  return {};
 }
 
 void Slider::sliderChange(QAbstractSlider::SliderChange change) {
@@ -146,3 +193,18 @@ void Slider::mousePressEvent(QMouseEvent *e) {
     emit sliderReleased();
   }
 }
+
+void Slider::mouseMoveEvent(QMouseEvent *e) {
+  QString thumb;
+  {
+    double seconds = (minimum() + e->pos().x() * ((maximum() - minimum()) / (double)width())) / 1000.0;
+    std::lock_guard lk(thumbnail_lock);
+    auto it = thumbnails.lowerBound((seconds + can->routeStartTime()) * 1e9);
+    if (it != thumbnails.end()) {
+      thumb = it.value();
+    }
+  }
+  QPoint pt = mapToGlobal({e->pos().x() - thumbnail_size.width() / 2, -thumbnail_size.height() - 30});
+  QToolTip::showText(pt, thumb, this, rect());
+  QSlider::mouseMoveEvent(e);
+}

tools/cabana/videowidget.h

@@ -1,5 +1,9 @@
 #pragma once
 
+#include <atomic>
+#include <mutex>
+
+#include <QFuture>
 #include <QLabel>
 #include <QPushButton>
 #include <QSlider>
@@ -12,12 +16,23 @@ class Slider : public QSlider {
 
 public:
   Slider(QWidget *parent);
+
+private:
   void mousePressEvent(QMouseEvent *e) override;
+  void mouseMoveEvent(QMouseEvent *e) override;
   void sliderChange(QAbstractSlider::SliderChange change) override;
   void paintEvent(QPaintEvent *ev) override;
+  void streamStarted();
+  void loadThumbnails();
+  QString getThumbnailString(const capnp::Data::Reader &data);
 
   int slider_x = -1;
   std::vector<std::tuple<int, int, TimelineType>> timeline;
+  std::mutex thumbnail_lock;
+  std::atomic<bool> abort_load_thumbnail = false;
+  QMap<uint64_t, QString> thumbnails;
+  QFuture<void> thumnail_future;
+  QSize thumbnail_size = {};
 };
 
 class VideoWidget : public QWidget {