cabana: dispaly alert info in thumbnail&video player (#27837)

* show alert info in thumbnail * dispaly alert info in video old-commit-hash: 2d061e2041574e250d3fa6b487fd9b4a2958864a
2026-07-02 20:12:07 +08:00 · 2023-04-09 02:34:37 +08:00
parent fb6cfc707a
commit fbaefab443
2 changed files with 92 additions and 25 deletions
@@ -3,6 +3,7 @@
 #include <QButtonGroup>
 #include <QMouseEvent>
 #include <QPainter>
+#include <QStackedLayout>
 #include <QStyleOptionSlider>
 #include <QVBoxLayout>
 #include <QtConcurrent>
@@ -19,7 +20,6 @@ static const QColor timeline_colors[] = {
  [(int)TimelineType::AlertCritical] = QColor(199, 0, 57),
 };

-
 bool sortTimelineBasedOnEventPriority(const std::tuple<int, int, TimelineType> &left, const std::tuple<int, int, TimelineType> &right){
  const static std::map<TimelineType, int> timelinePriority = {
    {  TimelineType::None, 0 },
@@ -29,7 +29,6 @@ bool sortTimelineBasedOnEventPriority(const std::tuple<int, int, TimelineType> &
    {  TimelineType::AlertCritical, 40 },
    {  TimelineType::UserFlag, 35 }
  };
-
  return timelinePriority.at(std::get<2>(left)) < timelinePriority.at(std::get<2>(right));
 }

@@ -92,10 +91,14 @@ QWidget *VideoWidget::createCameraWidget() {
  QWidget *w = new QWidget(this);
  QVBoxLayout *l = new QVBoxLayout(w);
  l->setContentsMargins(0, 0, 0, 0);
-  cam_widget = new CameraWidget("camerad", can->visionStreamType(), false);
+
+  QStackedLayout *stacked = new QStackedLayout();
+  stacked->setStackingMode(QStackedLayout::StackAll);
+  stacked->addWidget(cam_widget = new CameraWidget("camerad", can->visionStreamType(), false));
  cam_widget->setMinimumHeight(MIN_VIDEO_HEIGHT);
  cam_widget->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::MinimumExpanding);
-  l->addWidget(cam_widget);
+  stacked->addWidget(alert_label = new InfoLabel(this));
+  l->addLayout(stacked);

  // slider controls
  slider_layout = new QHBoxLayout();
@@ -130,8 +133,17 @@ void VideoWidget::rangeChanged(double min, double max, bool is_zoomed) {
 }

 void VideoWidget::updateState() {
-  if (!slider->isSliderDown())
+  if (!slider->isSliderDown()) {
    slider->setValue(can->currentSec() * 1000);
+  }
+  std::lock_guard lk(slider->thumbnail_lock);
+  uint64_t mono_time = (can->currentSec() + can->routeStartTime()) * 1e9;
+  auto it = slider->alerts.lower_bound(mono_time);
+  if (it != slider->alerts.end() && (it->first - mono_time) < 1e9) {
+    alert_label->showAlert(it->second);
+  } else {
+    alert_label->showAlert({});
+  }
 }

 void VideoWidget::updatePlayBtnState() {
@@ -144,7 +156,6 @@ Slider::Slider(QWidget *parent) : timer(this), thumbnail_label(this), QSlider(Qt
  timer.callOnTimeout([this]() {
    timeline = can->getTimeline();
    std::sort(timeline.begin(), timeline.end(), sortTimelineBasedOnEventPriority);
-
    update();
  });
  setMouseTracking(true);
@@ -169,11 +180,11 @@ void Slider::streamStarted() {
 void Slider::loadThumbnails() {
  const auto &segments = can->route()->segments();
  for (auto it = segments.rbegin(); it != segments.rend() && !abort_load_thumbnail; ++it) {
+    LogReader log;
    std::string qlog = it->second.qlog.toStdString();
-    if (!qlog.empty()) {
-      LogReader log;
-      if (log.load(qlog, &abort_load_thumbnail, {cereal::Event::Which::THUMBNAIL}, true, 0, 3)) {
-        for (auto ev = log.events.cbegin(); ev != log.events.cend() && !abort_load_thumbnail; ++ev) {
+    if (!qlog.empty() && log.load(qlog, &abort_load_thumbnail, {cereal::Event::Which::THUMBNAIL, cereal::Event::Which::CONTROLS_STATE}, true, 0, 3)) {
+      for (auto ev = log.events.cbegin(); ev != log.events.cend() && !abort_load_thumbnail; ++ev) {
+        if ((*ev)->which == cereal::Event::Which::THUMBNAIL) {
          auto thumb = (*ev)->event.getThumbnail();
          auto data = thumb.getThumbnail();
          if (QPixmap pm; pm.loadFromData(data.begin(), data.size(), "jpeg")) {
@@ -181,6 +192,12 @@ void Slider::loadThumbnails() {
            std::lock_guard lk(thumbnail_lock);
            thumbnails[thumb.getTimestampEof()] = pm;
          }
+        } else if ((*ev)->which == cereal::Event::Which::CONTROLS_STATE) {
+          auto cs = (*ev)->event.getControlsState();
+          if (cs.getAlertType().size() > 0 && cs.getAlertText1().size() > 0) {
+            std::lock_guard lk(thumbnail_lock);
+            alerts.emplace((*ev)->mono_time, AlertInfo{cs.getAlertStatus(), cs.getAlertText1().cStr(), cs.getAlertText2().cStr()});
+          }
        }
      }
    }
@@ -234,15 +251,21 @@ void Slider::mousePressEvent(QMouseEvent *e) {

 void Slider::mouseMoveEvent(QMouseEvent *e) {
  QPixmap thumb;
+  AlertInfo alert;
  double seconds = (minimum() + e->pos().x() * ((maximum() - minimum()) / (double)width())) / 1000.0;
  {
    std::lock_guard lk(thumbnail_lock);
-    auto it = thumbnails.lowerBound((seconds + can->routeStartTime()) * 1e9);
+    uint64_t mono_time = (seconds + can->routeStartTime()) * 1e9;
+    auto it = thumbnails.lowerBound(mono_time);
    if (it != thumbnails.end()) thumb = it.value();
+    auto alert_it = alerts.lower_bound(mono_time);
+    if (alert_it != alerts.end() && (alert_it->first - mono_time) < 1e9) {
+      alert = alert_it->second;
+    }
  }
  int x = std::clamp(e->pos().x() - thumb.width() / 2, THUMBNAIL_MARGIN, rect().right() - thumb.width() - THUMBNAIL_MARGIN);
  int y = -thumb.height() - THUMBNAIL_MARGIN - style()->pixelMetric(QStyle::PM_LayoutVerticalSpacing);
-  thumbnail_label.showPixmap(mapToGlobal({x, y}), utils::formatSeconds(seconds), thumb);
+  thumbnail_label.showPixmap(mapToGlobal({x, y}), utils::formatSeconds(seconds), thumb, alert);
  QSlider::mouseMoveEvent(e);
 }

@@ -251,16 +274,17 @@ void Slider::leaveEvent(QEvent *event) {
  QSlider::leaveEvent(event);
 }

-// ThumbnailLabel
+// InfoLabel

-ThumbnailLabel::ThumbnailLabel(QWidget *parent) : QWidget(parent, Qt::Tool | Qt::FramelessWindowHint) {
+InfoLabel::InfoLabel(QWidget *parent) : QWidget(parent, Qt::Tool | Qt::FramelessWindowHint) {
  setAttribute(Qt::WA_ShowWithoutActivating);
  setVisible(false);
 }

-void ThumbnailLabel::showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm) {
+void InfoLabel::showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm, const AlertInfo &alert) {
  pixmap = pm;
  second = sec;
+  alert_info = alert;
  setVisible(!pm.isNull());
  if (isVisible()) {
    setGeometry({pt, pm.size()});
@@ -268,10 +292,42 @@ void ThumbnailLabel::showPixmap(const QPoint &pt, const QString &sec, const QPix
  }
 }

-void ThumbnailLabel::paintEvent(QPaintEvent *event) {
-  QPainter p(this);
-  p.drawPixmap(0, 0, pixmap);
-  p.setPen(QPen(Qt::white, 2));
-  p.drawRect(rect());
-  p.drawText(rect().adjusted(0, 0, 0, -THUMBNAIL_MARGIN), second, Qt::AlignHCenter | Qt::AlignBottom);
+void InfoLabel::showAlert(const AlertInfo &alert) {
+  alert_info = alert;
+  pixmap = {};
+  setVisible(!alert_info.text1.isEmpty());
+  if (isVisible()) {
+    update();
+  }
+}
+
+void InfoLabel::paintEvent(QPaintEvent *event) {
+  QPainter p(this);
+  p.setPen(QPen(Qt::white, 2));
+  if (!pixmap.isNull()) {
+    p.drawPixmap(0, 0, pixmap);
+    p.drawRect(rect());
+    p.drawText(rect().adjusted(0, 0, 0, -THUMBNAIL_MARGIN), second, Qt::AlignHCenter | Qt::AlignBottom);
+  }
+  if (alert_info.text1.size() > 0) {
+    QColor color = timeline_colors[(int)TimelineType::AlertInfo];
+    if (alert_info.status == cereal::ControlsState::AlertStatus::USER_PROMPT) {
+      color = timeline_colors[(int)TimelineType::AlertWarning];
+    } else if (alert_info.status == cereal::ControlsState::AlertStatus::CRITICAL) {
+      color = timeline_colors[(int)TimelineType::AlertCritical];
+    }
+    color.setAlphaF(0.5);
+    QString text = alert_info.text1;
+    if (!alert_info.text2.isEmpty()) {
+      text += "\n" + alert_info.text2;
+    }
+
+    QFont font;
+    font.setPixelSize(!pixmap.isNull() ? 11 : QFont().pixelSize());
+    p.setFont(font);
+    QRect text_rect = rect().adjusted(2, 2, -2, -2);
+    QRect r = p.fontMetrics().boundingRect(text_rect, Qt::AlignTop | Qt::AlignHCenter | Qt::TextWordWrap, text);
+    p.fillRect(text_rect.left(), r.top(), text_rect.width(), r.height(), color);
+    p.drawText(text_rect, Qt::AlignTop | Qt::AlignHCenter | Qt::TextWordWrap, text);
+  }
 }
@@ -13,13 +13,21 @@
 #include "selfdrive/ui/qt/widgets/cameraview.h"
 #include "tools/cabana/streams/abstractstream.h"

-class ThumbnailLabel : public QWidget {
+struct AlertInfo {
+  cereal::ControlsState::AlertStatus status;
+  QString text1;
+  QString text2;
+};
+
+class InfoLabel : public QWidget {
 public:
-  ThumbnailLabel(QWidget *parent);
-  void showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm);
+  InfoLabel(QWidget *parent);
+  void showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm, const AlertInfo &alert);
+  void showAlert(const AlertInfo &alert);
  void paintEvent(QPaintEvent *event) override;
  QPixmap pixmap;
  QString second;
+  AlertInfo alert_info;
 };

 class Slider : public QSlider {
@@ -43,9 +51,11 @@ private:
  std::mutex thumbnail_lock;
  std::atomic<bool> abort_load_thumbnail = false;
  QMap<uint64_t, QPixmap> thumbnails;
+  std::map<uint64_t, AlertInfo> alerts;
  QFuture<void> thumnail_future;
-  ThumbnailLabel thumbnail_label;
+  InfoLabel thumbnail_label;
  QTimer timer;
+  friend class VideoWidget;
 };

 class VideoWidget : public QFrame {
@@ -65,5 +75,6 @@ protected:
  QLabel *time_label;
  QHBoxLayout *slider_layout;
  QPushButton *play_btn;
+  InfoLabel *alert_label;
  Slider *slider;
 };