store mic audio with toggle (#35595)

* store/send mic audio with toggle

* script to extract audio from logs

* change description and add translation placeholders

* microphone icon

* apply toggle in loggerd

* add legnth and counter

* startFrameIdx counter

* Revert "change description and add translation placeholders"

This reverts commit 7baa1f6de99c6ebe9f9906193da7e83dad79511a.

* send mic data first and then calc

* restore changed description/icon after revert

* adjust fft samples to keep old time window

* remove extract_audio.py since audio is now stored in qcam isntead of rlog

* qt microphone recording icon

* Revert "remove extract_audio.py since audio is now stored in qcam isntead of rlog"

This reverts commit 7a3a75bd8db5376d1e442a3ba931c67550b5f132.

* move extract_audio script and output file by default

* remove length field

* recording indicator swaps sides based on lhd/rhd

* use record icon from comma body

* Update toggle description

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

* update raylib toggle desc cause I did earlier

* microphone --> soundPressure, audioData --> rawAudioData

* cleanup unused var

* update README

* sidebar mic indicator instead of annotated camera

* improve logic readability

* remove startFrameIdx and sequenceNum

* use Q_PROPERTY/setProperty so that update() is actually called on value change

* specify old id for SoundPressure

* fix typo

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

selfdrive/assets/icons/microphone.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fc1f7f31d41f26ea7d6f52b3096f7a91844a3b897bc233a8489253c46f0403b
+size 6324

selfdrive/ui/layouts/settings/toggles.py

@@ -22,6 +22,7 @@ DESCRIPTIONS = {
   "AlwaysOnDM": "Enable driver monitoring even when openpilot is not engaged.",
   'RecordFront': "Upload data from the driver facing camera and help improve the driver monitoring algorithm.",
   "IsMetric": "Display speed in km/h instead of mph.",
+  "RecordAudio": "Record and store microphone audio while driving. The audio will be included in the dashcam video in comma connect.",
 }
 
 
@@ -77,6 +78,12 @@ class TogglesLayout(Widget):
       toggle_item(
         "Use Metric System", DESCRIPTIONS["IsMetric"], self._params.get_bool("IsMetric"), icon="monitoring.png"
       ),
+      toggle_item(
+        "Record Microphone Audio",
+        DESCRIPTIONS["RecordAudio"],
+        self._params.get_bool("RecordAudio"),
+        icon="microphone.png",
+      ),
     ]
 
     self._scroller = Scroller(items, line_separator=True, spacing=0)

selfdrive/ui/qt/offroad/settings.cc

@@ -68,6 +68,13 @@ TogglesPanel::TogglesPanel(SettingsWindow *parent) : ListWidget(parent) {
       "../assets/icons/metric.png",
       false,
     },
+    {
+      "RecordAudio",
+      tr("Record Microphone Audio"),
+      tr("Record and store microphone audio while driving. The audio will be included in the dashcam video in comma connect."),
+      "../assets/icons/microphone.png",
+      true,
+    },
   };
 
 

selfdrive/ui/qt/sidebar.cc

@@ -24,10 +24,11 @@ void Sidebar::drawMetric(QPainter &p, const QPair<QString, QString> &label, QCol
   p.drawText(rect.adjusted(22, 0, 0, 0), Qt::AlignCenter, label.first + "\n" + label.second);
 }
 
-Sidebar::Sidebar(QWidget *parent) : QFrame(parent), onroad(false), flag_pressed(false), settings_pressed(false) {
+Sidebar::Sidebar(QWidget *parent) : QFrame(parent), onroad(false), flag_pressed(false), settings_pressed(false), mic_indicator_pressed(false) {
   home_img = loadPixmap("../assets/images/button_home.png", home_btn.size());
   flag_img = loadPixmap("../assets/images/button_flag.png", home_btn.size());
   settings_img = loadPixmap("../assets/images/button_settings.png", settings_btn.size(), Qt::IgnoreAspectRatio);
+  mic_img = loadPixmap("../assets/icons/microphone.png", QSize(30, 30));
 
   connect(this, &Sidebar::valueChanged, [=] { update(); });
 
@@ -47,12 +48,15 @@ void Sidebar::mousePressEvent(QMouseEvent *event) {
   } else if (settings_btn.contains(event->pos())) {
     settings_pressed = true;
     update();
+  } else if (recording_audio && mic_indicator_btn.contains(event->pos())) {
+    mic_indicator_pressed = true;
+    update();
   }
 }
 
 void Sidebar::mouseReleaseEvent(QMouseEvent *event) {
-  if (flag_pressed || settings_pressed) {
-    flag_pressed = settings_pressed = false;
+  if (flag_pressed || settings_pressed || mic_indicator_pressed) {
+    flag_pressed = settings_pressed = mic_indicator_pressed = false;
     update();
   }
   if (onroad && home_btn.contains(event->pos())) {
@@ -61,6 +65,8 @@ void Sidebar::mouseReleaseEvent(QMouseEvent *event) {
     pm->send("userFlag", msg);
   } else if (settings_btn.contains(event->pos())) {
     emit openSettings();
+  } else if (recording_audio && mic_indicator_btn.contains(event->pos())) {
+    emit openSettings(2, "RecordAudio");
   }
 }
 
@@ -106,6 +112,8 @@ void Sidebar::updateState(const UIState &s) {
     pandaStatus = {{tr("NO"), tr("PANDA")}, danger_color};
   }
   setProperty("pandaStatus", QVariant::fromValue(pandaStatus));
+
+  setProperty("recordingAudio", s.scene.recording_audio);
 }
 
 void Sidebar::paintEvent(QPaintEvent *event) {
@@ -120,6 +128,14 @@ void Sidebar::paintEvent(QPaintEvent *event) {
   p.drawPixmap(settings_btn.x(), settings_btn.y(), settings_img);
   p.setOpacity(onroad && flag_pressed ? 0.65 : 1.0);
   p.drawPixmap(home_btn.x(), home_btn.y(), onroad ? flag_img : home_img);
+  if (recording_audio) {
+    p.setBrush(danger_color);
+    p.setOpacity(mic_indicator_pressed ? 0.65 : 1.0);
+    p.drawRoundedRect(mic_indicator_btn, mic_indicator_btn.height() / 2, mic_indicator_btn.height() / 2);
+    int icon_x = mic_indicator_btn.x() + (mic_indicator_btn.width() - mic_img.width()) / 2;
+    int icon_y = mic_indicator_btn.y() + (mic_indicator_btn.height() - mic_img.height()) / 2;
+    p.drawPixmap(icon_x, icon_y, mic_img);
+  }
   p.setOpacity(1.0);
 
   // network

selfdrive/ui/qt/sidebar.h

@@ -18,6 +18,7 @@ class Sidebar : public QFrame {
   Q_PROPERTY(ItemStatus tempStatus MEMBER temp_status NOTIFY valueChanged);
   Q_PROPERTY(QString netType MEMBER net_type NOTIFY valueChanged);
   Q_PROPERTY(int netStrength MEMBER net_strength NOTIFY valueChanged);
+  Q_PROPERTY(bool recordingAudio MEMBER recording_audio NOTIFY valueChanged);
 
 public:
   explicit Sidebar(QWidget* parent = 0);
@@ -36,8 +37,8 @@ protected:
   void mouseReleaseEvent(QMouseEvent *event) override;
   void drawMetric(QPainter &p, const QPair<QString, QString> &label, QColor c, int y);
 
-  QPixmap home_img, flag_img, settings_img;
-  bool onroad, flag_pressed, settings_pressed;
+  QPixmap home_img, flag_img, settings_img, mic_img;
+  bool onroad, recording_audio, flag_pressed, settings_pressed, mic_indicator_pressed;
   const QMap<cereal::DeviceState::NetworkType, QString> network_type = {
     {cereal::DeviceState::NetworkType::NONE, tr("--")},
     {cereal::DeviceState::NetworkType::WIFI, tr("Wi-Fi")},
@@ -50,6 +51,7 @@ protected:
 
   const QRect home_btn = QRect(60, 860, 180, 180);
   const QRect settings_btn = QRect(50, 35, 200, 117);
+  const QRect mic_indicator_btn = QRect(158, 252, 75, 40);
   const QColor good_color = QColor(255, 255, 255);
   const QColor warning_color = QColor(218, 202, 37);
   const QColor danger_color = QColor(201, 34, 49);

selfdrive/ui/soundd.py

@@ -135,7 +135,7 @@ class Soundd:
     # sounddevice must be imported after forking processes
     import sounddevice as sd
 
-    sm = messaging.SubMaster(['selfdriveState', 'microphone'])
+    sm = messaging.SubMaster(['selfdriveState', 'soundPressure'])
 
     with self.get_stream(sd) as stream:
       rk = Ratekeeper(20)
@@ -144,8 +144,8 @@ class Soundd:
       while True:
         sm.update(0)
 
-        if sm.updated['microphone'] and self.current_alert == AudibleAlert.none: # only update volume filter when not playing alert
-          self.spl_filter_weighted.update(sm["microphone"].soundPressureWeightedDb)
+        if sm.updated['soundPressure'] and self.current_alert == AudibleAlert.none: # only update volume filter when not playing alert
+          self.spl_filter_weighted.update(sm["soundPressure"].soundPressureWeightedDb)
           self.current_volume = self.calculate_volume(float(self.spl_filter_weighted.x))
 
         self.get_audible_alert(sm)

selfdrive/ui/ui.cc

@@ -60,6 +60,9 @@ static void update_state(UIState *s) {
     scene.light_sensor = -1;
   }
   scene.started = sm["deviceState"].getDeviceState().getStarted() && scene.ignition;
+
+  auto params = Params();
+  scene.recording_audio = params.getBool("RecordAudio") && scene.started;
 }
 
 void ui_update_params(UIState *s) {

selfdrive/ui/ui.h

@@ -58,7 +58,7 @@ typedef struct UIScene {
   cereal::LongitudinalPersonality personality;
 
   float light_sensor = -1;
-  bool started, ignition, is_metric;
+  bool started, ignition, is_metric, recording_audio;
   uint64_t started_frame;
 } UIScene;