Record feedback with LKAS button (#35888)

* record feedback with LKAS button

* fix alert test

* slightly simplify feedbackd

* "Audio Feedback Saved" upon time expiration or early stop

* earlySend --> earlyStop

* userFlag --> userBookmark

* RecordAudioFeedback param/toggle

* add audioFeedback test

* simplify feedbackd

* send bookmark regardless of toggle, show feedback event with higher priority

* add userBookmark to selfdrived sm

* fix mispelled param name

* default off and move to main

* segmentNum --> blockNum, earlyStop --> lastBlock

* preserve audioFeedback

* get rid of lastBlock and just send bookmark saved at the end

* update raylib side

* update toggle description and add raylib toggle

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

selfdrive/ui/feedback/feedbackd.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+import cereal.messaging as messaging
+from openpilot.common.params import Params
+from openpilot.common.swaglog import cloudlog
+from cereal import car
+from openpilot.system.micd import SAMPLE_RATE, SAMPLE_BUFFER
+
+FEEDBACK_MAX_DURATION = 10.0
+ButtonType = car.CarState.ButtonEvent.Type
+
+
+def main():
+  params = Params()
+  pm = messaging.PubMaster(['userBookmark', 'audioFeedback'])
+  sm = messaging.SubMaster(['rawAudioData', 'bookmarkButton', 'carState'])
+  should_record_audio = False
+  block_num = 0
+  waiting_for_release = False
+  early_stop_triggered = False
+
+  while True:
+    sm.update()
+    should_send_bookmark = False
+
+    if sm.updated['carState'] and sm['carState'].canValid:
+      for be in sm['carState'].buttonEvents:
+        if be.type == ButtonType.lkas:
+          if be.pressed:
+            if not should_record_audio:
+              if params.get_bool("RecordAudioFeedback"):  # Start recording on first press if toggle set
+                should_record_audio = True
+                block_num = 0
+                waiting_for_release = False
+                early_stop_triggered = False
+                cloudlog.info("LKAS button pressed - starting 10-second audio feedback")
+              else:
+                should_send_bookmark = True  # immediately send bookmark if toggle false
+                cloudlog.info("LKAS button pressed - bookmarking")
+            elif should_record_audio and not waiting_for_release:  # Wait for release of second press to stop recording early
+              waiting_for_release = True
+          elif waiting_for_release:  # Second press released
+            waiting_for_release = False
+            early_stop_triggered = True
+            cloudlog.info("LKAS button released - ending recording early")
+
+    if should_record_audio and sm.updated['rawAudioData']:
+      raw_audio = sm['rawAudioData']
+      msg = messaging.new_message('audioFeedback', valid=True)
+      msg.audioFeedback.audio.data = raw_audio.data
+      msg.audioFeedback.audio.sampleRate = raw_audio.sampleRate
+      msg.audioFeedback.blockNum = block_num
+      block_num += 1
+      if (block_num * SAMPLE_BUFFER / SAMPLE_RATE) >= FEEDBACK_MAX_DURATION or early_stop_triggered:  # Check for timeout or early stop
+        should_send_bookmark = True  # send bookmark at end of audio segment
+        should_record_audio = False
+        early_stop_triggered = False
+        cloudlog.info("10-second recording completed or second button press - stopping audio feedback")
+      pm.send('audioFeedback', msg)
+
+    if sm.updated['bookmarkButton']:
+      cloudlog.info("Bookmark button pressed!")
+      should_send_bookmark = True
+
+    if should_send_bookmark:
+      msg = messaging.new_message('userBookmark', valid=True)
+      pm.send('userBookmark', msg)
+
+
+if __name__ == '__main__':
+  main()

selfdrive/ui/layouts/main.py

@@ -19,7 +19,7 @@ class MainLayout(Widget):
   def __init__(self):
     super().__init__()
 
-    self._pm = messaging.PubMaster(['userFlag'])
+    self._pm = messaging.PubMaster(['bookmarkButton'])
 
     self._sidebar = Sidebar()
     self._current_mode = MainState.HOME
@@ -40,7 +40,7 @@ class MainLayout(Widget):
 
   def _setup_callbacks(self):
     self._sidebar.set_callbacks(on_settings=self._on_settings_clicked,
-                                on_flag=self._on_flag_clicked)
+                                on_flag=self._on_bookmark_clicked)
     self._layouts[MainState.HOME]._setup_widget.set_open_settings_callback(lambda: self.open_settings(PanelType.FIREHOSE))
     self._layouts[MainState.SETTINGS].set_callbacks(on_close=self._set_mode_for_state)
     self._layouts[MainState.ONROAD].set_callbacks(on_click=self._on_onroad_clicked)
@@ -76,10 +76,10 @@ class MainLayout(Widget):
   def _on_settings_clicked(self):
     self.open_settings(PanelType.DEVICE)
 
-  def _on_flag_clicked(self):
-    user_flag = messaging.new_message('userFlag')
-    user_flag.valid = True
-    self._pm.send('userFlag', user_flag)
+  def _on_bookmark_clicked(self):
+    user_bookmark = messaging.new_message('bookmarkButton')
+    user_bookmark.valid = True
+    self._pm.send('bookmarkButton', user_bookmark)
 
   def _on_onroad_clicked(self):
     self._sidebar.set_visible(not self._sidebar.is_visible)

selfdrive/ui/layouts/settings/toggles.py

@@ -23,6 +23,10 @@ DESCRIPTIONS = {
   'RecordFront': "Upload data from the driver facing camera and help improve the driver monitoring algorithm.",
   "IsMetric": "Display speed in km/h instead of mph.",
   "RecordAudio": "Record and store microphone audio while driving. The audio will be included in the dashcam video in comma connect.",
+  "RecordAudioFeedback": (
+    "Press the LKAS button to record audio feedback about openpilot. When this toggle is disabled, the button acts as a bookmark button. " +
+    "The event will be highlighted in comma connect and the segment will be preserved on your device's storage."
+  ),
 }
 
 
@@ -81,6 +85,12 @@ class TogglesLayout(Widget):
         self._params.get_bool("RecordAudio"),
         icon="microphone.png",
       ),
+      toggle_item(
+        "Record Audio Feedback with LKAS button",
+        DESCRIPTIONS["RecordAudioFeedback"],
+        self._params.get_bool("RecordAudioFeedback"),
+        icon="microphone.png",
+      ),
       toggle_item(
         "Use Metric System", DESCRIPTIONS["IsMetric"], self._params.get_bool("IsMetric"), icon="metric.png"
       ),

selfdrive/ui/qt/offroad/settings.cc

@@ -68,6 +68,13 @@ TogglesPanel::TogglesPanel(SettingsWindow *parent) : ListWidget(parent) {
       "../assets/icons/microphone.png",
       true,
     },
+    {
+      "RecordAudioFeedback",
+      tr("Record Audio Feedback with LKAS button"),
+      tr("Press the LKAS button to record audio feedback about openpilot. When this toggle is disabled, the button acts as a bookmark button. The event will be highlighted in comma connect and the segment will be preserved on your device's storage."),
+      "../assets/icons/microphone.png",
+      false,
+    },
     {
       "IsMetric",
       tr("Use Metric System"),

selfdrive/ui/qt/sidebar.cc

@@ -38,7 +38,7 @@ Sidebar::Sidebar(QWidget *parent) : QFrame(parent), onroad(false), flag_pressed(
 
   QObject::connect(uiState(), &UIState::uiUpdate, this, &Sidebar::updateState);
 
-  pm = std::make_unique<PubMaster>(std::vector<const char*>{"userFlag"});
+  pm = std::make_unique<PubMaster>(std::vector<const char*>{"bookmarkButton"});
 }
 
 void Sidebar::mousePressEvent(QMouseEvent *event) {
@@ -61,8 +61,8 @@ void Sidebar::mouseReleaseEvent(QMouseEvent *event) {
   }
   if (onroad && home_btn.contains(event->pos())) {
     MessageBuilder msg;
-    msg.initEvent().initUserFlag();
-    pm->send("userFlag", msg);
+    msg.initEvent().initBookmarkButton();
+    pm->send("bookmarkButton", msg);
   } else if (settings_btn.contains(event->pos())) {
     emit openSettings();
   } else if (recording_audio && mic_indicator_btn.contains(event->pos())) {

selfdrive/ui/tests/test_feedbackd.py

@@ -0,0 +1,52 @@
+import pytest
+import cereal.messaging as messaging
+from cereal import car
+from openpilot.common.params import Params
+from openpilot.system.manager.process_config import managed_processes
+
+
+class TestFeedbackd:
+  def setup_method(self):
+    self.pm = messaging.PubMaster(['carState', 'rawAudioData'])
+    self.sm = messaging.SubMaster(['audioFeedback'])
+
+  def _send_lkas_button(self, pressed: bool):
+    msg = messaging.new_message('carState')
+    msg.carState.canValid = True
+    msg.carState.buttonEvents = [{'type': car.CarState.ButtonEvent.Type.lkas, 'pressed': pressed}]
+    self.pm.send('carState', msg)
+
+  def _send_audio_data(self, count: int = 5):
+    for _ in range(count):
+      audio_msg = messaging.new_message('rawAudioData')
+      audio_msg.rawAudioData.data = bytes(1600)  # 800 samples of int16
+      audio_msg.rawAudioData.sampleRate = 16000
+      self.pm.send('rawAudioData', audio_msg)
+      self.sm.update(timeout=100)
+
+  @pytest.mark.parametrize("record_feedback", [False, True])
+  def test_audio_feedback(self, record_feedback):
+    Params().put_bool("RecordAudioFeedback", record_feedback)
+
+    managed_processes["feedbackd"].start()
+    assert self.pm.wait_for_readers_to_update('carState', timeout=5)
+    assert self.pm.wait_for_readers_to_update('rawAudioData', timeout=5)
+
+    self._send_lkas_button(pressed=True)
+    self._send_audio_data()
+    self._send_lkas_button(pressed=False)
+    self._send_audio_data()
+
+    if record_feedback:
+      assert self.sm.updated['audioFeedback'], "audioFeedback should be published when enabled"
+    else:
+      assert not self.sm.updated['audioFeedback'], "audioFeedback should not be published when disabled"
+
+    self._send_lkas_button(pressed=True)
+    self._send_audio_data()
+    self._send_lkas_button(pressed=False)
+    self._send_audio_data()
+
+    assert not self.sm.updated['audioFeedback'], "audioFeedback should not be published after second press"
+
+    managed_processes["feedbackd"].stop()