ui: multi touch keyboard support (#35912)

* start

* better

* 2

* dumb

system/ui/widgets/__init__.py

@@ -2,7 +2,7 @@ import abc
 import pyray as rl
 from enum import IntEnum
 from collections.abc import Callable
-from openpilot.system.ui.lib.application import gui_app, MousePos
+from openpilot.system.ui.lib.application import gui_app, MousePos, MAX_TOUCH_SLOT
 
 
 class DialogResult(IntEnum):
@@ -15,7 +15,8 @@ class Widget(abc.ABC):
   def __init__(self):
     self._rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0)
     self._parent_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0)
-    self._is_pressed = False
+    self._is_pressed = [False] * MAX_TOUCH_SLOT
+    self._multi_touch = False
     self._is_visible: bool | Callable[[], bool] = True
     self._touch_valid_callback: Callable[[], bool] | None = None
 
@@ -31,6 +32,10 @@ class Widget(abc.ABC):
   def is_visible(self) -> bool:
     return self._is_visible() if callable(self._is_visible) else self._is_visible
 
+  @property
+  def is_pressed(self) -> bool:
+    return any(self._is_pressed)
+
   @property
   def rect(self) -> rl.Rectangle:
     return self._rect
@@ -68,17 +73,19 @@ class Widget(abc.ABC):
 
     # Keep track of whether mouse down started within the widget's rectangle
     for mouse_event in gui_app.mouse_events:
+      if not self._multi_touch and mouse_event.slot != 0:
+        continue
       if mouse_event.left_pressed and self._touch_valid():
         if rl.check_collision_point_rec(mouse_event.pos, self._rect):
-          self._is_pressed = True
+          self._is_pressed[mouse_event.slot] = True
 
       elif not self._touch_valid():
-        self._is_pressed = False
+        self._is_pressed[mouse_event.slot] = False
 
       elif mouse_event.left_released:
-        if self._is_pressed and rl.check_collision_point_rec(mouse_event.pos, self._rect):
+        if self._is_pressed[mouse_event.slot] and rl.check_collision_point_rec(mouse_event.pos, self._rect):
           self._handle_mouse_release(mouse_event.pos)
-        self._is_pressed = False
+        self._is_pressed[mouse_event.slot] = False
 
     return ret
 

system/ui/widgets/button.py

@@ -179,6 +179,7 @@ class Button(Widget):
                text_padding: int = 20,
                enabled: bool = True,
                icon = None,
+               multi_touch: bool = False,
                ):
 
     super().__init__()
@@ -195,6 +196,7 @@ class Button(Widget):
     self._text_padding = text_padding
     self._text_size = measure_text_cached(gui_app.font(self._font_weight), self._text, self._font_size)
     self._icon = icon
+    self._multi_touch = multi_touch
     self.enabled = enabled
 
   def set_text(self, text):
@@ -208,7 +210,7 @@ class Button(Widget):
   def _update_state(self):
     if self.enabled:
       self._text_color = BUTTON_TEXT_COLOR[self._button_style]
-      if self._is_pressed:
+      if self.is_pressed:
         self._background_color = BUTTON_PRESSED_BACKGROUND_COLORS[self._button_style]
       else:
         self._background_color = BUTTON_BACKGROUND_COLORS[self._button_style]

system/ui/widgets/keyboard.py

@@ -98,11 +98,11 @@ class Keyboard(Widget):
           if key in self._key_icons:
             texture = self._key_icons[key]
             self._all_keys[key] = Button("", partial(self._key_callback, key), icon=texture,
-                                        button_style=ButtonStyle.PRIMARY if key == ENTER_KEY else ButtonStyle.KEYBOARD)
+                                        button_style=ButtonStyle.PRIMARY if key == ENTER_KEY else ButtonStyle.KEYBOARD, multi_touch=True)
           else:
-            self._all_keys[key] = Button(key, partial(self._key_callback, key), button_style=ButtonStyle.KEYBOARD, font_size=85)
+            self._all_keys[key] = Button(key, partial(self._key_callback, key), button_style=ButtonStyle.KEYBOARD, font_size=85, multi_touch=True)
     self._all_keys[CAPS_LOCK_KEY] = Button("", partial(self._key_callback, CAPS_LOCK_KEY), icon=self._key_icons[CAPS_LOCK_KEY],
-                                           button_style=ButtonStyle.KEYBOARD)
+                                           button_style=ButtonStyle.KEYBOARD, multi_touch=True)
 
   @property
   def text(self):
@@ -143,7 +143,7 @@ class Keyboard(Widget):
     self._render_input_area(input_box_rect)
 
     # Process backspace key repeat if it's held down
-    if not self._all_keys[BACKSPACE_KEY]._is_pressed:
+    if not self._all_keys[BACKSPACE_KEY].is_pressed:
       self._backspace_pressed = False
 
     if self._backspace_pressed:
@@ -179,7 +179,7 @@ class Keyboard(Widget):
 
         is_enabled = key != ENTER_KEY or len(self._input_box.text) >= self._min_text_size
 
-        if key == BACKSPACE_KEY and self._all_keys[BACKSPACE_KEY]._is_pressed and not self._backspace_pressed:
+        if key == BACKSPACE_KEY and self._all_keys[BACKSPACE_KEY].is_pressed and not self._backspace_pressed:
           self._backspace_pressed = True
           self._backspace_press_time = time.monotonic()
           self._backspace_last_repeat = time.monotonic()

system/ui/widgets/list_view.py

@@ -167,7 +167,7 @@ class MultipleButtonAction(ItemAction):
       # Check button state
       mouse_pos = rl.get_mouse_position()
       is_hovered = rl.check_collision_point_rec(mouse_pos, button_rect)
-      is_pressed = is_hovered and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._is_pressed
+      is_pressed = is_hovered and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self.is_pressed
       is_selected = i == self.selected_button
 
       # Button colors
@@ -188,7 +188,7 @@ class MultipleButtonAction(ItemAction):
       rl.draw_text_ex(self._font, text, rl.Vector2(text_x, text_y), 40, 0, rl.Color(228, 228, 228, 255))
 
       # Handle click
-      if is_hovered and rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._is_pressed:
+      if is_hovered and rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT) and self.is_pressed:
         clicked = i
 
     if clicked >= 0: