diff --git a/selfdrive/ui/tests/diff/replay_script.py b/selfdrive/ui/tests/diff/replay_script.py index c43442a33d..c53d2f116b 100644 --- a/selfdrive/ui/tests/diff/replay_script.py +++ b/selfdrive/ui/tests/diff/replay_script.py @@ -15,9 +15,16 @@ from openpilot.selfdrive.ui.tests.diff.replay import FPS, LayoutVariant from openpilot.system.updated.updated import parse_release_notes # Default frames to wait after events -WAIT = FPS // 2 +WAIT_LONG = FPS +WAIT_SHORT = FPS // 2 FAST_CLICK = FPS // 6 +# Direction vectors for drag gestures +DIR_LEFT = (-1, 0) +DIR_RIGHT = (1, 0) +DIR_UP = (0, -1) +DIR_DOWN = (0, 1) + AlertSize = log.SelfdriveState.AlertSize AlertStatus = log.SelfdriveState.AlertStatus @@ -61,26 +68,47 @@ class Script: """Add a delay for the given number of frames followed by an empty event.""" self.add(ScriptEvent(), before=frames) - def setup(self, fn: Callable, wait_after: int = WAIT) -> None: + def setup(self, fn: Callable, wait_after: int = WAIT_SHORT) -> None: """Add a setup function to be called immediately followed by a delay of the given number of frames.""" self.add(ScriptEvent(setup=fn), after=wait_after) - def set_send(self, fn: Callable, wait_after: int = WAIT) -> None: + def set_send(self, fn: Callable, wait_after: int = WAIT_SHORT) -> None: """Set a new persistent send function to be called every frame.""" self.add(ScriptEvent(send_fn=fn), after=wait_after) - # TODO: Also add more complex gestures, like swipe or drag - def click(self, x: int, y: int, wait_after: int = WAIT, wait_between: int = 2) -> None: + def click(self, x: int, y: int, wait_after: int = WAIT_SHORT, wait_between: int = 2) -> None: """Add a click event to the script for the given position and specify frames to wait between mouse events or after the click.""" # NOTE: By default we wait a couple frames between mouse events so pressed states will be rendered from openpilot.system.ui.lib.application import MouseEvent, MousePos - # TODO: Add support for long press (left_down=True) mouse_down = MouseEvent(pos=MousePos(x, y), slot=0, left_pressed=True, left_released=False, left_down=False, t=self.get_frame_time()) self.add(ScriptEvent(mouse_events=[mouse_down]), after=wait_between) mouse_up = MouseEvent(pos=MousePos(x, y), slot=0, left_pressed=False, left_released=True, left_down=False, t=self.get_frame_time()) self.add(ScriptEvent(mouse_events=[mouse_up]), after=wait_after) + def drag(self, start_x: int, start_y: int, direction: tuple[int, int], distance: int, duration_frames: int, wait_after: int = WAIT_LONG) -> None: + """Add a drag gesture to the script from start position in the specified direction by the given distance over the given number of frames.""" + from openpilot.system.ui.lib.application import MouseEvent, MousePos + + # Calculate delta and end position based on direction and distance + delta_x, delta_y = direction[0] * distance, direction[1] * distance + end_x, end_y = start_x + delta_x, start_y + delta_y + + # Mouse down at start + mouse_down = MouseEvent(pos=MousePos(start_x, start_y), slot=0, left_pressed=True, left_released=False, left_down=True, t=self.get_frame_time()) + self.add(ScriptEvent(mouse_events=[mouse_down]), after=1) + + # Interpolate positions over duration_frames + for i in range(1, duration_frames): + t = i / duration_frames + x, y = int(start_x + delta_x * t), int(start_y + delta_y * t) + mouse_move = MouseEvent(pos=MousePos(x, y), slot=0, left_pressed=False, left_released=False, left_down=True, t=self.get_frame_time()) + self.add(ScriptEvent(mouse_events=[mouse_move]), after=1) + + # Mouse up at end + mouse_up = MouseEvent(pos=MousePos(end_x, end_y), slot=0, left_pressed=False, left_released=True, left_down=False, t=self.get_frame_time()) + self.add(ScriptEvent(mouse_events=[mouse_up]), after=wait_after) + # --- Setup functions --- @@ -169,18 +197,181 @@ def make_alert_setup(pm: PubMaster, size, text1, text2, status) -> Callable: return _send +def test_onroad_alerts(script: Script, pm: PubMaster) -> None: + """Go through various alert types and sizes and add them to the script to test alert rendering. + Each alert is sent as a separate event with a delay in between.""" + # Small alert (normal) + script.set_send(make_alert_setup(pm, AlertSize.small, "Small Alert", "This is a small alert", AlertStatus.normal)) + # Medium alert (userPrompt) + script.set_send(make_alert_setup(pm, AlertSize.mid, "Medium Alert", "This is a medium alert", AlertStatus.userPrompt)) + # Full alert (critical) + script.set_send(make_alert_setup(pm, AlertSize.full, "DISENGAGE IMMEDIATELY", "Driver Distracted", AlertStatus.critical)) + # Full alert multiline + script.set_send(make_alert_setup(pm, AlertSize.full, "Reverse\nGear", "", AlertStatus.normal)) + # Full alert long text + script.set_send(make_alert_setup(pm, AlertSize.full, "TAKE CONTROL IMMEDIATELY", "Calibration Invalid: Remount Device & Recalibrate", AlertStatus.userPrompt)) + + # --- Script builders --- def build_mici_script(pm: PubMaster, main_layout, script: Script) -> None: """Build the replay script for the mici layout.""" from openpilot.system.ui.lib.application import gui_app - center = (gui_app.width // 2, gui_app.height // 2) + width, height = gui_app.width, gui_app.height + center = (width // 2, height // 2) + right = (width * 4 // 5, height // 2) + left = (width // 5, height // 2) + top = (width // 2, height // 10) + bottom = (width // 2, height * 9 // 10) + + DURATION = 5 + SWIPE_WAIT = FPS * 3 // 4 + + def click(times: int = 1, wait_after: int = WAIT_SHORT) -> None: + """Click at the center of the screen the given number of times with optional delay after.""" + for _ in range(times): + script.click(*center, wait_after=wait_after) + + def press(x: int, y: int, duration_frames: int = DURATION, wait_after: int = WAIT_SHORT) -> None: + """Perform a drag with no movement to simulate a left_down mouse event at the given position for the specified duration and delay after.""" + script.drag(x, y, (0, 0), 0, duration_frames, wait_after=wait_after) + + def swipe_left(distance: int = right[0] - left[0], duration_frames: int = DURATION, wait_after: int = SWIPE_WAIT) -> None: + """Drag from right edge to left (scroll right / slide confirmation).""" + script.drag(*right, DIR_LEFT, distance, duration_frames, wait_after) + + def swipe_right(distance: int = right[0] - left[0], duration_frames: int = DURATION, wait_after: int = SWIPE_WAIT) -> None: + """Drag from left edge to right (scroll left).""" + script.drag(*left, DIR_RIGHT, distance, duration_frames, wait_after) + + def swipe_down(distance: int = bottom[1] - top[1], duration_frames: int = DURATION, wait_after: int = SWIPE_WAIT) -> None: + """Drag from top edge to bottom (scroll up / go back).""" + script.drag(*top, DIR_DOWN, distance, duration_frames, wait_after) + + def swipe_up(distance: int = bottom[1] - top[1], duration_frames: int = DURATION, wait_after: int = SWIPE_WAIT) -> None: + """Drag from bottom edge to top (scroll down).""" + script.drag(*bottom, DIR_UP, distance, duration_frames, wait_after) + + ActionFn = Callable[[], None] | None + Cases = list[ActionFn] + + def run_actions(*actions: ActionFn, after_each: ActionFn = None) -> None: + """Helper function to run a sequence of actions in order for interaction tests, calling after_each callback after each action if provided.""" + for action in actions: + if action is not None: + action() + if after_each is not None: + after_each() + + def explore_setting(*actions: ActionFn) -> None: + """Helper function to open a settings item, run the given actions, and go back.""" + run_actions(click, *actions, swipe_down) # open, interact, go back + + def scroll_through_cases(cases: Cases) -> None: + """Helper function to explore a panel by calling the interaction callbacks for each item/page before swiping to the next one.""" + run_actions(*cases, after_each=lambda: swipe_left(210, 10)) # swipe to roughly the center of the next toggle after each case + + def interact_keyboard() -> None: + """Interact with the keyboard in various ways to test different actions and states. + Assumes it's a password keyboard with 8 characters required. Closes by pressing confirm at the end.""" + KEY = (250, 160) # key in the middle of the keyboard ('G') + SHIFT = (50, 210) + NUMBERS = (480, 210) + SPACE = (500, 160) + BACKSPACE = (490, 30) + CONFIRM = (50, 30) + # Begin interactions + press(*CONFIRM, wait_after=FAST_CLICK) # confirm while disabled should do nothing + swipe_left(duration_frames=FPS // 2) # swipe to type + swipe_up(duration_frames=FPS // 2) # swipe out of keyboard (nothing typed) + # press various keys to test different states: + for key in [ + SHIFT, KEY, KEY, SHIFT, SHIFT, KEY, KEY, # test casing (upper, lower, caps lock) + SPACE, SPACE, BACKSPACE, BACKSPACE, # test multiple space and backspace + NUMBERS, KEY, center, SHIFT, KEY # test numbers and symbols + ]: + press(*key, wait_after=FAST_CLICK) + # press confirm to close + script.wait(WAIT_SHORT) # wait for confirm to enable + press(*CONFIRM) + + toggle_cases: Cases = [ + lambda: click(times=3, wait_after=FAST_CLICK), # first toggle is personality, which has 3 states + None, None, None, None, None, None, # skip other toggles to save time + lambda: click(times=2, wait_after=FAST_CLICK), # test final toggle (enable openpilot) + ] + + network_cases: Cases = [ + explore_setting, # select wifi (just open and close) + None, None, + lambda: run_actions(click, interact_keyboard), # tether password keyboard + ] + + device_cases: Cases = [ + None, + click, # update + explore_setting, # pairing (just open and close) + lambda: explore_setting( + # training guide + lambda: swipe_left(width * 2), click, # first page, click next + lambda: swipe_left(width * 2), swipe_down # second page, go back (TODO: make driver cam preview work) + ), + None, # TODO: preview driver camera; enabling this causes MultiplePublishersError later in onroad alert tests + lambda: explore_setting(swipe_left), # terms & conditions (swipe to view QR code) + lambda: explore_setting(lambda: swipe_up(height * 3), lambda: swipe_down(height * 3)), # regulatory info + lambda: run_actions(click, lambda: swipe_left(width)), # reset calibration confirm (goes back automatically) + lambda: explore_setting(lambda: swipe_left(width)), # uninstall + lambda: run_actions( + lambda: explore_setting(lambda: swipe_left(width)), # reboot + lambda: script.click(430, 120), lambda: swipe_left(width), swipe_down, # shutdown + ), + ] + + developer_cases: Cases = [ + lambda: click(times=2, wait_after=FAST_CLICK), # toggle ssh mode + explore_setting, # SSH keys keyboard (just open and close) + None, # joystick mode + lambda: click(wait_after=FAST_CLICK), # longitudinal maneuver mode (disabled; should do nothing) + lambda: click(times=2, wait_after=FAST_CLICK), # toggle UI debug mode + ] + + settings_cases: Cases = [ + lambda: scroll_through_cases(toggle_cases), + lambda: scroll_through_cases(network_cases), + lambda: scroll_through_cases(device_cases), + lambda: script.wait(WAIT_SHORT), # pairing + lambda: run_actions(lambda: swipe_up(height * 3), lambda: swipe_down(height * 3)), # firehose (scroll down and back up) + lambda: scroll_through_cases(developer_cases), + ] + + # === Homescreen === # + script.wait(WAIT_SHORT) + swipe_left(width, wait_after=WAIT_SHORT) # onroad screen + swipe_right(width, wait_after=WAIT_SHORT) # back to home + + # === Offroad Alerts === + def setup_offroad_alerts_and_refresh() -> None: + """Setup function to trigger offroad alerts and force a refresh on the alerts layout.""" + setup_offroad_alerts() + main_layout._alerts_layout.refresh() + + swipe_right(width, wait_after=WAIT_SHORT) # open alerts + script.setup(setup_offroad_alerts_and_refresh) # show alerts + swipe_up(height) # scroll alerts + swipe_left(width, wait_after=WAIT_SHORT) # close alerts + + # === Settings === # + click() # open settings + scroll_through_cases([lambda case=case: explore_setting(case) for case in settings_cases]) # explore settings + swipe_down() # back to home + + # === Onroad === + script.set_send(lambda: send_onroad(pm)) + swipe_left(width, wait_after=WAIT_SHORT) # onroad screen + test_onroad_alerts(script, pm) + swipe_right(width) # back to home - # TODO: Explore more - script.wait(FPS) - script.click(*center, FPS) # Open settings - script.click(*center, FPS) # Open toggles script.end() @@ -313,18 +504,7 @@ def build_tizi_script(pm: PubMaster, main_layout, script: Script) -> None: # === Onroad === script.set_send(lambda: send_onroad(pm)) script.click(1000, 500) # click onroad to toggle sidebar - - # === Onroad alerts === - # Small alert (normal) - script.set_send(make_alert_setup(pm, AlertSize.small, "Small Alert", "This is a small alert", AlertStatus.normal)) - # Medium alert (userPrompt) - script.set_send(make_alert_setup(pm, AlertSize.mid, "Medium Alert", "This is a medium alert", AlertStatus.userPrompt)) - # Full alert (critical) - script.set_send(make_alert_setup(pm, AlertSize.full, "DISENGAGE IMMEDIATELY", "Driver Distracted", AlertStatus.critical)) - # Full alert multiline - script.set_send(make_alert_setup(pm, AlertSize.full, "Reverse\nGear", "", AlertStatus.normal)) - # Full alert long text - script.set_send(make_alert_setup(pm, AlertSize.full, "TAKE CONTROL IMMEDIATELY", "Calibration Invalid: Remount Device & Recalibrate", AlertStatus.userPrompt)) + test_onroad_alerts(script, pm) # End script.end()