Implement wheel action in webdriver (#36744)

https://w3c.github.io/webdriver/#wheel-actions

Test:
`tests/wpt/tests/webdriver/tests/classic/perform_actions/wheel.py::{test_null_response_value,test_params_actions_origin_outside_viewport[element],test_params_actions_origin_outside_viewport[viewport]},
tests/wpt/tests/webdriver/tests/classic/perform_actions/perform.py`
Fixes: https://github.com/servo/servo/issues/36720

cc: @xiaochengh @longvatrong111 @yezhizhen

Signed-off-by: PotatoCP <kenzieradityatirtarahardja.18@gmail.com>
Co-authored-by: PotatoCP <kenzieradityatirtarahardja.18@gmail.com>

components/compositing/compositor.rs

@@ -26,9 +26,9 @@ use crossbeam_channel::{Receiver, Sender};
 use dpi::PhysicalSize;
 use embedder_traits::{
     CompositorHitTestResult, Cursor, InputEvent, MouseButtonEvent, MouseMoveEvent, ShutdownState,
-    TouchEventType, UntrustedNodeAddress, ViewportDetails,
+    TouchEventType, UntrustedNodeAddress, ViewportDetails, WheelDelta, WheelEvent, WheelMode,
 };
-use euclid::{Point2D, Rect, Scale, Size2D, Transform3D};
+use euclid::{Point2D, Rect, Scale, Size2D, Transform3D, Vector2D};
 use fnv::FnvHashMap;
 use ipc_channel::ipc::{self, IpcSharedMemory};
 use libc::c_void;
@@ -646,6 +646,26 @@ impl IOCompositor {
                     .dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point }));
             },
 
+            CompositorMsg::WebDriverWheelScrollEvent(webview_id, x, y, delta_x, delta_y) => {
+                let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else {
+                    warn!("Handling input event for unknown webview: {webview_id}");
+                    return;
+                };
+                let delta = WheelDelta {
+                    x: delta_x,
+                    y: delta_y,
+                    z: 0.0,
+                    mode: WheelMode::DeltaPixel,
+                };
+                let dppx = webview_renderer.device_pixels_per_page_pixel();
+                let point = dppx.transform_point(Point2D::new(x, y));
+                let scroll_delta =
+                    dppx.transform_vector(Vector2D::new(delta_x as f32, delta_y as f32));
+                webview_renderer
+                    .dispatch_input_event(InputEvent::Wheel(WheelEvent { delta, point }));
+                webview_renderer.on_webdriver_wheel_action(scroll_delta, point);
+            },
+
             CompositorMsg::SendInitialTransaction(pipeline) => {
                 let mut txn = Transaction::new();
                 txn.set_display_list(WebRenderEpoch(0), (pipeline, Default::default()));

components/compositing/tracing.rs

@@ -42,6 +42,7 @@ mod from_constellation {
                 Self::LoadComplete(..) => target!("LoadComplete"),
                 Self::WebDriverMouseButtonEvent(..) => target!("WebDriverMouseButtonEvent"),
                 Self::WebDriverMouseMoveEvent(..) => target!("WebDriverMouseMoveEvent"),
+                Self::WebDriverWheelScrollEvent(..) => target!("WebDriverWheelScrollEvent"),
                 Self::SendInitialTransaction(..) => target!("SendInitialTransaction"),
                 Self::SendScrollNode(..) => target!("SendScrollNode"),
                 Self::SendDisplayList { .. } => target!("SendDisplayList"),

components/compositing/webview_renderer.rs

@@ -726,6 +726,22 @@ impl WebViewRenderer {
             }));
     }
 
+    /// Push scroll pending event when receiving wheel action from webdriver
+    pub(crate) fn on_webdriver_wheel_action(
+        &mut self,
+        scroll_delta: Vector2D<f32, DevicePixel>,
+        point: Point2D<f32, DevicePixel>,
+    ) {
+        if self.global.borrow().shutdown_state() != ShutdownState::NotShuttingDown {
+            return;
+        }
+
+        let scroll_location =
+            ScrollLocation::Delta(LayoutVector2D::from_untyped(scroll_delta.to_untyped()));
+        let cursor = DeviceIntPoint::new(point.x as i32, point.y as i32);
+        self.on_scroll_window_event(scroll_location, cursor)
+    }
+
     pub(crate) fn process_pending_scroll_events(&mut self, compositor: &mut IOCompositor) {
         if self.pending_scroll_zoom_events.is_empty() {
             return;

components/constellation/constellation.rs

@@ -4783,6 +4783,12 @@ where
                 self.compositor_proxy
                     .send(CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y));
             },
+            WebDriverCommandMsg::WheelScrollAction(webview, x, y, delta_x, delta_y) => {
+                self.compositor_proxy
+                    .send(CompositorMsg::WebDriverWheelScrollEvent(
+                        webview, x, y, delta_x, delta_y,
+                    ));
+            },
             WebDriverCommandMsg::TakeScreenshot(webview_id, rect, response_sender) => {
                 self.compositor_proxy.send(CompositorMsg::CreatePng(
                     webview_id,

components/shared/compositing/lib.rs

@@ -104,6 +104,8 @@ pub enum CompositorMsg {
     WebDriverMouseButtonEvent(WebViewId, MouseButtonAction, MouseButton, f32, f32),
     /// WebDriver mouse move event
     WebDriverMouseMoveEvent(WebViewId, f32, f32),
+    // Webdriver wheel scroll event
+    WebDriverWheelScrollEvent(WebViewId, f32, f32, f64, f64),
 
     /// Inform WebRender of the existence of this pipeline.
     SendInitialTransaction(WebRenderPipelineId),

components/shared/embedder/webdriver.rs

@@ -44,6 +44,8 @@ pub enum WebDriverCommandMsg {
     MouseButtonAction(WebViewId, MouseButtonAction, MouseButton, f32, f32),
     /// Act as if the mouse was moved in the browsing context with the given ID.
     MouseMoveAction(WebViewId, f32, f32),
+    /// Act as if the mouse wheel is scrolled in the browsing context given the given ID.
+    WheelScrollAction(WebViewId, f32, f32, f64, f64),
     /// Set the window size.
     SetWindowSize(WebViewId, DeviceIntSize, IpcSender<Size2D<f32, CSSPixel>>),
     /// Take a screenshot of the window.

components/webdriver_server/actions.rs

@@ -13,20 +13,23 @@ use keyboard_types::webdriver::KeyInputState;
 use webdriver::actions::{
     ActionSequence, ActionsType, GeneralAction, KeyAction, KeyActionItem, KeyDownAction,
     KeyUpAction, NullActionItem, PointerAction, PointerActionItem, PointerActionParameters,
-    PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction,
+    PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, WheelAction,
+    WheelActionItem, WheelScrollAction,
 };
 use webdriver::error::ErrorStatus;
 
 use crate::Handler;
 
-// Interval between pointerMove increments in ms, based on common vsync
+// Interval between wheelScroll and pointerMove increments in ms, based on common vsync
 static POINTERMOVE_INTERVAL: u64 = 17;
+static WHEELSCROLL_INTERVAL: u64 = 17;
 
 // https://w3c.github.io/webdriver/#dfn-input-source-state
 pub(crate) enum InputSourceState {
     Null,
     Key(KeyInputState),
     Pointer(PointerInputState),
+    Wheel,
 }
 
 // https://w3c.github.io/webdriver/#dfn-pointer-input-source
@@ -76,7 +79,15 @@ fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 {
             }
         },
         ActionsType::Key { actions: _ } => (),
-        ActionsType::Wheel { .. } => log::error!("not implemented"),
+        ActionsType::Wheel { actions } => {
+            for action in actions.iter() {
+                let action_duration = match action {
+                    WheelActionItem::General(GeneralAction::Pause(action)) => action.duration,
+                    WheelActionItem::Wheel(WheelAction::Scroll(action)) => action.duration,
+                };
+                duration = cmp::max(duration, action_duration.unwrap_or(0));
+            }
+        },
     }
     duration
 }
@@ -176,9 +187,26 @@ impl Handler {
                     }
                 }
             },
-            ActionsType::Wheel { .. } => {
-                log::error!("not yet implemented");
-                return Err(ErrorStatus::UnsupportedOperation);
+            ActionsType::Wheel { actions } => {
+                for action in actions.iter() {
+                    match action {
+                        WheelActionItem::General(_action) => {
+                            self.dispatch_general_action(source_id)
+                        },
+                        WheelActionItem::Wheel(action) => {
+                            self.session_mut()
+                                .unwrap()
+                                .input_state_table
+                                .entry(source_id.to_string())
+                                .or_insert(InputSourceState::Wheel);
+                            match action {
+                                WheelAction::Scroll(action) => {
+                                    self.dispatch_scroll_action(action, tick_duration)?
+                                },
+                            }
+                        },
+                    }
+                }
             },
         }
 
@@ -191,9 +219,8 @@ impl Handler {
 
         let raw_key = action.value.chars().next().unwrap();
         let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
-            InputSourceState::Null => unreachable!(),
             InputSourceState::Key(key_input_state) => key_input_state,
-            InputSourceState::Pointer(_) => unreachable!(),
+            _ => unreachable!(),
         };
 
         session.input_cancel_list.push(ActionSequence {
@@ -219,9 +246,8 @@ impl Handler {
 
         let raw_key = action.value.chars().next().unwrap();
         let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
-            InputSourceState::Null => unreachable!(),
             InputSourceState::Key(key_input_state) => key_input_state,
-            InputSourceState::Pointer(_) => unreachable!(),
+            _ => unreachable!(),
         };
 
         session.input_cancel_list.push(ActionSequence {
@@ -251,9 +277,8 @@ impl Handler {
         let session = self.session.as_mut().unwrap();
 
         let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
-            InputSourceState::Null => unreachable!(),
-            InputSourceState::Key(_) => unreachable!(),
             InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
+            _ => unreachable!(),
         };
 
         if pointer_input_state.pressed.contains(&action.button) {
@@ -298,9 +323,8 @@ impl Handler {
         let session = self.session.as_mut().unwrap();
 
         let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
-            InputSourceState::Null => unreachable!(),
-            InputSourceState::Key(_) => unreachable!(),
             InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
+            _ => unreachable!(),
         };
 
         if !pointer_input_state.pressed.contains(&action.button) {
@@ -362,14 +386,12 @@ impl Handler {
             .get(source_id)
             .unwrap()
         {
-            InputSourceState::Null => unreachable!(),
-            InputSourceState::Key(_) => unreachable!(),
             InputSourceState::Pointer(pointer_input_state) => {
                 (pointer_input_state.x, pointer_input_state.y)
             },
+            _ => unreachable!(),
         };
 
-        // Step 5 - 6
         let (x, y) = match action.origin {
             PointerOrigin::Viewport => (x_offset, y_offset),
             PointerOrigin::Pointer => (start_x + x_offset, start_y + y_offset),
@@ -387,18 +409,8 @@ impl Handler {
             },
         };
 
-        let (sender, receiver) = ipc::channel().unwrap();
-        let cmd_msg =
-            WebDriverCommandMsg::GetWindowSize(self.session.as_ref().unwrap().webview_id, sender);
-        self.constellation_chan
-            .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
-            .unwrap();
-
-        // Steps 7 - 8
-        let viewport_size = receiver.recv().unwrap();
-        if x < 0 || x as f32 > viewport_size.width || y < 0 || y as f32 > viewport_size.height {
-            return Err(ErrorStatus::MoveTargetOutOfBounds);
-        }
+        // Step 5 - 6
+        self.check_viewport_bound(x, y)?;
 
         // Step 9
         let duration = match action.duration {
@@ -432,9 +444,8 @@ impl Handler {
     ) {
         let session = self.session.as_mut().unwrap();
         let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
-            InputSourceState::Null => unreachable!(),
-            InputSourceState::Key(_) => unreachable!(),
             InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
+            _ => unreachable!(),
         };
 
         loop {
@@ -487,4 +498,168 @@ impl Handler {
             thread::sleep(Duration::from_millis(POINTERMOVE_INTERVAL));
         }
     }
+
+    /// <https://w3c.github.io/webdriver/#dfn-dispatch-a-scroll-action>
+    fn dispatch_scroll_action(
+        &mut self,
+        action: &WheelScrollAction,
+        tick_duration: u64,
+    ) -> Result<(), ErrorStatus> {
+        // Note: We have not implemented `extract an action sequence` which will calls
+        // `process a wheel action` that validate many of the variable used here.
+        // Hence, we do all the checking here until those functions is properly
+        // implemented.
+        // <https://w3c.github.io/webdriver/#dfn-process-a-wheel-action>
+
+        let tick_start = Instant::now();
+
+        // Step 1
+        let Some(x_offset) = action.x else {
+            return Err(ErrorStatus::InvalidArgument);
+        };
+
+        // Step 2
+        let Some(y_offset) = action.y else {
+            return Err(ErrorStatus::InvalidArgument);
+        };
+
+        // Step 3 - 4
+        // Get coordinates relative to an origin. Origin must be viewport.
+        let (x, y) = match action.origin {
+            PointerOrigin::Viewport => (x_offset, y_offset),
+            _ => return Err(ErrorStatus::InvalidArgument),
+        };
+
+        // Step 5 - 6
+        self.check_viewport_bound(x, y)?;
+
+        // Step 7 - 8
+        let Some(delta_x) = action.deltaX else {
+            return Err(ErrorStatus::InvalidArgument);
+        };
+
+        let Some(delta_y) = action.deltaY else {
+            return Err(ErrorStatus::InvalidArgument);
+        };
+
+        // Step 9
+        let duration = match action.duration {
+            Some(duration) => duration,
+            None => tick_duration,
+        };
+
+        // Step 10
+        if duration > 0 {
+            thread::sleep(Duration::from_millis(WHEELSCROLL_INTERVAL));
+        }
+
+        // Step 11
+        self.perform_scroll(duration, x, y, delta_x, delta_y, 0, 0, tick_start);
+
+        // Step 12
+        Ok(())
+    }
+
+    /// <https://w3c.github.io/webdriver/#dfn-perform-a-scroll>
+    #[allow(clippy::too_many_arguments)]
+    fn perform_scroll(
+        &mut self,
+        duration: u64,
+        x: i64,
+        y: i64,
+        target_delta_x: i64,
+        target_delta_y: i64,
+        mut curr_delta_x: i64,
+        mut curr_delta_y: i64,
+        tick_start: Instant,
+    ) {
+        let session = self.session.as_mut().unwrap();
+
+        // Step 1
+        let time_delta = tick_start.elapsed().as_millis();
+
+        // Step 2
+        let duration_ratio = if duration > 0 {
+            time_delta as f64 / duration as f64
+        } else {
+            1.0
+        };
+
+        // Step 3
+        let last = 1.0 - duration_ratio < 0.001;
+
+        // Step 4
+        let (delta_x, delta_y) = if last {
+            (target_delta_x - curr_delta_x, target_delta_y - curr_delta_y)
+        } else {
+            (
+                (duration_ratio * target_delta_x as f64) as i64 - curr_delta_x,
+                (duration_ratio * target_delta_y as f64) as i64 - curr_delta_y,
+            )
+        };
+
+        // Step 5
+        if delta_x != 0 || delta_y != 0 {
+            // Perform implementation-specific action dispatch steps
+            let cmd_msg = WebDriverCommandMsg::WheelScrollAction(
+                session.webview_id,
+                x as f32,
+                y as f32,
+                delta_x as f64,
+                delta_y as f64,
+            );
+            self.constellation_chan
+                .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
+                .unwrap();
+
+            curr_delta_x += delta_x;
+            curr_delta_y += delta_y;
+        }
+
+        // Step 6
+        if last {
+            return;
+        }
+
+        // Step 7
+        // TODO: The two steps should be done in parallel
+        // 7.1. Asynchronously wait for an implementation defined amount of time to pass.
+        thread::sleep(Duration::from_millis(WHEELSCROLL_INTERVAL));
+        // 7.2. Perform a scroll with arguments duration, x, y, target delta x,
+        // target delta y, current delta x, current delta y.
+        self.perform_scroll(
+            duration,
+            x,
+            y,
+            target_delta_x,
+            target_delta_y,
+            curr_delta_x,
+            curr_delta_y,
+            tick_start,
+        );
+    }
+
+    fn check_viewport_bound(&self, x: i64, y: i64) -> Result<(), ErrorStatus> {
+        let (sender, receiver) = ipc::channel().unwrap();
+        let cmd_msg =
+            WebDriverCommandMsg::GetWindowSize(self.session.as_ref().unwrap().webview_id, sender);
+        self.constellation_chan
+            .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
+            .unwrap();
+
+        match receiver.recv() {
+            Ok(viewport_size) => {
+                if x < 0 ||
+                    x as f32 > viewport_size.width ||
+                    y < 0 ||
+                    y as f32 > viewport_size.height
+                {
+                    Err(ErrorStatus::MoveTargetOutOfBounds)
+                } else {
+                    Ok(())
+                }
+            },
+            Err(_) => Err(ErrorStatus::UnknownError),
+        }
+    }
 }