From eaf9224799b5b017d3cdc1016ed25795a476d8e7 Mon Sep 17 00:00:00 2001 From: Kenzie Raditya Tirtarahardja Date: Wed, 7 May 2025 16:41:34 +0800 Subject: [PATCH] Implement wheel action in webdriver (#36744) https://w3c.github.io/webdriver/#wheel-actions Test: `tests/wpt/tests/webdriver/tests/classic/perform_actions/wheel.py::{test_null_response_value,test_params_actions_origin_outside_viewport[element],test_params_actions_origin_outside_viewport[viewport]}, tests/wpt/tests/webdriver/tests/classic/perform_actions/perform.py` Fixes: https://github.com/servo/servo/issues/36720 cc: @xiaochengh @longvatrong111 @yezhizhen Signed-off-by: PotatoCP Co-authored-by: PotatoCP --- components/compositing/compositor.rs | 24 +- components/compositing/tracing.rs | 1 + components/compositing/webview_renderer.rs | 16 ++ components/constellation/constellation.rs | 6 + components/shared/compositing/lib.rs | 2 + components/shared/embedder/webdriver.rs | 2 + components/webdriver_server/actions.rs | 237 +++++++++++++++--- .../classic/perform_actions/perform.py.ini | 9 - .../classic/perform_actions/wheel.py.ini | 9 - 9 files changed, 255 insertions(+), 51 deletions(-) delete mode 100644 tests/wpt/meta/webdriver/tests/classic/perform_actions/perform.py.ini diff --git a/components/compositing/compositor.rs b/components/compositing/compositor.rs index 41286a2760a..b1669277ba1 100644 --- a/components/compositing/compositor.rs +++ b/components/compositing/compositor.rs @@ -26,9 +26,9 @@ use crossbeam_channel::{Receiver, Sender}; use dpi::PhysicalSize; use embedder_traits::{ CompositorHitTestResult, Cursor, InputEvent, MouseButtonEvent, MouseMoveEvent, ShutdownState, - TouchEventType, UntrustedNodeAddress, ViewportDetails, + TouchEventType, UntrustedNodeAddress, ViewportDetails, WheelDelta, WheelEvent, WheelMode, }; -use euclid::{Point2D, Rect, Scale, Size2D, Transform3D}; +use euclid::{Point2D, Rect, Scale, Size2D, Transform3D, Vector2D}; use fnv::FnvHashMap; use ipc_channel::ipc::{self, IpcSharedMemory}; use libc::c_void; @@ -646,6 +646,26 @@ impl IOCompositor { .dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point })); }, + CompositorMsg::WebDriverWheelScrollEvent(webview_id, x, y, delta_x, delta_y) => { + let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else { + warn!("Handling input event for unknown webview: {webview_id}"); + return; + }; + let delta = WheelDelta { + x: delta_x, + y: delta_y, + z: 0.0, + mode: WheelMode::DeltaPixel, + }; + let dppx = webview_renderer.device_pixels_per_page_pixel(); + let point = dppx.transform_point(Point2D::new(x, y)); + let scroll_delta = + dppx.transform_vector(Vector2D::new(delta_x as f32, delta_y as f32)); + webview_renderer + .dispatch_input_event(InputEvent::Wheel(WheelEvent { delta, point })); + webview_renderer.on_webdriver_wheel_action(scroll_delta, point); + }, + CompositorMsg::SendInitialTransaction(pipeline) => { let mut txn = Transaction::new(); txn.set_display_list(WebRenderEpoch(0), (pipeline, Default::default())); diff --git a/components/compositing/tracing.rs b/components/compositing/tracing.rs index ae7338106d0..a8bb8b42bb8 100644 --- a/components/compositing/tracing.rs +++ b/components/compositing/tracing.rs @@ -42,6 +42,7 @@ mod from_constellation { Self::LoadComplete(..) => target!("LoadComplete"), Self::WebDriverMouseButtonEvent(..) => target!("WebDriverMouseButtonEvent"), Self::WebDriverMouseMoveEvent(..) => target!("WebDriverMouseMoveEvent"), + Self::WebDriverWheelScrollEvent(..) => target!("WebDriverWheelScrollEvent"), Self::SendInitialTransaction(..) => target!("SendInitialTransaction"), Self::SendScrollNode(..) => target!("SendScrollNode"), Self::SendDisplayList { .. } => target!("SendDisplayList"), diff --git a/components/compositing/webview_renderer.rs b/components/compositing/webview_renderer.rs index 614ef0ff4c3..f76dc68013d 100644 --- a/components/compositing/webview_renderer.rs +++ b/components/compositing/webview_renderer.rs @@ -726,6 +726,22 @@ impl WebViewRenderer { })); } + /// Push scroll pending event when receiving wheel action from webdriver + pub(crate) fn on_webdriver_wheel_action( + &mut self, + scroll_delta: Vector2D, + point: Point2D, + ) { + if self.global.borrow().shutdown_state() != ShutdownState::NotShuttingDown { + return; + } + + let scroll_location = + ScrollLocation::Delta(LayoutVector2D::from_untyped(scroll_delta.to_untyped())); + let cursor = DeviceIntPoint::new(point.x as i32, point.y as i32); + self.on_scroll_window_event(scroll_location, cursor) + } + pub(crate) fn process_pending_scroll_events(&mut self, compositor: &mut IOCompositor) { if self.pending_scroll_zoom_events.is_empty() { return; diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index f3a15d7708d..e493a97d184 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -4783,6 +4783,12 @@ where self.compositor_proxy .send(CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y)); }, + WebDriverCommandMsg::WheelScrollAction(webview, x, y, delta_x, delta_y) => { + self.compositor_proxy + .send(CompositorMsg::WebDriverWheelScrollEvent( + webview, x, y, delta_x, delta_y, + )); + }, WebDriverCommandMsg::TakeScreenshot(webview_id, rect, response_sender) => { self.compositor_proxy.send(CompositorMsg::CreatePng( webview_id, diff --git a/components/shared/compositing/lib.rs b/components/shared/compositing/lib.rs index 2bc2cc74d50..a6701ca2b52 100644 --- a/components/shared/compositing/lib.rs +++ b/components/shared/compositing/lib.rs @@ -104,6 +104,8 @@ pub enum CompositorMsg { WebDriverMouseButtonEvent(WebViewId, MouseButtonAction, MouseButton, f32, f32), /// WebDriver mouse move event WebDriverMouseMoveEvent(WebViewId, f32, f32), + // Webdriver wheel scroll event + WebDriverWheelScrollEvent(WebViewId, f32, f32, f64, f64), /// Inform WebRender of the existence of this pipeline. SendInitialTransaction(WebRenderPipelineId), diff --git a/components/shared/embedder/webdriver.rs b/components/shared/embedder/webdriver.rs index 9577163411e..3716a29951a 100644 --- a/components/shared/embedder/webdriver.rs +++ b/components/shared/embedder/webdriver.rs @@ -44,6 +44,8 @@ pub enum WebDriverCommandMsg { MouseButtonAction(WebViewId, MouseButtonAction, MouseButton, f32, f32), /// Act as if the mouse was moved in the browsing context with the given ID. MouseMoveAction(WebViewId, f32, f32), + /// Act as if the mouse wheel is scrolled in the browsing context given the given ID. + WheelScrollAction(WebViewId, f32, f32, f64, f64), /// Set the window size. SetWindowSize(WebViewId, DeviceIntSize, IpcSender>), /// Take a screenshot of the window. diff --git a/components/webdriver_server/actions.rs b/components/webdriver_server/actions.rs index fbede5b5887..a5df2e6c523 100644 --- a/components/webdriver_server/actions.rs +++ b/components/webdriver_server/actions.rs @@ -13,20 +13,23 @@ use keyboard_types::webdriver::KeyInputState; use webdriver::actions::{ ActionSequence, ActionsType, GeneralAction, KeyAction, KeyActionItem, KeyDownAction, KeyUpAction, NullActionItem, PointerAction, PointerActionItem, PointerActionParameters, - PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, + PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, WheelAction, + WheelActionItem, WheelScrollAction, }; use webdriver::error::ErrorStatus; use crate::Handler; -// Interval between pointerMove increments in ms, based on common vsync +// Interval between wheelScroll and pointerMove increments in ms, based on common vsync static POINTERMOVE_INTERVAL: u64 = 17; +static WHEELSCROLL_INTERVAL: u64 = 17; // https://w3c.github.io/webdriver/#dfn-input-source-state pub(crate) enum InputSourceState { Null, Key(KeyInputState), Pointer(PointerInputState), + Wheel, } // https://w3c.github.io/webdriver/#dfn-pointer-input-source @@ -76,7 +79,15 @@ fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 { } }, ActionsType::Key { actions: _ } => (), - ActionsType::Wheel { .. } => log::error!("not implemented"), + ActionsType::Wheel { actions } => { + for action in actions.iter() { + let action_duration = match action { + WheelActionItem::General(GeneralAction::Pause(action)) => action.duration, + WheelActionItem::Wheel(WheelAction::Scroll(action)) => action.duration, + }; + duration = cmp::max(duration, action_duration.unwrap_or(0)); + } + }, } duration } @@ -176,9 +187,26 @@ impl Handler { } } }, - ActionsType::Wheel { .. } => { - log::error!("not yet implemented"); - return Err(ErrorStatus::UnsupportedOperation); + ActionsType::Wheel { actions } => { + for action in actions.iter() { + match action { + WheelActionItem::General(_action) => { + self.dispatch_general_action(source_id) + }, + WheelActionItem::Wheel(action) => { + self.session_mut() + .unwrap() + .input_state_table + .entry(source_id.to_string()) + .or_insert(InputSourceState::Wheel); + match action { + WheelAction::Scroll(action) => { + self.dispatch_scroll_action(action, tick_duration)? + }, + } + }, + } + } }, } @@ -191,9 +219,8 @@ impl Handler { let raw_key = action.value.chars().next().unwrap(); let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() { - InputSourceState::Null => unreachable!(), InputSourceState::Key(key_input_state) => key_input_state, - InputSourceState::Pointer(_) => unreachable!(), + _ => unreachable!(), }; session.input_cancel_list.push(ActionSequence { @@ -219,9 +246,8 @@ impl Handler { let raw_key = action.value.chars().next().unwrap(); let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() { - InputSourceState::Null => unreachable!(), InputSourceState::Key(key_input_state) => key_input_state, - InputSourceState::Pointer(_) => unreachable!(), + _ => unreachable!(), }; session.input_cancel_list.push(ActionSequence { @@ -251,9 +277,8 @@ impl Handler { let session = self.session.as_mut().unwrap(); let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() { - InputSourceState::Null => unreachable!(), - InputSourceState::Key(_) => unreachable!(), InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + _ => unreachable!(), }; if pointer_input_state.pressed.contains(&action.button) { @@ -298,9 +323,8 @@ impl Handler { let session = self.session.as_mut().unwrap(); let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() { - InputSourceState::Null => unreachable!(), - InputSourceState::Key(_) => unreachable!(), InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + _ => unreachable!(), }; if !pointer_input_state.pressed.contains(&action.button) { @@ -362,14 +386,12 @@ impl Handler { .get(source_id) .unwrap() { - InputSourceState::Null => unreachable!(), - InputSourceState::Key(_) => unreachable!(), InputSourceState::Pointer(pointer_input_state) => { (pointer_input_state.x, pointer_input_state.y) }, + _ => unreachable!(), }; - // Step 5 - 6 let (x, y) = match action.origin { PointerOrigin::Viewport => (x_offset, y_offset), PointerOrigin::Pointer => (start_x + x_offset, start_y + y_offset), @@ -387,18 +409,8 @@ impl Handler { }, }; - let (sender, receiver) = ipc::channel().unwrap(); - let cmd_msg = - WebDriverCommandMsg::GetWindowSize(self.session.as_ref().unwrap().webview_id, sender); - self.constellation_chan - .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg)) - .unwrap(); - - // Steps 7 - 8 - let viewport_size = receiver.recv().unwrap(); - if x < 0 || x as f32 > viewport_size.width || y < 0 || y as f32 > viewport_size.height { - return Err(ErrorStatus::MoveTargetOutOfBounds); - } + // Step 5 - 6 + self.check_viewport_bound(x, y)?; // Step 9 let duration = match action.duration { @@ -432,9 +444,8 @@ impl Handler { ) { let session = self.session.as_mut().unwrap(); let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() { - InputSourceState::Null => unreachable!(), - InputSourceState::Key(_) => unreachable!(), InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + _ => unreachable!(), }; loop { @@ -487,4 +498,168 @@ impl Handler { thread::sleep(Duration::from_millis(POINTERMOVE_INTERVAL)); } } + + /// + fn dispatch_scroll_action( + &mut self, + action: &WheelScrollAction, + tick_duration: u64, + ) -> Result<(), ErrorStatus> { + // Note: We have not implemented `extract an action sequence` which will calls + // `process a wheel action` that validate many of the variable used here. + // Hence, we do all the checking here until those functions is properly + // implemented. + // + + let tick_start = Instant::now(); + + // Step 1 + let Some(x_offset) = action.x else { + return Err(ErrorStatus::InvalidArgument); + }; + + // Step 2 + let Some(y_offset) = action.y else { + return Err(ErrorStatus::InvalidArgument); + }; + + // Step 3 - 4 + // Get coordinates relative to an origin. Origin must be viewport. + let (x, y) = match action.origin { + PointerOrigin::Viewport => (x_offset, y_offset), + _ => return Err(ErrorStatus::InvalidArgument), + }; + + // Step 5 - 6 + self.check_viewport_bound(x, y)?; + + // Step 7 - 8 + let Some(delta_x) = action.deltaX else { + return Err(ErrorStatus::InvalidArgument); + }; + + let Some(delta_y) = action.deltaY else { + return Err(ErrorStatus::InvalidArgument); + }; + + // Step 9 + let duration = match action.duration { + Some(duration) => duration, + None => tick_duration, + }; + + // Step 10 + if duration > 0 { + thread::sleep(Duration::from_millis(WHEELSCROLL_INTERVAL)); + } + + // Step 11 + self.perform_scroll(duration, x, y, delta_x, delta_y, 0, 0, tick_start); + + // Step 12 + Ok(()) + } + + /// + #[allow(clippy::too_many_arguments)] + fn perform_scroll( + &mut self, + duration: u64, + x: i64, + y: i64, + target_delta_x: i64, + target_delta_y: i64, + mut curr_delta_x: i64, + mut curr_delta_y: i64, + tick_start: Instant, + ) { + let session = self.session.as_mut().unwrap(); + + // Step 1 + let time_delta = tick_start.elapsed().as_millis(); + + // Step 2 + let duration_ratio = if duration > 0 { + time_delta as f64 / duration as f64 + } else { + 1.0 + }; + + // Step 3 + let last = 1.0 - duration_ratio < 0.001; + + // Step 4 + let (delta_x, delta_y) = if last { + (target_delta_x - curr_delta_x, target_delta_y - curr_delta_y) + } else { + ( + (duration_ratio * target_delta_x as f64) as i64 - curr_delta_x, + (duration_ratio * target_delta_y as f64) as i64 - curr_delta_y, + ) + }; + + // Step 5 + if delta_x != 0 || delta_y != 0 { + // Perform implementation-specific action dispatch steps + let cmd_msg = WebDriverCommandMsg::WheelScrollAction( + session.webview_id, + x as f32, + y as f32, + delta_x as f64, + delta_y as f64, + ); + self.constellation_chan + .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg)) + .unwrap(); + + curr_delta_x += delta_x; + curr_delta_y += delta_y; + } + + // Step 6 + if last { + return; + } + + // Step 7 + // TODO: The two steps should be done in parallel + // 7.1. Asynchronously wait for an implementation defined amount of time to pass. + thread::sleep(Duration::from_millis(WHEELSCROLL_INTERVAL)); + // 7.2. Perform a scroll with arguments duration, x, y, target delta x, + // target delta y, current delta x, current delta y. + self.perform_scroll( + duration, + x, + y, + target_delta_x, + target_delta_y, + curr_delta_x, + curr_delta_y, + tick_start, + ); + } + + fn check_viewport_bound(&self, x: i64, y: i64) -> Result<(), ErrorStatus> { + let (sender, receiver) = ipc::channel().unwrap(); + let cmd_msg = + WebDriverCommandMsg::GetWindowSize(self.session.as_ref().unwrap().webview_id, sender); + self.constellation_chan + .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg)) + .unwrap(); + + match receiver.recv() { + Ok(viewport_size) => { + if x < 0 || + x as f32 > viewport_size.width || + y < 0 || + y as f32 > viewport_size.height + { + Err(ErrorStatus::MoveTargetOutOfBounds) + } else { + Ok(()) + } + }, + Err(_) => Err(ErrorStatus::UnknownError), + } + } } diff --git a/tests/wpt/meta/webdriver/tests/classic/perform_actions/perform.py.ini b/tests/wpt/meta/webdriver/tests/classic/perform_actions/perform.py.ini deleted file mode 100644 index b4a8841b9ae..00000000000 --- a/tests/wpt/meta/webdriver/tests/classic/perform_actions/perform.py.ini +++ /dev/null @@ -1,9 +0,0 @@ -[perform.py] - [test_input_source_action_sequence_actions_pause_duration_valid[wheel\]] - expected: FAIL - - [test_input_source_action_sequence_actions_pause_duration_missing[wheel\]] - expected: FAIL - - [test_input_source_action_sequence_pointer_parameters_not_processed[wheel\]] - expected: FAIL diff --git a/tests/wpt/meta/webdriver/tests/classic/perform_actions/wheel.py.ini b/tests/wpt/meta/webdriver/tests/classic/perform_actions/wheel.py.ini index 3f6abc70f3e..c8a0364b783 100644 --- a/tests/wpt/meta/webdriver/tests/classic/perform_actions/wheel.py.ini +++ b/tests/wpt/meta/webdriver/tests/classic/perform_actions/wheel.py.ini @@ -1,19 +1,10 @@ [wheel.py] - [test_null_response_value] - expected: FAIL - [test_no_top_browsing_context] expected: FAIL [test_no_browsing_context] expected: FAIL - [test_params_actions_origin_outside_viewport[element\]] - expected: FAIL - - [test_params_actions_origin_outside_viewport[viewport\]] - expected: FAIL - [test_scroll_not_scrollable] expected: FAIL