From f52fa9b672fe8b22f61669549db66ba89ba41dbe Mon Sep 17 00:00:00 2001 From: batu_hoang <55729155+longvatrong111@users.noreply.github.com> Date: Wed, 21 May 2025 19:03:04 +0800 Subject: [PATCH] Synchronize `dispatch_actions` in WebDriver (#36932) Implement missing synchronization in `dispatch_actions` of `WebDriver`. https://w3c.github.io/webdriver/#dispatching-actions > The user agent event loop has spun enough times to process the DOM events generated by the last invocation of the >[dispatch tick actions](https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions) steps. - Add a way for `ScriptThread` to notify `WebDriver` about the completion of input commands. - Add a `webdriver_id` field for `InputEvent`. `ScriptThread` uses it to distinguish WebDriver events and sends notification. Tests: `./mach test-wpt --product servodriver -r tests\wpt\tests\webdriver\tests\classic\element_click\events.py` pass if `hit_testing` pass. Check [issue](https://github.com/servo/servo/issues/36676#issuecomment-2882917136) cc: @xiaochengh --------- Signed-off-by: batu_hoang Signed-off-by: Martin Robinson Co-authored-by: Martin Robinson --- components/compositing/compositor.rs | 26 +- components/compositing/webview_renderer.rs | 14 +- components/constellation/constellation.rs | 30 ++- components/constellation/tracing.rs | 1 + components/script/script_thread.rs | 38 ++- components/shared/compositing/lib.rs | 12 +- .../constellation/from_script_message.rs | 3 + components/shared/embedder/input_events.rs | 56 ++++ components/shared/embedder/webdriver.rs | 26 +- components/webdriver_server/actions.rs | 254 +++++++++++++----- components/webdriver_server/lib.rs | 111 +++++--- ports/servoshell/desktop/headed_window.rs | 8 +- ports/servoshell/egl/app_state.rs | 32 +-- 13 files changed, 471 insertions(+), 140 deletions(-) diff --git a/components/compositing/compositor.rs b/components/compositing/compositor.rs index 591470b60c9..855f60e57b2 100644 --- a/components/compositing/compositor.rs +++ b/components/compositing/compositor.rs @@ -620,29 +620,37 @@ impl IOCompositor { } }, - CompositorMsg::WebDriverMouseButtonEvent(webview_id, action, button, x, y) => { + CompositorMsg::WebDriverMouseButtonEvent( + webview_id, + action, + button, + x, + y, + message_id, + ) => { let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else { warn!("Handling input event for unknown webview: {webview_id}"); return; }; let dppx = webview_renderer.device_pixels_per_page_pixel(); let point = dppx.transform_point(Point2D::new(x, y)); - webview_renderer.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent { - point, - action, - button, - })); + webview_renderer.dispatch_input_event( + InputEvent::MouseButton(MouseButtonEvent::new(action, button, point)) + .with_webdriver_message_id(Some(message_id)), + ); }, - CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y) => { + CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y, message_id) => { let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else { warn!("Handling input event for unknown webview: {webview_id}"); return; }; let dppx = webview_renderer.device_pixels_per_page_pixel(); let point = dppx.transform_point(Point2D::new(x, y)); - webview_renderer - .dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point })); + webview_renderer.dispatch_input_event( + InputEvent::MouseMove(MouseMoveEvent::new(point)) + .with_webdriver_message_id(Some(message_id)), + ); }, CompositorMsg::WebDriverWheelScrollEvent(webview_id, x, y, delta_x, delta_y) => { diff --git a/components/compositing/webview_renderer.rs b/components/compositing/webview_renderer.rs index 0a6bdf9ae0a..b0e91ccb02e 100644 --- a/components/compositing/webview_renderer.rs +++ b/components/compositing/webview_renderer.rs @@ -687,17 +687,17 @@ impl WebViewRenderer { /// fn simulate_mouse_click(&mut self, point: DevicePoint) { let button = MouseButton::Left; - self.dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point })); - self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent { + self.dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point))); + self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Down, button, - action: MouseButtonAction::Down, point, - })); - self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent { + ))); + self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Up, button, - action: MouseButtonAction::Up, point, - })); + ))); } pub(crate) fn notify_scroll_event( diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 5db37800c42..1816cf05373 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -132,7 +132,7 @@ use embedder_traits::{ FocusSequenceNumber, ImeEvent, InputEvent, JSValue, JavaScriptEvaluationError, JavaScriptEvaluationId, MediaSessionActionType, MediaSessionEvent, MediaSessionPlaybackState, MouseButton, MouseButtonAction, MouseButtonEvent, Theme, ViewportDetails, WebDriverCommandMsg, - WebDriverLoadStatus, + WebDriverCommandResponse, WebDriverLoadStatus, }; use euclid::Size2D; use euclid::default::Size2D as UntypedSize2D; @@ -532,6 +532,8 @@ pub struct InitialConstellationState { struct WebDriverData { load_channel: Option<(PipelineId, IpcSender)>, resize_channel: Option>>, + // Forward responses from the script thread to the webdriver server. + input_command_response_sender: Option>, } impl WebDriverData { @@ -539,6 +541,7 @@ impl WebDriverData { WebDriverData { load_channel: None, resize_channel: None, + input_command_response_sender: None, } } } @@ -1867,6 +1870,18 @@ where ScriptToConstellationMessage::FinishJavaScriptEvaluation(evaluation_id, result) => { self.handle_finish_javascript_evaluation(evaluation_id, result) }, + ScriptToConstellationMessage::WebDriverInputComplete(msg_id) => { + if let Some(ref reply_sender) = self.webdriver.input_command_response_sender { + reply_sender + .send(WebDriverCommandResponse { id: msg_id }) + .unwrap_or_else(|_| { + warn!("Failed to send WebDriverInputComplete {:?}", msg_id); + self.webdriver.input_command_response_sender = None; + }); + } else { + warn!("No WebDriver input_command_response_sender"); + } + }, } } @@ -4836,7 +4851,11 @@ where mouse_button, x, y, + msg_id, + response_sender, ) => { + self.webdriver.input_command_response_sender = Some(response_sender); + self.compositor_proxy .send(CompositorMsg::WebDriverMouseButtonEvent( webview_id, @@ -4844,11 +4863,16 @@ where mouse_button, x, y, + msg_id, )); }, - WebDriverCommandMsg::MouseMoveAction(webview_id, x, y) => { + WebDriverCommandMsg::MouseMoveAction(webview_id, x, y, msg_id, response_sender) => { + self.webdriver.input_command_response_sender = Some(response_sender); + self.compositor_proxy - .send(CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y)); + .send(CompositorMsg::WebDriverMouseMoveEvent( + webview_id, x, y, msg_id, + )); }, WebDriverCommandMsg::WheelScrollAction(webview, x, y, delta_x, delta_y) => { self.compositor_proxy diff --git a/components/constellation/tracing.rs b/components/constellation/tracing.rs index 940cc9614cc..6237665b87f 100644 --- a/components/constellation/tracing.rs +++ b/components/constellation/tracing.rs @@ -177,6 +177,7 @@ mod from_script { Self::TitleChanged(..) => target!("TitleChanged"), Self::IFrameSizes(..) => target!("IFrameSizes"), Self::ReportMemory(..) => target!("ReportMemory"), + Self::WebDriverInputComplete(..) => target!("WebDriverInputComplete"), Self::FinishJavaScriptEvaluation(..) => target!("FinishJavaScriptEvaluation"), } } diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 4815e6feae1..3ee5bfbd662 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -1081,6 +1081,10 @@ impl ScriptThread { for event in document.take_pending_input_events().into_iter() { document.update_active_keyboard_modifiers(event.active_keyboard_modifiers); + // We do this now, because the event is consumed below, but the order doesn't really + // matter as the event will be handled before any new ScriptThread messages are processed. + self.notify_webdriver_input_event_completed(pipeline_id, &event.event); + match event.event { InputEvent::MouseButton(mouse_button_event) => { document.handle_mouse_button_event( @@ -1144,6 +1148,19 @@ impl ScriptThread { ScriptThread::set_user_interacting(false); } + fn notify_webdriver_input_event_completed(&self, pipeline_id: PipelineId, event: &InputEvent) { + let Some(id) = event.webdriver_message_id() else { + return; + }; + + if let Err(error) = self.senders.pipeline_to_constellation_sender.send(( + pipeline_id, + ScriptToConstellationMessage::WebDriverInputComplete(id), + )) { + warn!("ScriptThread failed to send WebDriverInputComplete {id:?}: {error:?}",); + } + } + /// /// /// Attempt to update the rendering and then do a microtask checkpoint if rendering was actually @@ -3420,7 +3437,7 @@ impl ScriptThread { // the pointer, when the user presses down and releases the primary pointer button" // Servo-specific: Trigger if within 10px of the down point - if let InputEvent::MouseButton(mouse_button_event) = event.event { + if let InputEvent::MouseButton(mouse_button_event) = &event.event { if let MouseButton::Left = mouse_button_event.button { match mouse_button_event.action { MouseButtonAction::Up => { @@ -3429,16 +3446,23 @@ impl ScriptThread { let pixel_dist = (pixel_dist.x * pixel_dist.x + pixel_dist.y * pixel_dist.y).sqrt(); if pixel_dist < 10.0 * document.window().device_pixel_ratio().get() { - document.note_pending_input_event(event.clone()); + // Pass webdriver_id to the newly generated click event + document.note_pending_input_event(ConstellationInputEvent { + hit_test_result: event.hit_test_result.clone(), + pressed_mouse_buttons: event.pressed_mouse_buttons, + active_keyboard_modifiers: event.active_keyboard_modifiers, + event: event.event.clone().with_webdriver_message_id(None), + }); document.note_pending_input_event(ConstellationInputEvent { hit_test_result: event.hit_test_result, pressed_mouse_buttons: event.pressed_mouse_buttons, active_keyboard_modifiers: event.active_keyboard_modifiers, - event: InputEvent::MouseButton(MouseButtonEvent { - action: MouseButtonAction::Click, - button: mouse_button_event.button, - point: mouse_button_event.point, - }), + event: InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Click, + mouse_button_event.button, + mouse_button_event.point, + )) + .with_webdriver_message_id(event.event.webdriver_message_id()), }); return; } diff --git a/components/shared/compositing/lib.rs b/components/shared/compositing/lib.rs index 061dfe023df..d88217142cc 100644 --- a/components/shared/compositing/lib.rs +++ b/components/shared/compositing/lib.rs @@ -10,6 +10,7 @@ use base::id::{PipelineId, WebViewId}; use crossbeam_channel::Sender; use embedder_traits::{ AnimationState, EventLoopWaker, MouseButton, MouseButtonAction, TouchEventResult, + WebDriverMessageId, }; use euclid::Rect; use ipc_channel::ipc::IpcSender; @@ -101,9 +102,16 @@ pub enum CompositorMsg { /// The load of a page has completed LoadComplete(WebViewId), /// WebDriver mouse button event - WebDriverMouseButtonEvent(WebViewId, MouseButtonAction, MouseButton, f32, f32), + WebDriverMouseButtonEvent( + WebViewId, + MouseButtonAction, + MouseButton, + f32, + f32, + WebDriverMessageId, + ), /// WebDriver mouse move event - WebDriverMouseMoveEvent(WebViewId, f32, f32), + WebDriverMouseMoveEvent(WebViewId, f32, f32, WebDriverMessageId), // Webdriver wheel scroll event WebDriverWheelScrollEvent(WebViewId, f32, f32, f64, f64), diff --git a/components/shared/constellation/from_script_message.rs b/components/shared/constellation/from_script_message.rs index a5424abe6d1..fa391f93859 100644 --- a/components/shared/constellation/from_script_message.rs +++ b/components/shared/constellation/from_script_message.rs @@ -17,6 +17,7 @@ use devtools_traits::{DevtoolScriptControlMsg, ScriptToDevtoolsControlMsg, Worke use embedder_traits::{ AnimationState, EmbedderMsg, FocusSequenceNumber, JSValue, JavaScriptEvaluationError, JavaScriptEvaluationId, MediaSessionEvent, TouchEventResult, ViewportDetails, + WebDriverMessageId, }; use euclid::default::Size2D as UntypedSize2D; use http::{HeaderMap, Method}; @@ -652,6 +653,8 @@ pub enum ScriptToConstellationMessage { JavaScriptEvaluationId, Result, ), + /// Notify the completion of a webdriver command. + WebDriverInputComplete(WebDriverMessageId), } impl fmt::Debug for ScriptToConstellationMessage { diff --git a/components/shared/embedder/input_events.rs b/components/shared/embedder/input_events.rs index acaa9afb3ff..869c4eee004 100644 --- a/components/shared/embedder/input_events.rs +++ b/components/shared/embedder/input_events.rs @@ -8,6 +8,8 @@ use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; use webrender_api::units::DevicePoint; +use crate::WebDriverMessageId; + /// An input event that is sent from the embedder to Servo. #[derive(Clone, Debug, Deserialize, Serialize)] pub enum InputEvent { @@ -42,6 +44,38 @@ impl InputEvent { InputEvent::Wheel(event) => Some(event.point), } } + + pub fn webdriver_message_id(&self) -> Option { + match self { + InputEvent::EditingAction(..) => None, + InputEvent::Gamepad(..) => None, + InputEvent::Ime(..) => None, + InputEvent::Keyboard(..) => None, + InputEvent::MouseButton(event) => event.webdriver_id, + InputEvent::MouseMove(event) => event.webdriver_id, + InputEvent::Touch(..) => None, + InputEvent::Wheel(..) => None, + } + } + + pub fn with_webdriver_message_id(self, webdriver_id: Option) -> Self { + match self { + InputEvent::EditingAction(..) => {}, + InputEvent::Gamepad(..) => {}, + InputEvent::Ime(..) => {}, + InputEvent::Keyboard(..) => {}, + InputEvent::MouseButton(mut event) => { + event.webdriver_id = webdriver_id; + }, + InputEvent::MouseMove(mut event) => { + event.webdriver_id = webdriver_id; + }, + InputEvent::Touch(..) => {}, + InputEvent::Wheel(..) => {}, + }; + + self + } } #[derive(Clone, Copy, Debug, Deserialize, Serialize)] @@ -49,6 +83,18 @@ pub struct MouseButtonEvent { pub action: MouseButtonAction, pub button: MouseButton, pub point: DevicePoint, + webdriver_id: Option, +} + +impl MouseButtonEvent { + pub fn new(action: MouseButtonAction, button: MouseButton, point: DevicePoint) -> Self { + Self { + action, + button, + point, + webdriver_id: None, + } + } } #[derive(Clone, Copy, Debug, Deserialize, Serialize)] @@ -102,6 +148,16 @@ pub enum MouseButtonAction { #[derive(Clone, Copy, Debug, Deserialize, Serialize)] pub struct MouseMoveEvent { pub point: DevicePoint, + webdriver_id: Option, +} + +impl MouseMoveEvent { + pub fn new(point: DevicePoint) -> Self { + Self { + point, + webdriver_id: None, + } + } } /// The type of input represented by a multi-touch event. diff --git a/components/shared/embedder/webdriver.rs b/components/shared/embedder/webdriver.rs index 3716a29951a..e7118d32737 100644 --- a/components/shared/embedder/webdriver.rs +++ b/components/shared/embedder/webdriver.rs @@ -24,6 +24,9 @@ use webrender_api::units::DeviceIntSize; use crate::{MouseButton, MouseButtonAction}; +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +pub struct WebDriverMessageId(pub usize); + /// Messages to the constellation originating from the WebDriver server. #[derive(Debug, Deserialize, Serialize)] pub enum WebDriverCommandMsg { @@ -41,9 +44,23 @@ pub enum WebDriverCommandMsg { /// Act as if keys were pressed or release in the browsing context with the given ID. KeyboardAction(BrowsingContextId, KeyboardEvent), /// Act as if the mouse was clicked in the browsing context with the given ID. - MouseButtonAction(WebViewId, MouseButtonAction, MouseButton, f32, f32), + MouseButtonAction( + WebViewId, + MouseButtonAction, + MouseButton, + f32, + f32, + WebDriverMessageId, + IpcSender, + ), /// Act as if the mouse was moved in the browsing context with the given ID. - MouseMoveAction(WebViewId, f32, f32), + MouseMoveAction( + WebViewId, + f32, + f32, + WebDriverMessageId, + IpcSender, + ), /// Act as if the mouse wheel is scrolled in the browsing context given the given ID. WheelScrollAction(WebViewId, f32, f32, f64, f64), /// Set the window size. @@ -188,6 +205,11 @@ pub enum WebDriverFrameId { Parent, } +#[derive(Debug, Deserialize, Serialize)] +pub struct WebDriverCommandResponse { + pub id: WebDriverMessageId, +} + #[derive(Debug, Deserialize, Serialize)] pub enum WebDriverLoadStatus { Complete, diff --git a/components/webdriver_server/actions.rs b/components/webdriver_server/actions.rs index 7965120b0fd..cde418f920c 100644 --- a/components/webdriver_server/actions.rs +++ b/components/webdriver_server/actions.rs @@ -98,20 +98,83 @@ fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 { impl Handler { // https://w3c.github.io/webdriver/#dfn-dispatch-actions pub(crate) fn dispatch_actions( - &mut self, + &self, actions_by_tick: &[ActionSequence], ) -> Result<(), ErrorStatus> { + // Step 1. Wait for an action queue token with input state. + let new_token = self.id_generator.next(); + assert!(self.current_action_id.get().is_none()); + self.current_action_id.set(Some(new_token)); + + // Step 2. Let actions result be the result of dispatch actions inner. + let res = self.dispatch_actions_inner(actions_by_tick); + + // Step 3. Dequeue input state's actions queue. + self.current_action_id.set(None); + + // Step 4. Return actions result. + res + } + + // https://w3c.github.io/webdriver/#dfn-dispatch-actions-inner + fn dispatch_actions_inner( + &self, + actions_by_tick: &[ActionSequence], + ) -> Result<(), ErrorStatus> { + // Step 1. For each item tick actions in actions by tick for tick_actions in actions_by_tick.iter() { + // Step 1.2. Let tick duration be the result of + // computing the tick duration with argument tick actions. let tick_duration = compute_tick_duration(tick_actions); + + // Step 1.3. Try to dispatch tick actions self.dispatch_tick_actions(tick_actions, tick_duration)?; + + // Step 1.4. Wait for + // The user agent event loop has spun enough times to process the DOM events + // generated by the last invocation of the dispatch tick actions steps. + // + // To ensure we wait for all events to be processed, only the last event in + // this tick action step holds the message id. + // Whenever a new event is generated, the message id is passed to it. + // + // TO-DO: remove the first match after webdriver_id is implemented in all commands + match tick_actions.actions { + ActionsType::Key { .. } | ActionsType::Wheel { .. } | ActionsType::Null { .. } => { + return Ok(()); + }, + _ => {}, + } + + match self.constellation_receiver.recv() { + Ok(response) => { + let current_waiting_id = self + .current_action_id + .get() + .expect("Current id should be set before dispat_actions_inner is called"); + + if current_waiting_id != response.id { + dbg!("Dispatch actions completed with wrong id in response"); + return Err(ErrorStatus::UnknownError); + } + }, + Err(error) => { + dbg!("Dispatch actions completed with IPC error: {:?}", error); + return Err(ErrorStatus::UnknownError); + }, + }; } + + // Step 2. Return success with data null. + dbg!("Dispatch actions completed successfully"); Ok(()) } - fn dispatch_general_action(&mut self, source_id: &str) { - self.session_mut() + fn dispatch_general_action(&self, source_id: &str) { + self.session() .unwrap() .input_state_table + .borrow_mut() .entry(source_id.to_string()) .or_insert(InputSourceState::Null); // https://w3c.github.io/webdriver/#dfn-dispatch-a-pause-action @@ -120,7 +183,7 @@ impl Handler { // https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions fn dispatch_tick_actions( - &mut self, + &self, tick_actions: &ActionSequence, tick_duration: u64, ) -> Result<(), ErrorStatus> { @@ -138,9 +201,10 @@ impl Handler { self.dispatch_general_action(source_id); }, KeyActionItem::Key(action) => { - self.session_mut() + self.session() .unwrap() .input_state_table + .borrow_mut() .entry(source_id.to_string()) .or_insert(InputSourceState::Key(KeyInputState::new())); match action { @@ -149,7 +213,7 @@ impl Handler { // Step 9. If subtype is "keyDown", append a copy of action // object with the subtype property changed to "keyUp" to // input state's input cancel list. - self.session_mut().unwrap().input_cancel_list.push( + self.session().unwrap().input_cancel_list.borrow_mut().push( ActionSequence { id: source_id.into(), actions: ActionsType::Key { @@ -180,9 +244,10 @@ impl Handler { self.dispatch_general_action(source_id); }, PointerActionItem::Pointer(action) => { - self.session_mut() + self.session() .unwrap() .input_state_table + .borrow_mut() .entry(source_id.to_string()) .or_insert(InputSourceState::Pointer(PointerInputState::new( ¶meters.pointer_type, @@ -195,7 +260,7 @@ impl Handler { // Step 10. If subtype is "pointerDown", append a copy of action // object with the subtype property changed to "pointerUp" to // input state's input cancel list. - self.session_mut().unwrap().input_cancel_list.push( + self.session().unwrap().input_cancel_list.borrow_mut().push( ActionSequence { id: source_id.into(), actions: ActionsType::Pointer { @@ -232,9 +297,10 @@ impl Handler { self.dispatch_general_action(source_id) }, WheelActionItem::Wheel(action) => { - self.session_mut() + self.session() .unwrap() .input_state_table + .borrow_mut() .entry(source_id.to_string()) .or_insert(InputSourceState::Wheel); match action { @@ -252,12 +318,25 @@ impl Handler { } // https://w3c.github.io/webdriver/#dfn-dispatch-a-keydown-action - fn dispatch_keydown_action(&mut self, source_id: &str, action: &KeyDownAction) { - // Step 1 - let raw_key = action.value.chars().next().unwrap(); - let key_input_state = self.get_key_input_state_mut(source_id); + fn dispatch_keydown_action(&self, source_id: &str, action: &KeyDownAction) { + let session = self.session().unwrap(); + + let raw_key = action.value.chars().next().unwrap(); + let mut input_state_table = session.input_state_table.borrow_mut(); + let key_input_state = match input_state_table.get_mut(source_id).unwrap() { + InputSourceState::Key(key_input_state) => key_input_state, + _ => unreachable!(), + }; + + session.input_cancel_list.borrow_mut().push(ActionSequence { + id: source_id.into(), + actions: ActionsType::Key { + actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction { + value: action.value.clone(), + }))], + }, + }); - // Step 2 - 11. Done by `keyboard-types` crate. let keyboard_event = key_input_state.dispatch_keydown(raw_key); // Step 12 @@ -271,12 +350,25 @@ impl Handler { } // https://w3c.github.io/webdriver/#dfn-dispatch-a-keyup-action - fn dispatch_keyup_action(&mut self, source_id: &str, action: &KeyUpAction) { - // Step 1 - let raw_key = action.value.chars().next().unwrap(); - let key_input_state = self.get_key_input_state_mut(source_id); + fn dispatch_keyup_action(&self, source_id: &str, action: &KeyUpAction) { + let session = self.session().unwrap(); + + let raw_key = action.value.chars().next().unwrap(); + let mut input_state_table = session.input_state_table.borrow_mut(); + let key_input_state = match input_state_table.get_mut(source_id).unwrap() { + InputSourceState::Key(key_input_state) => key_input_state, + _ => unreachable!(), + }; + + session.input_cancel_list.borrow_mut().push(ActionSequence { + id: source_id.into(), + actions: ActionsType::Key { + actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction { + value: action.value.clone(), + }))], + }, + }); - // Step 2 - 11. Done by `keyboard-types` crate. if let Some(keyboard_event) = key_input_state.dispatch_keyup(raw_key) { // Step 12 let cmd_msg = WebDriverCommandMsg::KeyboardAction( @@ -289,44 +381,49 @@ impl Handler { } } - fn get_pointer_input_state_mut(&mut self, source_id: &str) -> &mut PointerInputState { - let session = self.session_mut().unwrap(); - let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() { + /// + pub(crate) fn dispatch_pointerdown_action(&self, source_id: &str, action: &PointerDownAction) { + let session = self.session().unwrap(); + + let mut input_state_table = session.input_state_table.borrow_mut(); + let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() { InputSourceState::Pointer(pointer_input_state) => pointer_input_state, _ => unreachable!(), }; - pointer_input_state - } - - fn get_key_input_state_mut(&mut self, source_id: &str) -> &mut KeyInputState { - let session = self.session_mut().unwrap(); - let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() { - InputSourceState::Key(key_input_state) => key_input_state, - _ => unreachable!(), - }; - key_input_state - } - - // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerdown-action - pub(crate) fn dispatch_pointerdown_action( - &mut self, - source_id: &str, - action: &PointerDownAction, - ) { - let webview_id = self.session().unwrap().webview_id; - let pointer_input_state = self.get_pointer_input_state_mut(source_id); if pointer_input_state.pressed.contains(&action.button) { return; } pointer_input_state.pressed.insert(action.button); + session.input_cancel_list.borrow_mut().push(ActionSequence { + id: source_id.into(), + actions: ActionsType::Pointer { + parameters: PointerActionParameters { + pointer_type: match pointer_input_state.subtype { + PointerType::Mouse => PointerType::Mouse, + PointerType::Pen => PointerType::Pen, + PointerType::Touch => PointerType::Touch, + }, + }, + actions: vec![PointerActionItem::Pointer(PointerAction::Up( + PointerUpAction { + button: action.button, + ..Default::default() + }, + ))], + }, + }); + + let msg_id = self.current_action_id.get().unwrap(); let cmd_msg = WebDriverCommandMsg::MouseButtonAction( - webview_id, + session.webview_id, MouseButtonAction::Down, action.button.into(), pointer_input_state.x as f32, pointer_input_state.y as f32, + msg_id, + self.constellation_sender.clone(), ); self.constellation_chan .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg)) @@ -334,21 +431,48 @@ impl Handler { } // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerup-action - pub(crate) fn dispatch_pointerup_action(&mut self, source_id: &str, action: &PointerUpAction) { - let webview_id = self.session().unwrap().webview_id; - let pointer_input_state = self.get_pointer_input_state_mut(source_id); + pub(crate) fn dispatch_pointerup_action(&self, source_id: &str, action: &PointerUpAction) { + let session = self.session().unwrap(); + + let mut input_state_table = session.input_state_table.borrow_mut(); + let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() { + InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + _ => unreachable!(), + }; if !pointer_input_state.pressed.contains(&action.button) { return; } pointer_input_state.pressed.remove(&action.button); + session.input_cancel_list.borrow_mut().push(ActionSequence { + id: source_id.into(), + actions: ActionsType::Pointer { + parameters: PointerActionParameters { + pointer_type: match pointer_input_state.subtype { + PointerType::Mouse => PointerType::Mouse, + PointerType::Pen => PointerType::Pen, + PointerType::Touch => PointerType::Touch, + }, + }, + actions: vec![PointerActionItem::Pointer(PointerAction::Down( + PointerDownAction { + button: action.button, + ..Default::default() + }, + ))], + }, + }); + + let msg_id = self.current_action_id.get().unwrap(); let cmd_msg = WebDriverCommandMsg::MouseButtonAction( - webview_id, + session.webview_id, MouseButtonAction::Up, action.button.into(), pointer_input_state.x as f32, pointer_input_state.y as f32, + msg_id, + self.constellation_sender.clone(), ); self.constellation_chan .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg)) @@ -357,7 +481,7 @@ impl Handler { // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointermove-action pub(crate) fn dispatch_pointermove_action( - &mut self, + &self, source_id: &str, action: &PointerMoveAction, tick_duration: u64, @@ -370,10 +494,10 @@ impl Handler { // Steps 3 - 4 let (start_x, start_y) = match self - .session - .as_ref() + .session() .unwrap() .input_state_table + .borrow_mut() .get(source_id) .unwrap() { @@ -416,7 +540,7 @@ impl Handler { /// #[allow(clippy::too_many_arguments)] fn perform_pointer_move( - &mut self, + &self, source_id: &str, duration: u64, start_x: f64, @@ -425,9 +549,13 @@ impl Handler { target_y: f64, tick_start: Instant, ) { - let webview_id = self.session().unwrap().webview_id; - let constellation_chan = self.constellation_chan.clone(); - let pointer_input_state = self.get_pointer_input_state_mut(source_id); + let session = self.session().unwrap(); + let mut input_state_table = session.input_state_table.borrow_mut(); + let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() { + InputSourceState::Pointer(pointer_input_state) => pointer_input_state, + _ => unreachable!(), + }; + loop { // Step 1 let time_delta = tick_start.elapsed().as_millis(); @@ -459,9 +587,15 @@ impl Handler { // Step 7 if x != current_x || y != current_y { // Step 7.2 - let cmd_msg = WebDriverCommandMsg::MouseMoveAction(webview_id, x as f32, y as f32); - //TODO: Need Synchronization here before updating `pointer_input_state` - constellation_chan + let msg_id = self.current_action_id.get().unwrap(); + let cmd_msg = WebDriverCommandMsg::MouseMoveAction( + session.webview_id, + x as f32, + y as f32, + msg_id, + self.constellation_sender.clone(), + ); + self.constellation_chan .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg)) .unwrap(); // Step 7.3 @@ -481,7 +615,7 @@ impl Handler { /// fn dispatch_scroll_action( - &mut self, + &self, action: &WheelScrollAction, tick_duration: u64, ) -> Result<(), ErrorStatus> { @@ -546,7 +680,7 @@ impl Handler { /// #[allow(clippy::too_many_arguments)] fn perform_scroll( - &mut self, + &self, duration: u64, x: i64, y: i64, @@ -556,7 +690,7 @@ impl Handler { mut curr_delta_y: i64, tick_start: Instant, ) { - let session = self.session_mut().unwrap(); + let session = self.session().unwrap(); // Step 1 let time_delta = tick_start.elapsed().as_millis(); diff --git a/components/webdriver_server/lib.rs b/components/webdriver_server/lib.rs index 09c466cad3c..0e3fa9058d6 100644 --- a/components/webdriver_server/lib.rs +++ b/components/webdriver_server/lib.rs @@ -10,11 +10,12 @@ mod actions; mod capabilities; use std::borrow::ToOwned; +use std::cell::{Cell, RefCell}; use std::collections::{BTreeMap, HashMap}; use std::io::Cursor; use std::net::{SocketAddr, SocketAddrV4}; use std::time::Duration; -use std::{env, fmt, mem, process, thread}; +use std::{env, fmt, process, thread}; use base::id::{BrowsingContextId, WebViewId}; use base64::Engine; @@ -23,8 +24,9 @@ use constellation_traits::{EmbedderToConstellationMessage, TraversalDirection}; use cookie::{CookieBuilder, Expiration}; use crossbeam_channel::{Receiver, Sender, after, select, unbounded}; use embedder_traits::{ - MouseButton, WebDriverCommandMsg, WebDriverCookieError, WebDriverFrameId, WebDriverJSError, - WebDriverJSResult, WebDriverJSValue, WebDriverLoadStatus, WebDriverScriptCommand, + MouseButton, WebDriverCommandMsg, WebDriverCommandResponse, WebDriverCookieError, + WebDriverFrameId, WebDriverJSError, WebDriverJSResult, WebDriverJSValue, WebDriverLoadStatus, + WebDriverMessageId, WebDriverScriptCommand, }; use euclid::{Rect, Size2D}; use http::method::Method; @@ -43,8 +45,8 @@ use servo_url::ServoUrl; use style_traits::CSSPixel; use uuid::Uuid; use webdriver::actions::{ - ActionSequence, PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, - PointerUpAction, + ActionSequence, ActionsType, PointerAction, PointerActionItem, PointerActionParameters, + PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, }; use webdriver::capabilities::CapabilitiesMatching; use webdriver::command::{ @@ -64,6 +66,26 @@ use webdriver::server::{self, Session, SessionTeardownKind, WebDriverHandler}; use crate::actions::{InputSourceState, PointerInputState}; +#[derive(Default)] +pub struct WebDriverMessageIdGenerator { + counter: Cell, +} + +impl WebDriverMessageIdGenerator { + pub fn new() -> Self { + Self { + counter: Cell::new(0), + } + } + + /// Returns a unique ID. + pub fn next(&self) -> WebDriverMessageId { + let id = self.counter.get(); + self.counter.set(id + 1); + WebDriverMessageId(id) + } +} + fn extension_routes() -> Vec<(Method, &'static str, ServoExtensionRoute)> { vec![ ( @@ -145,10 +167,11 @@ pub struct WebDriverSession { unhandled_prompt_behavior: String, - // https://w3c.github.io/webdriver/#dfn-input-state-table - input_state_table: HashMap, - // https://w3c.github.io/webdriver/#dfn-input-cancel-list - input_cancel_list: Vec, + /// + input_state_table: RefCell>, + + /// + input_cancel_list: RefCell>, } impl WebDriverSession { @@ -172,8 +195,8 @@ impl WebDriverSession { strict_file_interactability: false, unhandled_prompt_behavior: "dismiss and notify".to_string(), - input_state_table: HashMap::new(), - input_cancel_list: Vec::new(), + input_state_table: RefCell::new(HashMap::new()), + input_cancel_list: RefCell::new(Vec::new()), } } } @@ -187,8 +210,22 @@ struct Handler { /// for it to send us a load-status. Messages sent on it /// will be forwarded to the load_status_receiver. load_status_sender: IpcSender, + session: Option, + + /// The channel for sending Webdriver messages to the constellation. constellation_chan: Sender, + + /// The IPC sender which we can clone and pass along to the constellation + constellation_sender: IpcSender, + + /// Receiver notification from the constellation when a command is completed + constellation_receiver: IpcReceiver, + + id_generator: WebDriverMessageIdGenerator, + + current_action_id: Cell>, + resize_timeout: u32, } @@ -409,11 +446,18 @@ impl Handler { let (load_status_sender, receiver) = ipc::channel().unwrap(); let (sender, load_status_receiver) = unbounded(); ROUTER.route_ipc_receiver_to_crossbeam_sender(receiver, sender); + + let (constellation_sender, constellation_receiver) = ipc::channel().unwrap(); + Handler { load_status_sender, load_status_receiver, session: None, constellation_chan, + constellation_sender, + constellation_receiver, + id_generator: WebDriverMessageIdGenerator::new(), + current_action_id: Cell::new(None), resize_timeout: 500, } } @@ -1445,18 +1489,13 @@ impl Handler { } fn handle_release_actions(&mut self) -> WebDriverResult { - let input_cancel_list = { - let session = self.session_mut()?; - session.input_cancel_list.reverse(); - mem::take(&mut session.input_cancel_list) - }; - + let input_cancel_list = self.session().unwrap().input_cancel_list.borrow(); if let Err(error) = self.dispatch_actions(&input_cancel_list) { return Err(WebDriverError::new(error, "")); } - let session = self.session_mut()?; - session.input_state_table = HashMap::new(); + let session = self.session()?; + session.input_state_table.borrow_mut().clear(); Ok(WebDriverResponse::Void) } @@ -1614,7 +1653,7 @@ impl Handler { let id = Uuid::new_v4().to_string(); // Step 8.1 - self.session_mut()?.input_state_table.insert( + self.session_mut()?.input_state_table.borrow_mut().insert( id.clone(), InputSourceState::Pointer(PointerInputState::new(&PointerType::Mouse)), ); @@ -1645,19 +1684,31 @@ impl Handler { ..Default::default() }; - // Step 8.16 Dispatch a list of actions with input state, - // actions, session's current browsing context, and actions options. - if let Err(error) = - self.dispatch_pointermove_action(&id, &pointer_move_action, 0) - { - return Err(WebDriverError::new(error, "")); - } + let action_sequence = ActionSequence { + id: id.clone(), + actions: ActionsType::Pointer { + parameters: PointerActionParameters { + pointer_type: PointerType::Mouse, + }, + actions: vec![ + PointerActionItem::Pointer(PointerAction::Move( + pointer_move_action, + )), + PointerActionItem::Pointer(PointerAction::Down( + pointer_down_action, + )), + PointerActionItem::Pointer(PointerAction::Up(pointer_up_action)), + ], + }, + }; - self.dispatch_pointerdown_action(&id, &pointer_down_action); - self.dispatch_pointerup_action(&id, &pointer_up_action); + let _ = self.dispatch_actions(&[action_sequence]); // Step 8.17 Remove an input source with input state and input id. - self.session_mut()?.input_state_table.remove(&id); + self.session_mut()? + .input_state_table + .borrow_mut() + .remove(&id); // Step 13 Ok(WebDriverResponse::Void) diff --git a/ports/servoshell/desktop/headed_window.rs b/ports/servoshell/desktop/headed_window.rs index 8efb16954d8..eac8d72331d 100644 --- a/ports/servoshell/desktop/headed_window.rs +++ b/ports/servoshell/desktop/headed_window.rs @@ -262,11 +262,11 @@ impl Window { ElementState::Released => MouseButtonAction::Up, }; - webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent { + webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new( action, - button: mouse_button, + mouse_button, point, - })); + ))); } /// Handle key events before sending them to Servo. @@ -563,7 +563,7 @@ impl WindowPortsMethods for Window { point.y -= (self.toolbar_height() * self.hidpi_scale_factor()).0; self.webview_relative_mouse_point.set(point); - webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent { point })); + webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point))); }, WindowEvent::MouseWheel { delta, phase, .. } => { let (mut dx, mut dy, mode) = match delta { diff --git a/ports/servoshell/egl/app_state.rs b/ports/servoshell/egl/app_state.rs index 737a2f23b7d..486b01060e8 100644 --- a/ports/servoshell/egl/app_state.rs +++ b/ports/servoshell/egl/app_state.rs @@ -537,31 +537,31 @@ impl RunningAppState { /// Register a mouse movement. pub fn mouse_move(&self, x: f32, y: f32) { self.active_webview() - .notify_input_event(InputEvent::MouseMove(MouseMoveEvent { - point: Point2D::new(x, y), - })); + .notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(Point2D::new( + x, y, + )))); self.perform_updates(); } /// Register a mouse button press. pub fn mouse_down(&self, x: f32, y: f32, button: MouseButton) { self.active_webview() - .notify_input_event(InputEvent::MouseButton(MouseButtonEvent { - action: MouseButtonAction::Down, + .notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Down, button, - point: Point2D::new(x, y), - })); + Point2D::new(x, y), + ))); self.perform_updates(); } /// Register a mouse button release. pub fn mouse_up(&self, x: f32, y: f32, button: MouseButton) { self.active_webview() - .notify_input_event(InputEvent::MouseButton(MouseButtonEvent { - action: MouseButtonAction::Up, + .notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Up, button, - point: Point2D::new(x, y), - })); + Point2D::new(x, y), + ))); self.perform_updates(); } @@ -589,11 +589,11 @@ impl RunningAppState { /// Perform a click. pub fn click(&self, x: f32, y: f32) { self.active_webview() - .notify_input_event(InputEvent::MouseButton(MouseButtonEvent { - action: MouseButtonAction::Click, - button: MouseButton::Left, - point: Point2D::new(x, y), - })); + .notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new( + MouseButtonAction::Click, + MouseButton::Left, + Point2D::new(x, y), + ))); self.perform_updates(); }