Synchronize dispatch_actions in WebDriver (#36932)

Implement missing synchronization in `dispatch_actions` of `WebDriver`.
https://w3c.github.io/webdriver/#dispatching-actions

> The user agent event loop has spun enough times to process the DOM
events generated by the last invocation of the >[dispatch tick
actions](https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions)
steps.

- Add a way for `ScriptThread` to notify `WebDriver` about the
completion of input commands.
- Add a `webdriver_id` field for `InputEvent`. `ScriptThread` uses it to
distinguish WebDriver events and sends notification.

Tests:
`./mach test-wpt --product servodriver -r
tests\wpt\tests\webdriver\tests\classic\element_click\events.py` pass if
`hit_testing` pass. Check
[issue](https://github.com/servo/servo/issues/36676#issuecomment-2882917136)

cc: @xiaochengh

---------

Signed-off-by: batu_hoang <longvatrong111@gmail.com>
Signed-off-by: Martin Robinson <mrobinson@igalia.com>
Co-authored-by: Martin Robinson <mrobinson@igalia.com>

components/compositing/compositor.rs

@@ -620,29 +620,37 @@ impl IOCompositor {
                 }
             },
 
-            CompositorMsg::WebDriverMouseButtonEvent(webview_id, action, button, x, y) => {
+            CompositorMsg::WebDriverMouseButtonEvent(
+                webview_id,
+                action,
+                button,
+                x,
+                y,
+                message_id,
+            ) => {
                 let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else {
                     warn!("Handling input event for unknown webview: {webview_id}");
                     return;
                 };
                 let dppx = webview_renderer.device_pixels_per_page_pixel();
                 let point = dppx.transform_point(Point2D::new(x, y));
-                webview_renderer.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent {
+                webview_renderer.dispatch_input_event(
-                    point,
+                    InputEvent::MouseButton(MouseButtonEvent::new(action, button, point))
-                    action,
+                        .with_webdriver_message_id(Some(message_id)),
-                    button,
+                );
-                }));
             },
 
-            CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y) => {
+            CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y, message_id) => {
                 let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else {
                     warn!("Handling input event for unknown webview: {webview_id}");
                     return;
                 };
                 let dppx = webview_renderer.device_pixels_per_page_pixel();
                 let point = dppx.transform_point(Point2D::new(x, y));
-                webview_renderer
+                webview_renderer.dispatch_input_event(
-                    .dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point }));
+                    InputEvent::MouseMove(MouseMoveEvent::new(point))
+                        .with_webdriver_message_id(Some(message_id)),
+                );
             },
 
             CompositorMsg::WebDriverWheelScrollEvent(webview_id, x, y, delta_x, delta_y) => {

components/compositing/webview_renderer.rs

@@ -687,17 +687,17 @@ impl WebViewRenderer {
     /// <http://w3c.github.io/touch-events/#mouse-events>
     fn simulate_mouse_click(&mut self, point: DevicePoint) {
         let button = MouseButton::Left;
-        self.dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point }));
+        self.dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));
-        self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent {
+        self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
+            MouseButtonAction::Down,
             button,
-            action: MouseButtonAction::Down,
             point,
-        }));
+        )));
-        self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent {
+        self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
+            MouseButtonAction::Up,
             button,
-            action: MouseButtonAction::Up,
             point,
-        }));
+        )));
     }
 
     pub(crate) fn notify_scroll_event(

components/constellation/constellation.rs

@@ -132,7 +132,7 @@ use embedder_traits::{
     FocusSequenceNumber, ImeEvent, InputEvent, JSValue, JavaScriptEvaluationError,
     JavaScriptEvaluationId, MediaSessionActionType, MediaSessionEvent, MediaSessionPlaybackState,
     MouseButton, MouseButtonAction, MouseButtonEvent, Theme, ViewportDetails, WebDriverCommandMsg,
-    WebDriverLoadStatus,
+    WebDriverCommandResponse, WebDriverLoadStatus,
 };
 use euclid::Size2D;
 use euclid::default::Size2D as UntypedSize2D;
@@ -532,6 +532,8 @@ pub struct InitialConstellationState {
 struct WebDriverData {
     load_channel: Option<(PipelineId, IpcSender<WebDriverLoadStatus>)>,
     resize_channel: Option<IpcSender<Size2D<f32, CSSPixel>>>,
+    // Forward responses from the script thread to the webdriver server.
+    input_command_response_sender: Option<IpcSender<WebDriverCommandResponse>>,
 }
 
 impl WebDriverData {
@@ -539,6 +541,7 @@ impl WebDriverData {
         WebDriverData {
             load_channel: None,
             resize_channel: None,
+            input_command_response_sender: None,
         }
     }
 }
@@ -1867,6 +1870,18 @@ where
             ScriptToConstellationMessage::FinishJavaScriptEvaluation(evaluation_id, result) => {
                 self.handle_finish_javascript_evaluation(evaluation_id, result)
             },
+            ScriptToConstellationMessage::WebDriverInputComplete(msg_id) => {
+                if let Some(ref reply_sender) = self.webdriver.input_command_response_sender {
+                    reply_sender
+                        .send(WebDriverCommandResponse { id: msg_id })
+                        .unwrap_or_else(|_| {
+                            warn!("Failed to send WebDriverInputComplete {:?}", msg_id);
+                            self.webdriver.input_command_response_sender = None;
+                        });
+                } else {
+                    warn!("No WebDriver input_command_response_sender");
+                }
+            },
         }
     }
 
@@ -4836,7 +4851,11 @@ where
                 mouse_button,
                 x,
                 y,
+                msg_id,
+                response_sender,
             ) => {
+                self.webdriver.input_command_response_sender = Some(response_sender);
+
                 self.compositor_proxy
                     .send(CompositorMsg::WebDriverMouseButtonEvent(
                         webview_id,
@@ -4844,11 +4863,16 @@ where
                         mouse_button,
                         x,
                         y,
+                        msg_id,
                     ));
             },
-            WebDriverCommandMsg::MouseMoveAction(webview_id, x, y) => {
+            WebDriverCommandMsg::MouseMoveAction(webview_id, x, y, msg_id, response_sender) => {
+                self.webdriver.input_command_response_sender = Some(response_sender);
+
                 self.compositor_proxy
-                    .send(CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y));
+                    .send(CompositorMsg::WebDriverMouseMoveEvent(
+                        webview_id, x, y, msg_id,
+                    ));
             },
             WebDriverCommandMsg::WheelScrollAction(webview, x, y, delta_x, delta_y) => {
                 self.compositor_proxy

components/constellation/tracing.rs

@@ -177,6 +177,7 @@ mod from_script {
                 Self::TitleChanged(..) => target!("TitleChanged"),
                 Self::IFrameSizes(..) => target!("IFrameSizes"),
                 Self::ReportMemory(..) => target!("ReportMemory"),
+                Self::WebDriverInputComplete(..) => target!("WebDriverInputComplete"),
                 Self::FinishJavaScriptEvaluation(..) => target!("FinishJavaScriptEvaluation"),
             }
         }

components/script/script_thread.rs

@@ -1081,6 +1081,10 @@ impl ScriptThread {
         for event in document.take_pending_input_events().into_iter() {
             document.update_active_keyboard_modifiers(event.active_keyboard_modifiers);
 
+            // We do this now, because the event is consumed below, but the order doesn't really
+            // matter as the event will be handled before any new ScriptThread messages are processed.
+            self.notify_webdriver_input_event_completed(pipeline_id, &event.event);
+
             match event.event {
                 InputEvent::MouseButton(mouse_button_event) => {
                     document.handle_mouse_button_event(
@@ -1144,6 +1148,19 @@ impl ScriptThread {
         ScriptThread::set_user_interacting(false);
     }
 
+    fn notify_webdriver_input_event_completed(&self, pipeline_id: PipelineId, event: &InputEvent) {
+        let Some(id) = event.webdriver_message_id() else {
+            return;
+        };
+
+        if let Err(error) = self.senders.pipeline_to_constellation_sender.send((
+            pipeline_id,
+            ScriptToConstellationMessage::WebDriverInputComplete(id),
+        )) {
+            warn!("ScriptThread failed to send WebDriverInputComplete {id:?}: {error:?}",);
+        }
+    }
+
     /// <https://html.spec.whatwg.org/multipage/#update-the-rendering>
     ///
     /// Attempt to update the rendering and then do a microtask checkpoint if rendering was actually
@@ -3420,7 +3437,7 @@ impl ScriptThread {
         // the pointer, when the user presses down and releases the primary pointer button"
 
         // Servo-specific: Trigger if within 10px of the down point
-        if let InputEvent::MouseButton(mouse_button_event) = event.event {
+        if let InputEvent::MouseButton(mouse_button_event) = &event.event {
             if let MouseButton::Left = mouse_button_event.button {
                 match mouse_button_event.action {
                     MouseButtonAction::Up => {
@@ -3429,16 +3446,23 @@ impl ScriptThread {
                         let pixel_dist =
                             (pixel_dist.x * pixel_dist.x + pixel_dist.y * pixel_dist.y).sqrt();
                         if pixel_dist < 10.0 * document.window().device_pixel_ratio().get() {
-                            document.note_pending_input_event(event.clone());
+                            // Pass webdriver_id to the newly generated click event
+                            document.note_pending_input_event(ConstellationInputEvent {
+                                hit_test_result: event.hit_test_result.clone(),
+                                pressed_mouse_buttons: event.pressed_mouse_buttons,
+                                active_keyboard_modifiers: event.active_keyboard_modifiers,
+                                event: event.event.clone().with_webdriver_message_id(None),
+                            });
                             document.note_pending_input_event(ConstellationInputEvent {
                                 hit_test_result: event.hit_test_result,
                                 pressed_mouse_buttons: event.pressed_mouse_buttons,
                                 active_keyboard_modifiers: event.active_keyboard_modifiers,
-                                event: InputEvent::MouseButton(MouseButtonEvent {
+                                event: InputEvent::MouseButton(MouseButtonEvent::new(
-                                    action: MouseButtonAction::Click,
+                                    MouseButtonAction::Click,
-                                    button: mouse_button_event.button,
+                                    mouse_button_event.button,
-                                    point: mouse_button_event.point,
+                                    mouse_button_event.point,
-                                }),
+                                ))
+                                .with_webdriver_message_id(event.event.webdriver_message_id()),
                             });
                             return;
                         }

components/shared/compositing/lib.rs

@@ -10,6 +10,7 @@ use base::id::{PipelineId, WebViewId};
 use crossbeam_channel::Sender;
 use embedder_traits::{
     AnimationState, EventLoopWaker, MouseButton, MouseButtonAction, TouchEventResult,
+    WebDriverMessageId,
 };
 use euclid::Rect;
 use ipc_channel::ipc::IpcSender;
@@ -101,9 +102,16 @@ pub enum CompositorMsg {
     /// The load of a page has completed
     LoadComplete(WebViewId),
     /// WebDriver mouse button event
-    WebDriverMouseButtonEvent(WebViewId, MouseButtonAction, MouseButton, f32, f32),
+    WebDriverMouseButtonEvent(
+        WebViewId,
+        MouseButtonAction,
+        MouseButton,
+        f32,
+        f32,
+        WebDriverMessageId,
+    ),
     /// WebDriver mouse move event
-    WebDriverMouseMoveEvent(WebViewId, f32, f32),
+    WebDriverMouseMoveEvent(WebViewId, f32, f32, WebDriverMessageId),
     // Webdriver wheel scroll event
     WebDriverWheelScrollEvent(WebViewId, f32, f32, f64, f64),
 

components/shared/constellation/from_script_message.rs

@@ -17,6 +17,7 @@ use devtools_traits::{DevtoolScriptControlMsg, ScriptToDevtoolsControlMsg, Worke
 use embedder_traits::{
     AnimationState, EmbedderMsg, FocusSequenceNumber, JSValue, JavaScriptEvaluationError,
     JavaScriptEvaluationId, MediaSessionEvent, TouchEventResult, ViewportDetails,
+    WebDriverMessageId,
 };
 use euclid::default::Size2D as UntypedSize2D;
 use http::{HeaderMap, Method};
@@ -652,6 +653,8 @@ pub enum ScriptToConstellationMessage {
         JavaScriptEvaluationId,
         Result<JSValue, JavaScriptEvaluationError>,
     ),
+    /// Notify the completion of a webdriver command.
+    WebDriverInputComplete(WebDriverMessageId),
 }
 
 impl fmt::Debug for ScriptToConstellationMessage {

components/shared/embedder/input_events.rs

@@ -8,6 +8,8 @@ use malloc_size_of_derive::MallocSizeOf;
 use serde::{Deserialize, Serialize};
 use webrender_api::units::DevicePoint;
 
+use crate::WebDriverMessageId;
+
 /// An input event that is sent from the embedder to Servo.
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub enum InputEvent {
@@ -42,6 +44,38 @@ impl InputEvent {
             InputEvent::Wheel(event) => Some(event.point),
         }
     }
+
+    pub fn webdriver_message_id(&self) -> Option<WebDriverMessageId> {
+        match self {
+            InputEvent::EditingAction(..) => None,
+            InputEvent::Gamepad(..) => None,
+            InputEvent::Ime(..) => None,
+            InputEvent::Keyboard(..) => None,
+            InputEvent::MouseButton(event) => event.webdriver_id,
+            InputEvent::MouseMove(event) => event.webdriver_id,
+            InputEvent::Touch(..) => None,
+            InputEvent::Wheel(..) => None,
+        }
+    }
+
+    pub fn with_webdriver_message_id(self, webdriver_id: Option<WebDriverMessageId>) -> Self {
+        match self {
+            InputEvent::EditingAction(..) => {},
+            InputEvent::Gamepad(..) => {},
+            InputEvent::Ime(..) => {},
+            InputEvent::Keyboard(..) => {},
+            InputEvent::MouseButton(mut event) => {
+                event.webdriver_id = webdriver_id;
+            },
+            InputEvent::MouseMove(mut event) => {
+                event.webdriver_id = webdriver_id;
+            },
+            InputEvent::Touch(..) => {},
+            InputEvent::Wheel(..) => {},
+        };
+
+        self
+    }
 }
 
 #[derive(Clone, Copy, Debug, Deserialize, Serialize)]
@@ -49,6 +83,18 @@ pub struct MouseButtonEvent {
     pub action: MouseButtonAction,
     pub button: MouseButton,
     pub point: DevicePoint,
+    webdriver_id: Option<WebDriverMessageId>,
+}
+
+impl MouseButtonEvent {
+    pub fn new(action: MouseButtonAction, button: MouseButton, point: DevicePoint) -> Self {
+        Self {
+            action,
+            button,
+            point,
+            webdriver_id: None,
+        }
+    }
 }
 
 #[derive(Clone, Copy, Debug, Deserialize, Serialize)]
@@ -102,6 +148,16 @@ pub enum MouseButtonAction {
 #[derive(Clone, Copy, Debug, Deserialize, Serialize)]
 pub struct MouseMoveEvent {
     pub point: DevicePoint,
+    webdriver_id: Option<WebDriverMessageId>,
+}
+
+impl MouseMoveEvent {
+    pub fn new(point: DevicePoint) -> Self {
+        Self {
+            point,
+            webdriver_id: None,
+        }
+    }
 }
 
 /// The type of input represented by a multi-touch event.

components/shared/embedder/webdriver.rs

@@ -24,6 +24,9 @@ use webrender_api::units::DeviceIntSize;
 
 use crate::{MouseButton, MouseButtonAction};
 
+#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)]
+pub struct WebDriverMessageId(pub usize);
+
 /// Messages to the constellation originating from the WebDriver server.
 #[derive(Debug, Deserialize, Serialize)]
 pub enum WebDriverCommandMsg {
@@ -41,9 +44,23 @@ pub enum WebDriverCommandMsg {
     /// Act as if keys were pressed or release in the browsing context with the given ID.
     KeyboardAction(BrowsingContextId, KeyboardEvent),
     /// Act as if the mouse was clicked in the browsing context with the given ID.
-    MouseButtonAction(WebViewId, MouseButtonAction, MouseButton, f32, f32),
+    MouseButtonAction(
+        WebViewId,
+        MouseButtonAction,
+        MouseButton,
+        f32,
+        f32,
+        WebDriverMessageId,
+        IpcSender<WebDriverCommandResponse>,
+    ),
     /// Act as if the mouse was moved in the browsing context with the given ID.
-    MouseMoveAction(WebViewId, f32, f32),
+    MouseMoveAction(
+        WebViewId,
+        f32,
+        f32,
+        WebDriverMessageId,
+        IpcSender<WebDriverCommandResponse>,
+    ),
     /// Act as if the mouse wheel is scrolled in the browsing context given the given ID.
     WheelScrollAction(WebViewId, f32, f32, f64, f64),
     /// Set the window size.
@@ -188,6 +205,11 @@ pub enum WebDriverFrameId {
     Parent,
 }
 
+#[derive(Debug, Deserialize, Serialize)]
+pub struct WebDriverCommandResponse {
+    pub id: WebDriverMessageId,
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub enum WebDriverLoadStatus {
     Complete,

components/webdriver_server/actions.rs

@@ -98,20 +98,83 @@ fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 {
 impl Handler {
     // https://w3c.github.io/webdriver/#dfn-dispatch-actions
     pub(crate) fn dispatch_actions(
-        &mut self,
+        &self,
         actions_by_tick: &[ActionSequence],
     ) -> Result<(), ErrorStatus> {
+        // Step 1. Wait for an action queue token with input state.
+        let new_token = self.id_generator.next();
+        assert!(self.current_action_id.get().is_none());
+        self.current_action_id.set(Some(new_token));
+
+        // Step 2. Let actions result be the result of dispatch actions inner.
+        let res = self.dispatch_actions_inner(actions_by_tick);
+
+        // Step 3. Dequeue input state's actions queue.
+        self.current_action_id.set(None);
+
+        // Step 4. Return actions result.
+        res
+    }
+
+    // https://w3c.github.io/webdriver/#dfn-dispatch-actions-inner
+    fn dispatch_actions_inner(
+        &self,
+        actions_by_tick: &[ActionSequence],
+    ) -> Result<(), ErrorStatus> {
+        // Step 1. For each item tick actions in actions by tick
         for tick_actions in actions_by_tick.iter() {
+            // Step 1.2. Let tick duration be the result of
+            // computing the tick duration with argument tick actions.
             let tick_duration = compute_tick_duration(tick_actions);
+
+            // Step 1.3. Try to dispatch tick actions
             self.dispatch_tick_actions(tick_actions, tick_duration)?;
+
+            // Step 1.4. Wait for
+            // The user agent event loop has spun enough times to process the DOM events
+            // generated by the last invocation of the dispatch tick actions steps.
+            //
+            // To ensure we wait for all events to be processed, only the last event in
+            // this tick action step holds the message id.
+            // Whenever a new event is generated, the message id is passed to it.
+            //
+            // TO-DO: remove the first match after webdriver_id is implemented in all commands
+            match tick_actions.actions {
+                ActionsType::Key { .. } | ActionsType::Wheel { .. } | ActionsType::Null { .. } => {
+                    return Ok(());
+                },
+                _ => {},
+            }
+
+            match self.constellation_receiver.recv() {
+                Ok(response) => {
+                    let current_waiting_id = self
+                        .current_action_id
+                        .get()
+                        .expect("Current id should be set before dispat_actions_inner is called");
+
+                    if current_waiting_id != response.id {
+                        dbg!("Dispatch actions completed with wrong id in response");
+                        return Err(ErrorStatus::UnknownError);
+                    }
+                },
+                Err(error) => {
+                    dbg!("Dispatch actions completed with IPC error: {:?}", error);
+                    return Err(ErrorStatus::UnknownError);
+                },
+            };
         }
+
+        // Step 2. Return success with data null.
+        dbg!("Dispatch actions completed successfully");
         Ok(())
     }
 
-    fn dispatch_general_action(&mut self, source_id: &str) {
+    fn dispatch_general_action(&self, source_id: &str) {
-        self.session_mut()
+        self.session()
             .unwrap()
             .input_state_table
+            .borrow_mut()
             .entry(source_id.to_string())
             .or_insert(InputSourceState::Null);
         // https://w3c.github.io/webdriver/#dfn-dispatch-a-pause-action
@@ -120,7 +183,7 @@ impl Handler {
 
     // https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions
     fn dispatch_tick_actions(
-        &mut self,
+        &self,
         tick_actions: &ActionSequence,
         tick_duration: u64,
     ) -> Result<(), ErrorStatus> {
@@ -138,9 +201,10 @@ impl Handler {
                             self.dispatch_general_action(source_id);
                         },
                         KeyActionItem::Key(action) => {
-                            self.session_mut()
+                            self.session()
                                 .unwrap()
                                 .input_state_table
+                                .borrow_mut()
                                 .entry(source_id.to_string())
                                 .or_insert(InputSourceState::Key(KeyInputState::new()));
                             match action {
@@ -149,7 +213,7 @@ impl Handler {
                                     // Step 9. If subtype is "keyDown", append a copy of action
                                     // object with the subtype property changed to "keyUp" to
                                     // input state's input cancel list.
-                                    self.session_mut().unwrap().input_cancel_list.push(
+                                    self.session().unwrap().input_cancel_list.borrow_mut().push(
                                         ActionSequence {
                                             id: source_id.into(),
                                             actions: ActionsType::Key {
@@ -180,9 +244,10 @@ impl Handler {
                             self.dispatch_general_action(source_id);
                         },
                         PointerActionItem::Pointer(action) => {
-                            self.session_mut()
+                            self.session()
                                 .unwrap()
                                 .input_state_table
+                                .borrow_mut()
                                 .entry(source_id.to_string())
                                 .or_insert(InputSourceState::Pointer(PointerInputState::new(
                                     &parameters.pointer_type,
@@ -195,7 +260,7 @@ impl Handler {
                                     // Step 10. If subtype is "pointerDown", append a copy of action
                                     // object with the subtype property changed to "pointerUp" to
                                     // input state's input cancel list.
-                                    self.session_mut().unwrap().input_cancel_list.push(
+                                    self.session().unwrap().input_cancel_list.borrow_mut().push(
                                         ActionSequence {
                                             id: source_id.into(),
                                             actions: ActionsType::Pointer {
@@ -232,9 +297,10 @@ impl Handler {
                             self.dispatch_general_action(source_id)
                         },
                         WheelActionItem::Wheel(action) => {
-                            self.session_mut()
+                            self.session()
                                 .unwrap()
                                 .input_state_table
+                                .borrow_mut()
                                 .entry(source_id.to_string())
                                 .or_insert(InputSourceState::Wheel);
                             match action {
@@ -252,12 +318,25 @@ impl Handler {
     }
 
     // https://w3c.github.io/webdriver/#dfn-dispatch-a-keydown-action
-    fn dispatch_keydown_action(&mut self, source_id: &str, action: &KeyDownAction) {
+    fn dispatch_keydown_action(&self, source_id: &str, action: &KeyDownAction) {
-        // Step 1
+        let session = self.session().unwrap();
-        let raw_key = action.value.chars().next().unwrap();
+
-        let key_input_state = self.get_key_input_state_mut(source_id);
+        let raw_key = action.value.chars().next().unwrap();
+        let mut input_state_table = session.input_state_table.borrow_mut();
+        let key_input_state = match input_state_table.get_mut(source_id).unwrap() {
+            InputSourceState::Key(key_input_state) => key_input_state,
+            _ => unreachable!(),
+        };
+
+        session.input_cancel_list.borrow_mut().push(ActionSequence {
+            id: source_id.into(),
+            actions: ActionsType::Key {
+                actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction {
+                    value: action.value.clone(),
+                }))],
+            },
+        });
 
-        // Step 2 - 11. Done by `keyboard-types` crate.
         let keyboard_event = key_input_state.dispatch_keydown(raw_key);
 
         // Step 12
@@ -271,12 +350,25 @@ impl Handler {
     }
 
     // https://w3c.github.io/webdriver/#dfn-dispatch-a-keyup-action
-    fn dispatch_keyup_action(&mut self, source_id: &str, action: &KeyUpAction) {
+    fn dispatch_keyup_action(&self, source_id: &str, action: &KeyUpAction) {
-        // Step 1
+        let session = self.session().unwrap();
-        let raw_key = action.value.chars().next().unwrap();
+
-        let key_input_state = self.get_key_input_state_mut(source_id);
+        let raw_key = action.value.chars().next().unwrap();
+        let mut input_state_table = session.input_state_table.borrow_mut();
+        let key_input_state = match input_state_table.get_mut(source_id).unwrap() {
+            InputSourceState::Key(key_input_state) => key_input_state,
+            _ => unreachable!(),
+        };
+
+        session.input_cancel_list.borrow_mut().push(ActionSequence {
+            id: source_id.into(),
+            actions: ActionsType::Key {
+                actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction {
+                    value: action.value.clone(),
+                }))],
+            },
+        });
 
-        // Step 2 - 11. Done by `keyboard-types` crate.
         if let Some(keyboard_event) = key_input_state.dispatch_keyup(raw_key) {
             // Step 12
             let cmd_msg = WebDriverCommandMsg::KeyboardAction(
@@ -289,44 +381,49 @@ impl Handler {
         }
     }
 
-    fn get_pointer_input_state_mut(&mut self, source_id: &str) -> &mut PointerInputState {
+    /// <https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerdown-action>
-        let session = self.session_mut().unwrap();
+    pub(crate) fn dispatch_pointerdown_action(&self, source_id: &str, action: &PointerDownAction) {
-        let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
+        let session = self.session().unwrap();
+
+        let mut input_state_table = session.input_state_table.borrow_mut();
+        let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() {
             InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
             _ => unreachable!(),
         };
-        pointer_input_state
-    }
-
-    fn get_key_input_state_mut(&mut self, source_id: &str) -> &mut KeyInputState {
-        let session = self.session_mut().unwrap();
-        let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
-            InputSourceState::Key(key_input_state) => key_input_state,
-            _ => unreachable!(),
-        };
-        key_input_state
-    }
-
-    // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerdown-action
-    pub(crate) fn dispatch_pointerdown_action(
-        &mut self,
-        source_id: &str,
-        action: &PointerDownAction,
-    ) {
-        let webview_id = self.session().unwrap().webview_id;
-        let pointer_input_state = self.get_pointer_input_state_mut(source_id);
 
         if pointer_input_state.pressed.contains(&action.button) {
             return;
         }
         pointer_input_state.pressed.insert(action.button);
 
+        session.input_cancel_list.borrow_mut().push(ActionSequence {
+            id: source_id.into(),
+            actions: ActionsType::Pointer {
+                parameters: PointerActionParameters {
+                    pointer_type: match pointer_input_state.subtype {
+                        PointerType::Mouse => PointerType::Mouse,
+                        PointerType::Pen => PointerType::Pen,
+                        PointerType::Touch => PointerType::Touch,
+                    },
+                },
+                actions: vec![PointerActionItem::Pointer(PointerAction::Up(
+                    PointerUpAction {
+                        button: action.button,
+                        ..Default::default()
+                    },
+                ))],
+            },
+        });
+
+        let msg_id = self.current_action_id.get().unwrap();
         let cmd_msg = WebDriverCommandMsg::MouseButtonAction(
-            webview_id,
+            session.webview_id,
             MouseButtonAction::Down,
             action.button.into(),
             pointer_input_state.x as f32,
             pointer_input_state.y as f32,
+            msg_id,
+            self.constellation_sender.clone(),
         );
         self.constellation_chan
             .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
@@ -334,21 +431,48 @@ impl Handler {
     }
 
     // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerup-action
-    pub(crate) fn dispatch_pointerup_action(&mut self, source_id: &str, action: &PointerUpAction) {
+    pub(crate) fn dispatch_pointerup_action(&self, source_id: &str, action: &PointerUpAction) {
-        let webview_id = self.session().unwrap().webview_id;
+        let session = self.session().unwrap();
-        let pointer_input_state = self.get_pointer_input_state_mut(source_id);
+
+        let mut input_state_table = session.input_state_table.borrow_mut();
+        let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() {
+            InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
+            _ => unreachable!(),
+        };
 
         if !pointer_input_state.pressed.contains(&action.button) {
             return;
         }
         pointer_input_state.pressed.remove(&action.button);
 
+        session.input_cancel_list.borrow_mut().push(ActionSequence {
+            id: source_id.into(),
+            actions: ActionsType::Pointer {
+                parameters: PointerActionParameters {
+                    pointer_type: match pointer_input_state.subtype {
+                        PointerType::Mouse => PointerType::Mouse,
+                        PointerType::Pen => PointerType::Pen,
+                        PointerType::Touch => PointerType::Touch,
+                    },
+                },
+                actions: vec![PointerActionItem::Pointer(PointerAction::Down(
+                    PointerDownAction {
+                        button: action.button,
+                        ..Default::default()
+                    },
+                ))],
+            },
+        });
+
+        let msg_id = self.current_action_id.get().unwrap();
         let cmd_msg = WebDriverCommandMsg::MouseButtonAction(
-            webview_id,
+            session.webview_id,
             MouseButtonAction::Up,
             action.button.into(),
             pointer_input_state.x as f32,
             pointer_input_state.y as f32,
+            msg_id,
+            self.constellation_sender.clone(),
         );
         self.constellation_chan
             .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
@@ -357,7 +481,7 @@ impl Handler {
 
     // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointermove-action
     pub(crate) fn dispatch_pointermove_action(
-        &mut self,
+        &self,
         source_id: &str,
         action: &PointerMoveAction,
         tick_duration: u64,
@@ -370,10 +494,10 @@ impl Handler {
 
         // Steps 3 - 4
         let (start_x, start_y) = match self
-            .session
+            .session()
-            .as_ref()
             .unwrap()
             .input_state_table
+            .borrow_mut()
             .get(source_id)
             .unwrap()
         {
@@ -416,7 +540,7 @@ impl Handler {
     /// <https://w3c.github.io/webdriver/#dfn-perform-a-pointer-move>
     #[allow(clippy::too_many_arguments)]
     fn perform_pointer_move(
-        &mut self,
+        &self,
         source_id: &str,
         duration: u64,
         start_x: f64,
@@ -425,9 +549,13 @@ impl Handler {
         target_y: f64,
         tick_start: Instant,
     ) {
-        let webview_id = self.session().unwrap().webview_id;
+        let session = self.session().unwrap();
-        let constellation_chan = self.constellation_chan.clone();
+        let mut input_state_table = session.input_state_table.borrow_mut();
-        let pointer_input_state = self.get_pointer_input_state_mut(source_id);
+        let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() {
+            InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
+            _ => unreachable!(),
+        };
+
         loop {
             // Step 1
             let time_delta = tick_start.elapsed().as_millis();
@@ -459,9 +587,15 @@ impl Handler {
             // Step 7
             if x != current_x || y != current_y {
                 // Step 7.2
-                let cmd_msg = WebDriverCommandMsg::MouseMoveAction(webview_id, x as f32, y as f32);
+                let msg_id = self.current_action_id.get().unwrap();
-                //TODO: Need Synchronization here before updating `pointer_input_state`
+                let cmd_msg = WebDriverCommandMsg::MouseMoveAction(
-                constellation_chan
+                    session.webview_id,
+                    x as f32,
+                    y as f32,
+                    msg_id,
+                    self.constellation_sender.clone(),
+                );
+                self.constellation_chan
                     .send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
                     .unwrap();
                 // Step 7.3
@@ -481,7 +615,7 @@ impl Handler {
 
     /// <https://w3c.github.io/webdriver/#dfn-dispatch-a-scroll-action>
     fn dispatch_scroll_action(
-        &mut self,
+        &self,
         action: &WheelScrollAction,
         tick_duration: u64,
     ) -> Result<(), ErrorStatus> {
@@ -546,7 +680,7 @@ impl Handler {
     /// <https://w3c.github.io/webdriver/#dfn-perform-a-scroll>
     #[allow(clippy::too_many_arguments)]
     fn perform_scroll(
-        &mut self,
+        &self,
         duration: u64,
         x: i64,
         y: i64,
@@ -556,7 +690,7 @@ impl Handler {
         mut curr_delta_y: i64,
         tick_start: Instant,
     ) {
-        let session = self.session_mut().unwrap();
+        let session = self.session().unwrap();
 
         // Step 1
         let time_delta = tick_start.elapsed().as_millis();

components/webdriver_server/lib.rs

@@ -10,11 +10,12 @@ mod actions;
 mod capabilities;
 
 use std::borrow::ToOwned;
+use std::cell::{Cell, RefCell};
 use std::collections::{BTreeMap, HashMap};
 use std::io::Cursor;
 use std::net::{SocketAddr, SocketAddrV4};
 use std::time::Duration;
-use std::{env, fmt, mem, process, thread};
+use std::{env, fmt, process, thread};
 
 use base::id::{BrowsingContextId, WebViewId};
 use base64::Engine;
@@ -23,8 +24,9 @@ use constellation_traits::{EmbedderToConstellationMessage, TraversalDirection};
 use cookie::{CookieBuilder, Expiration};
 use crossbeam_channel::{Receiver, Sender, after, select, unbounded};
 use embedder_traits::{
-    MouseButton, WebDriverCommandMsg, WebDriverCookieError, WebDriverFrameId, WebDriverJSError,
+    MouseButton, WebDriverCommandMsg, WebDriverCommandResponse, WebDriverCookieError,
-    WebDriverJSResult, WebDriverJSValue, WebDriverLoadStatus, WebDriverScriptCommand,
+    WebDriverFrameId, WebDriverJSError, WebDriverJSResult, WebDriverJSValue, WebDriverLoadStatus,
+    WebDriverMessageId, WebDriverScriptCommand,
 };
 use euclid::{Rect, Size2D};
 use http::method::Method;
@@ -43,8 +45,8 @@ use servo_url::ServoUrl;
 use style_traits::CSSPixel;
 use uuid::Uuid;
 use webdriver::actions::{
-    ActionSequence, PointerDownAction, PointerMoveAction, PointerOrigin, PointerType,
+    ActionSequence, ActionsType, PointerAction, PointerActionItem, PointerActionParameters,
-    PointerUpAction,
+    PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction,
 };
 use webdriver::capabilities::CapabilitiesMatching;
 use webdriver::command::{
@@ -64,6 +66,26 @@ use webdriver::server::{self, Session, SessionTeardownKind, WebDriverHandler};
 
 use crate::actions::{InputSourceState, PointerInputState};
 
+#[derive(Default)]
+pub struct WebDriverMessageIdGenerator {
+    counter: Cell<usize>,
+}
+
+impl WebDriverMessageIdGenerator {
+    pub fn new() -> Self {
+        Self {
+            counter: Cell::new(0),
+        }
+    }
+
+    /// Returns a unique ID.
+    pub fn next(&self) -> WebDriverMessageId {
+        let id = self.counter.get();
+        self.counter.set(id + 1);
+        WebDriverMessageId(id)
+    }
+}
+
 fn extension_routes() -> Vec<(Method, &'static str, ServoExtensionRoute)> {
     vec![
         (
@@ -145,10 +167,11 @@ pub struct WebDriverSession {
 
     unhandled_prompt_behavior: String,
 
-    // https://w3c.github.io/webdriver/#dfn-input-state-table
+    /// <https://w3c.github.io/webdriver/#dfn-input-state-map>
-    input_state_table: HashMap<String, InputSourceState>,
+    input_state_table: RefCell<HashMap<String, InputSourceState>>,
-    // https://w3c.github.io/webdriver/#dfn-input-cancel-list
+
-    input_cancel_list: Vec<ActionSequence>,
+    /// <https://w3c.github.io/webdriver/#dfn-input-cancel-list>
+    input_cancel_list: RefCell<Vec<ActionSequence>>,
 }
 
 impl WebDriverSession {
@@ -172,8 +195,8 @@ impl WebDriverSession {
             strict_file_interactability: false,
             unhandled_prompt_behavior: "dismiss and notify".to_string(),
 
-            input_state_table: HashMap::new(),
+            input_state_table: RefCell::new(HashMap::new()),
-            input_cancel_list: Vec::new(),
+            input_cancel_list: RefCell::new(Vec::new()),
         }
     }
 }
@@ -187,8 +210,22 @@ struct Handler {
     /// for it to send us a load-status. Messages sent on it
     /// will be forwarded to the load_status_receiver.
     load_status_sender: IpcSender<WebDriverLoadStatus>,
+
     session: Option<WebDriverSession>,
+
+    /// The channel for sending Webdriver messages to the constellation.
     constellation_chan: Sender<EmbedderToConstellationMessage>,
+
+    /// The IPC sender which we can clone and pass along to the constellation
+    constellation_sender: IpcSender<WebDriverCommandResponse>,
+
+    /// Receiver notification from the constellation when a command is completed
+    constellation_receiver: IpcReceiver<WebDriverCommandResponse>,
+
+    id_generator: WebDriverMessageIdGenerator,
+
+    current_action_id: Cell<Option<WebDriverMessageId>>,
+
     resize_timeout: u32,
 }
 
@@ -409,11 +446,18 @@ impl Handler {
         let (load_status_sender, receiver) = ipc::channel().unwrap();
         let (sender, load_status_receiver) = unbounded();
         ROUTER.route_ipc_receiver_to_crossbeam_sender(receiver, sender);
+
+        let (constellation_sender, constellation_receiver) = ipc::channel().unwrap();
+
         Handler {
             load_status_sender,
             load_status_receiver,
             session: None,
             constellation_chan,
+            constellation_sender,
+            constellation_receiver,
+            id_generator: WebDriverMessageIdGenerator::new(),
+            current_action_id: Cell::new(None),
             resize_timeout: 500,
         }
     }
@@ -1445,18 +1489,13 @@ impl Handler {
     }
 
     fn handle_release_actions(&mut self) -> WebDriverResult<WebDriverResponse> {
-        let input_cancel_list = {
+        let input_cancel_list = self.session().unwrap().input_cancel_list.borrow();
-            let session = self.session_mut()?;
-            session.input_cancel_list.reverse();
-            mem::take(&mut session.input_cancel_list)
-        };
-
         if let Err(error) = self.dispatch_actions(&input_cancel_list) {
             return Err(WebDriverError::new(error, ""));
         }
 
-        let session = self.session_mut()?;
+        let session = self.session()?;
-        session.input_state_table = HashMap::new();
+        session.input_state_table.borrow_mut().clear();
 
         Ok(WebDriverResponse::Void)
     }
@@ -1614,7 +1653,7 @@ impl Handler {
                     let id = Uuid::new_v4().to_string();
 
                     // Step 8.1
-                    self.session_mut()?.input_state_table.insert(
+                    self.session_mut()?.input_state_table.borrow_mut().insert(
                         id.clone(),
                         InputSourceState::Pointer(PointerInputState::new(&PointerType::Mouse)),
                     );
@@ -1645,19 +1684,31 @@ impl Handler {
                         ..Default::default()
                     };
 
-                    // Step 8.16 Dispatch a list of actions with input state,
+                    let action_sequence = ActionSequence {
-                    // actions, session's current browsing context, and actions options.
+                        id: id.clone(),
-                    if let Err(error) =
+                        actions: ActionsType::Pointer {
-                        self.dispatch_pointermove_action(&id, &pointer_move_action, 0)
+                            parameters: PointerActionParameters {
-                    {
+                                pointer_type: PointerType::Mouse,
-                        return Err(WebDriverError::new(error, ""));
+                            },
-                    }
+                            actions: vec![
+                                PointerActionItem::Pointer(PointerAction::Move(
+                                    pointer_move_action,
+                                )),
+                                PointerActionItem::Pointer(PointerAction::Down(
+                                    pointer_down_action,
+                                )),
+                                PointerActionItem::Pointer(PointerAction::Up(pointer_up_action)),
+                            ],
+                        },
+                    };
 
-                    self.dispatch_pointerdown_action(&id, &pointer_down_action);
+                    let _ = self.dispatch_actions(&[action_sequence]);
-                    self.dispatch_pointerup_action(&id, &pointer_up_action);
 
                     // Step 8.17 Remove an input source with input state and input id.
-                    self.session_mut()?.input_state_table.remove(&id);
+                    self.session_mut()?
+                        .input_state_table
+                        .borrow_mut()
+                        .remove(&id);
 
                     // Step 13
                     Ok(WebDriverResponse::Void)

ports/servoshell/desktop/headed_window.rs

@@ -262,11 +262,11 @@ impl Window {
             ElementState::Released => MouseButtonAction::Up,
         };
 
-        webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
+        webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
             action,
-            button: mouse_button,
+            mouse_button,
             point,
-        }));
+        )));
     }
 
     /// Handle key events before sending them to Servo.
@@ -563,7 +563,7 @@ impl WindowPortsMethods for Window {
                 point.y -= (self.toolbar_height() * self.hidpi_scale_factor()).0;
 
                 self.webview_relative_mouse_point.set(point);
-                webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent { point }));
+                webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));
             },
             WindowEvent::MouseWheel { delta, phase, .. } => {
                 let (mut dx, mut dy, mode) = match delta {

ports/servoshell/egl/app_state.rs

@@ -537,31 +537,31 @@ impl RunningAppState {
     /// Register a mouse movement.
     pub fn mouse_move(&self, x: f32, y: f32) {
         self.active_webview()
-            .notify_input_event(InputEvent::MouseMove(MouseMoveEvent {
+            .notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(Point2D::new(
-                point: Point2D::new(x, y),
+                x, y,
-            }));
+            ))));
         self.perform_updates();
     }
 
     /// Register a mouse button press.
     pub fn mouse_down(&self, x: f32, y: f32, button: MouseButton) {
         self.active_webview()
-            .notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
+            .notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
-                action: MouseButtonAction::Down,
+                MouseButtonAction::Down,
                 button,
-                point: Point2D::new(x, y),
+                Point2D::new(x, y),
-            }));
+            )));
         self.perform_updates();
     }
 
     /// Register a mouse button release.
     pub fn mouse_up(&self, x: f32, y: f32, button: MouseButton) {
         self.active_webview()
-            .notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
+            .notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
-                action: MouseButtonAction::Up,
+                MouseButtonAction::Up,
                 button,
-                point: Point2D::new(x, y),
+                Point2D::new(x, y),
-            }));
+            )));
         self.perform_updates();
     }
 
@@ -589,11 +589,11 @@ impl RunningAppState {
     /// Perform a click.
     pub fn click(&self, x: f32, y: f32) {
         self.active_webview()
-            .notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
+            .notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
-                action: MouseButtonAction::Click,
+                MouseButtonAction::Click,
-                button: MouseButton::Left,
+                MouseButton::Left,
-                point: Point2D::new(x, y),
+                Point2D::new(x, y),
-            }));
+            )));
         self.perform_updates();
     }