Auto merge of #23805 - georgeroman:initial_actions_support_in_webdriver, r=jdm

Initial Actions support in WebDriver

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors

<!-- Either: -->
- [X] There are tests for these changes

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/23805)
<!-- Reviewable:end -->

components/constellation/constellation.rs

@@ -3468,6 +3468,28 @@ where
                     }
                 }
             },
+            WebDriverCommandMsg::KeyboardAction(browsing_context_id, event) => {
+                let pipeline_id = match self.browsing_contexts.get(&browsing_context_id) {
+                    Some(browsing_context) => browsing_context.pipeline_id,
+                    None => {
+                        return warn!(
+                            "Browsing context {} KeyboardAction after closure.",
+                            browsing_context_id
+                        );
+                    },
+                };
+                let event_loop = match self.pipelines.get(&pipeline_id) {
+                    Some(pipeline) => pipeline.event_loop.clone(),
+                    None => return warn!("Pipeline {} KeyboardAction after closure.", pipeline_id),
+                };
+                let control_msg = ConstellationControlMsg::SendEvent(
+                    pipeline_id,
+                    CompositorEvent::KeyboardEvent(event),
+                );
+                if let Err(e) = event_loop.send(control_msg) {
+                    return self.handle_send_error(pipeline_id, e);
+                }
+            },
             WebDriverCommandMsg::TakeScreenshot(_, reply) => {
                 self.compositor_proxy
                     .send(ToCompositorMsg::CreatePng(reply));

components/script_traits/lib.rs

@@ -798,6 +798,8 @@ pub enum WebDriverCommandMsg {
     ScriptCommand(BrowsingContextId, WebDriverScriptCommand),
     /// Act as if keys were pressed in the browsing context with the given ID.
     SendKeys(BrowsingContextId, Vec<WebDriverInputEvent>),
+    /// Act as if keys were pressed or release in the browsing context with the given ID.
+    KeyboardAction(BrowsingContextId, KeyboardEvent),
     /// Set the window size.
     SetWindowSize(
         TopLevelBrowsingContextId,

components/webdriver_server/actions.rs

@@ -0,0 +1,220 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+use crate::Handler;
+use keyboard_types::webdriver::KeyInputState;
+use script_traits::{ConstellationMsg, WebDriverCommandMsg};
+use std::cmp;
+use std::collections::HashSet;
+use webdriver::actions::{ActionSequence, ActionsType, GeneralAction, NullActionItem};
+use webdriver::actions::{KeyAction, KeyActionItem, KeyDownAction, KeyUpAction};
+use webdriver::actions::{PointerAction, PointerActionItem, PointerType};
+
+// https://w3c.github.io/webdriver/#dfn-input-source-state
+pub(crate) enum InputSourceState {
+    Null,
+    Key(KeyInputState),
+    Pointer(PointerInputState),
+}
+
+// https://w3c.github.io/webdriver/#dfn-pointer-input-source
+pub(crate) struct PointerInputState {
+    _subtype: PointerType,
+    _pressed: HashSet<u64>,
+    _x: u64,
+    _y: u64,
+}
+
+impl PointerInputState {
+    pub fn new(subtype: &PointerType) -> PointerInputState {
+        PointerInputState {
+            _subtype: match subtype {
+                PointerType::Mouse => PointerType::Mouse,
+                PointerType::Pen => PointerType::Pen,
+                PointerType::Touch => PointerType::Touch,
+            },
+            _pressed: HashSet::new(),
+            _x: 0,
+            _y: 0,
+        }
+    }
+}
+
+// https://w3c.github.io/webdriver/#dfn-computing-the-tick-duration
+fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 {
+    let mut duration = 0;
+    match &tick_actions.actions {
+        ActionsType::Null { actions } => {
+            for action in actions.iter() {
+                let NullActionItem::General(GeneralAction::Pause(pause_action)) = action;
+                duration = cmp::max(duration, pause_action.duration.unwrap_or(0));
+            }
+        },
+        ActionsType::Pointer {
+            parameters: _,
+            actions,
+        } => {
+            for action in actions.iter() {
+                let action_duration = match action {
+                    PointerActionItem::General(GeneralAction::Pause(action)) => action.duration,
+                    PointerActionItem::Pointer(PointerAction::Move(action)) => action.duration,
+                    _ => None,
+                };
+                duration = cmp::max(duration, action_duration.unwrap_or(0));
+            }
+        },
+        ActionsType::Key { actions: _ } => (),
+    }
+    duration
+}
+
+impl Handler {
+    // https://w3c.github.io/webdriver/#dfn-dispatch-actions
+    pub(crate) fn dispatch_actions(&mut self, actions_by_tick: &[ActionSequence]) {
+        for tick_actions in actions_by_tick.iter() {
+            let tick_duration = compute_tick_duration(&tick_actions);
+            self.dispatch_tick_actions(&tick_actions, tick_duration);
+        }
+    }
+
+    fn dispatch_general_action(&mut self, source_id: &str) {
+        self.session_mut()
+            .unwrap()
+            .input_state_table
+            .entry(source_id.to_string())
+            .or_insert(InputSourceState::Null);
+        // https://w3c.github.io/webdriver/#dfn-dispatch-a-pause-action
+        // Nothing to be done
+    }
+
+    // https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions
+    fn dispatch_tick_actions(&mut self, tick_actions: &ActionSequence, tick_duration: u64) {
+        let source_id = &tick_actions.id;
+        match &tick_actions.actions {
+            ActionsType::Null { actions } => {
+                for _action in actions.iter() {
+                    self.dispatch_general_action(source_id);
+                }
+            },
+            ActionsType::Key { actions } => {
+                for action in actions.iter() {
+                    match action {
+                        KeyActionItem::General(_action) => {
+                            self.dispatch_general_action(source_id);
+                        },
+                        KeyActionItem::Key(action) => {
+                            self.session_mut()
+                                .unwrap()
+                                .input_state_table
+                                .entry(source_id.to_string())
+                                .or_insert(InputSourceState::Key(KeyInputState::new()));
+                            match action {
+                                KeyAction::Down(action) => {
+                                    self.dispatch_keydown_action(&source_id, &action, tick_duration)
+                                },
+                                KeyAction::Up(action) => {
+                                    self.dispatch_keyup_action(&source_id, &action, tick_duration)
+                                },
+                            };
+                        },
+                    }
+                }
+            },
+            ActionsType::Pointer {
+                parameters,
+                actions,
+            } => {
+                for action in actions.iter() {
+                    match action {
+                        PointerActionItem::General(_action) => {
+                            self.dispatch_general_action(source_id);
+                        },
+                        PointerActionItem::Pointer(action) => {
+                            self.session_mut()
+                                .unwrap()
+                                .input_state_table
+                                .entry(source_id.to_string())
+                                .or_insert(InputSourceState::Pointer(PointerInputState::new(
+                                    &parameters.pointer_type,
+                                )));
+                            match action {
+                                PointerAction::Cancel => (),
+                                PointerAction::Down(_action) => (),
+                                PointerAction::Move(_action) => (),
+                                PointerAction::Up(_action) => (),
+                            }
+                        },
+                    }
+                }
+            },
+        }
+    }
+
+    // https://w3c.github.io/webdriver/#dfn-dispatch-a-keydown-action
+    fn dispatch_keydown_action(
+        &mut self,
+        source_id: &str,
+        action: &KeyDownAction,
+        _tick_duration: u64,
+    ) {
+        let session = self.session.as_mut().unwrap();
+
+        let raw_key = action.value.chars().next().unwrap();
+        let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
+            InputSourceState::Null => unreachable!(),
+            InputSourceState::Key(key_input_state) => key_input_state,
+            InputSourceState::Pointer(_) => unreachable!(),
+        };
+
+        session.input_cancel_list.push(ActionSequence {
+            id: source_id.into(),
+            actions: ActionsType::Key {
+                actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction {
+                    value: action.value.clone(),
+                }))],
+            },
+        });
+
+        let keyboard_event = key_input_state.dispatch_keydown(raw_key);
+        let cmd_msg =
+            WebDriverCommandMsg::KeyboardAction(session.browsing_context_id, keyboard_event);
+        self.constellation_chan
+            .send(ConstellationMsg::WebDriverCommand(cmd_msg))
+            .unwrap();
+    }
+
+    // https://w3c.github.io/webdriver/#dfn-dispatch-a-keyup-action
+    fn dispatch_keyup_action(
+        &mut self,
+        source_id: &str,
+        action: &KeyUpAction,
+        _tick_duration: u64,
+    ) {
+        let session = self.session.as_mut().unwrap();
+
+        let raw_key = action.value.chars().next().unwrap();
+        let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
+            InputSourceState::Null => unreachable!(),
+            InputSourceState::Key(key_input_state) => key_input_state,
+            InputSourceState::Pointer(_) => unreachable!(),
+        };
+
+        session.input_cancel_list.push(ActionSequence {
+            id: source_id.into(),
+            actions: ActionsType::Key {
+                actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction {
+                    value: action.value.clone(),
+                }))],
+            },
+        });
+
+        if let Some(keyboard_event) = key_input_state.dispatch_keyup(raw_key) {
+            let cmd_msg =
+                WebDriverCommandMsg::KeyboardAction(session.browsing_context_id, keyboard_event);
+            self.constellation_chan
+                .send(ConstellationMsg::WebDriverCommand(cmd_msg))
+                .unwrap();
+        }
+    }
+}

components/webdriver_server/lib.rs

@@ -13,8 +13,10 @@ extern crate serde;
 #[macro_use]
 extern crate serde_json;
 
+mod actions;
 mod capabilities;
 
+use crate::actions::InputSourceState;
 use base64;
 use capabilities::ServoCapabilities;
 use crossbeam_channel::Sender;
@@ -36,14 +38,16 @@ use serde_json::{json, Value};
 use servo_config::{prefs, prefs::PrefValue};
 use servo_url::ServoUrl;
 use std::borrow::ToOwned;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fmt;
+use std::mem;
 use std::net::{SocketAddr, SocketAddrV4};
 use std::thread;
 use std::time::Duration;
 use uuid::Uuid;
+use webdriver::actions::ActionSequence;
 use webdriver::capabilities::{Capabilities, CapabilitiesMatching};
-use webdriver::command::SwitchToWindowParameters;
+use webdriver::command::{ActionsParameters, SwitchToWindowParameters};
 use webdriver::command::{
     AddCookieParameters, GetParameters, JavascriptCommandParameters, LocatorParameters,
 };
@@ -110,7 +114,7 @@ pub fn start_server(port: u16, constellation_chan: Sender<ConstellationMsg>) {
 }
 
 /// Represents the current WebDriver session and holds relevant session state.
-struct WebDriverSession {
+pub struct WebDriverSession {
     id: Uuid,
     browsing_context_id: BrowsingContextId,
     top_level_browsing_context_id: TopLevelBrowsingContextId,
@@ -130,6 +134,13 @@ struct WebDriverSession {
     secure_tls: bool,
     strict_file_interactability: bool,
     unhandled_prompt_behavior: String,
+
+    // https://w3c.github.io/webdriver/#dfn-active-input-sources
+    active_input_sources: Vec<InputSourceState>,
+    // https://w3c.github.io/webdriver/#dfn-input-state-table
+    input_state_table: HashMap<String, InputSourceState>,
+    // https://w3c.github.io/webdriver/#dfn-input-cancel-list
+    input_cancel_list: Vec<ActionSequence>,
 }
 
 impl WebDriverSession {
@@ -150,6 +161,10 @@ impl WebDriverSession {
             secure_tls: true,
             strict_file_interactability: false,
             unhandled_prompt_behavior: "dismiss and notify".to_string(),
+
+            active_input_sources: Vec::new(),
+            input_state_table: HashMap::new(),
+            input_cancel_list: Vec::new(),
         }
     }
 }
@@ -1347,6 +1362,30 @@ impl Handler {
         }
     }
 
+    fn handle_perform_actions(
+        &mut self,
+        parameters: &ActionsParameters,
+    ) -> WebDriverResult<WebDriverResponse> {
+        self.dispatch_actions(&parameters.actions);
+
+        Ok(WebDriverResponse::Void)
+    }
+
+    fn handle_release_actions(&mut self) -> WebDriverResult<WebDriverResponse> {
+        let input_cancel_list = {
+            let session = self.session_mut()?;
+            session.input_cancel_list.reverse();
+            mem::replace(&mut session.input_cancel_list, Vec::new())
+        };
+        self.dispatch_actions(&input_cancel_list);
+
+        let session = self.session_mut()?;
+        session.input_state_table = HashMap::new();
+        session.active_input_sources = Vec::new();
+
+        Ok(WebDriverResponse::Void)
+    }
+
     fn handle_execute_script(
         &self,
         parameters: &JavascriptCommandParameters,
@@ -1628,6 +1667,8 @@ impl WebDriverHandler<ServoExtensionRoute> for Handler {
                 self.handle_element_css(element, name)
             },
             WebDriverCommand::GetPageSource => self.handle_get_page_source(),
+            WebDriverCommand::PerformActions(ref x) => self.handle_perform_actions(x),
+            WebDriverCommand::ReleaseActions => self.handle_release_actions(),
             WebDriverCommand::ExecuteScript(ref x) => self.handle_execute_script(x),
             WebDriverCommand::ExecuteAsyncScript(ref x) => self.handle_execute_async_script(x),
             WebDriverCommand::ElementSendKeys(ref element, ref keys) => {