diff --git a/components/webdriver_server/actions.rs b/components/webdriver_server/actions.rs index f33310ac001..b86ce5e8792 100644 --- a/components/webdriver_server/actions.rs +++ b/components/webdriver_server/actions.rs @@ -2,9 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; +use std::thread; use std::time::{Duration, Instant}; -use std::{cmp, thread}; use constellation_traits::EmbedderToConstellationMessage; use embedder_traits::{MouseButtonAction, WebDriverCommandMsg, WebDriverScriptCommand}; @@ -12,10 +12,11 @@ use ipc_channel::ipc; use keyboard_types::webdriver::KeyInputState; use webdriver::actions::{ ActionSequence, ActionsType, GeneralAction, KeyAction, KeyActionItem, KeyDownAction, - KeyUpAction, NullActionItem, PointerAction, PointerActionItem, PointerActionParameters, - PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, WheelAction, - WheelActionItem, WheelScrollAction, + KeyUpAction, NullActionItem, PointerAction, PointerActionItem, PointerDownAction, + PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, WheelAction, WheelActionItem, + WheelScrollAction, }; +use webdriver::command::ActionsParameters; use webdriver::error::{ErrorStatus, WebDriverError}; use crate::{Handler, WebElement, wait_for_script_response}; @@ -24,11 +25,32 @@ use crate::{Handler, WebElement, wait_for_script_response}; static POINTERMOVE_INTERVAL: u64 = 17; static WHEELSCROLL_INTERVAL: u64 = 17; -// https://w3c.github.io/webdriver/#dfn-input-source-state +// A single action, corresponding to an `action object` in the spec. +// In the spec, `action item` refers to a plain JSON object. +// However, we use the name ActionItem here +// to be consistent with type names from webdriver crate. +pub(crate) enum ActionItem { + Null(NullActionItem), + Key(KeyActionItem), + Pointer(PointerActionItem), + Wheel(WheelActionItem), +} + +// A set of actions with multiple sources executed within a single tick. +// The order in which they are performed is not guaranteed. +// The `id` is used to identify the source of the actions. +pub(crate) type TickActions = HashMap; + +// Consumed by the `dispatch_actions` method. +pub(crate) type ActionsByTick = Vec; + +/// pub(crate) enum InputSourceState { Null, + #[allow(dead_code)] Key(KeyInputState), Pointer(PointerInputState), + #[allow(dead_code)] Wheel, } @@ -44,13 +66,9 @@ pub(crate) struct PointerInputState { } impl PointerInputState { - pub fn new(subtype: &PointerType) -> PointerInputState { + pub fn new(subtype: PointerType) -> PointerInputState { PointerInputState { - subtype: match subtype { - PointerType::Mouse => PointerType::Mouse, - PointerType::Pen => PointerType::Pen, - PointerType::Touch => PointerType::Touch, - }, + subtype, pressed: HashSet::new(), x: 0.0, y: 0.0, @@ -58,48 +76,44 @@ impl PointerInputState { } } -// https://w3c.github.io/webdriver/#dfn-computing-the-tick-duration -fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 { - let mut duration = 0; - match &tick_actions.actions { - ActionsType::Null { actions } => { - for action in actions.iter() { - let NullActionItem::General(GeneralAction::Pause(pause_action)) = action; - duration = cmp::max(duration, pause_action.duration.unwrap_or(0)); +/// +fn compute_tick_duration(tick_actions: &TickActions) -> u64 { + // Step 1. Let max duration be 0. + // Step 2. For each action in tick actions: + tick_actions + .iter() + .filter_map(|(_, action_item)| { + // If action object has subtype property set to "pause" or + // action object has type property set to "pointer" and subtype property set to "pointerMove", + // or action object has type property set to "wheel" and subtype property set to "scroll", + // let duration be equal to the duration property of action object. + match action_item { + ActionItem::Null(NullActionItem::General(GeneralAction::Pause(pause_action))) | + ActionItem::Key(KeyActionItem::General(GeneralAction::Pause(pause_action))) | + ActionItem::Pointer(PointerActionItem::General(GeneralAction::Pause( + pause_action, + ))) | + ActionItem::Wheel(WheelActionItem::General(GeneralAction::Pause(pause_action))) => { + pause_action.duration + }, + ActionItem::Pointer(PointerActionItem::Pointer(PointerAction::Move(action))) => { + action.duration + }, + ActionItem::Wheel(WheelActionItem::Wheel(WheelAction::Scroll(action))) => { + action.duration + }, + _ => None, } - }, - ActionsType::Pointer { - parameters: _, - actions, - } => { - for action in actions.iter() { - let action_duration = match action { - PointerActionItem::General(GeneralAction::Pause(action)) => action.duration, - PointerActionItem::Pointer(PointerAction::Move(action)) => action.duration, - _ => None, - }; - duration = cmp::max(duration, action_duration.unwrap_or(0)); - } - }, - ActionsType::Key { actions: _ } => (), - ActionsType::Wheel { actions } => { - for action in actions.iter() { - let action_duration = match action { - WheelActionItem::General(GeneralAction::Pause(action)) => action.duration, - WheelActionItem::Wheel(WheelAction::Scroll(action)) => action.duration, - }; - duration = cmp::max(duration, action_duration.unwrap_or(0)); - } - }, - } - duration + }) + .max() + .unwrap_or(0) } impl Handler { - // https://w3c.github.io/webdriver/#dfn-dispatch-actions + /// pub(crate) fn dispatch_actions( &self, - actions_by_tick: &[ActionSequence], + actions_by_tick: ActionsByTick, ) -> Result<(), ErrorStatus> { // Step 1. Wait for an action queue token with input state. let new_token = self.id_generator.next(); @@ -116,11 +130,8 @@ impl Handler { res } - // https://w3c.github.io/webdriver/#dfn-dispatch-actions-inner - fn dispatch_actions_inner( - &self, - actions_by_tick: &[ActionSequence], - ) -> Result<(), ErrorStatus> { + /// + fn dispatch_actions_inner(&self, actions_by_tick: ActionsByTick) -> Result<(), ErrorStatus> { // Step 1. For each item tick actions in actions by tick for tick_actions in actions_by_tick.iter() { // Step 1.2. Let tick duration be the result of @@ -137,15 +148,6 @@ impl Handler { // To ensure we wait for all events to be processed, only the last event in // this tick action step holds the message id. // Whenever a new event is generated, the message id is passed to it. - // - // TO-DO: remove the first match after webdriver_id is implemented in all commands - match tick_actions.actions { - ActionsType::Key { .. } | ActionsType::Wheel { .. } | ActionsType::Null { .. } => { - return Ok(()); - }, - _ => {}, - } - match self.constellation_receiver.recv() { Ok(response) => { let current_waiting_id = self @@ -170,6 +172,86 @@ impl Handler { Ok(()) } + /// + fn dispatch_tick_actions( + &self, + tick_actions: &TickActions, + tick_duration: u64, + ) -> Result<(), ErrorStatus> { + // Step 1. For each action object in tick actions: + // Step 1.1. Let input_id be the value of the id property of action object. + for (input_id, action) in tick_actions.iter() { + // Step 6. Let subtype be action object's subtype. + // Steps 7, 8. Try to run specific algorithm based on the action type. + match action { + ActionItem::Null(NullActionItem::General(_)) => { + self.dispatch_general_action(input_id); + }, + ActionItem::Key(KeyActionItem::General(_)) => { + self.dispatch_general_action(input_id); + }, + ActionItem::Key(KeyActionItem::Key(KeyAction::Down(keydown_action))) => { + self.dispatch_keydown_action(input_id, keydown_action); + + // Step 9. If subtype is "keyDown", append a copy of action + // object with the subtype property changed to "keyUp" to + // input state's input cancel list. + self.session() + .unwrap() + .input_cancel_list + .borrow_mut() + .push(ActionItem::Key(KeyActionItem::Key(KeyAction::Up( + KeyUpAction { + value: keydown_action.value.clone(), + }, + )))); + }, + ActionItem::Key(KeyActionItem::Key(KeyAction::Up(keyup_action))) => { + self.dispatch_keyup_action(input_id, keyup_action); + }, + ActionItem::Pointer(PointerActionItem::General(_)) => { + self.dispatch_general_action(input_id); + }, + ActionItem::Pointer(PointerActionItem::Pointer(PointerAction::Down( + pointer_down_action, + ))) => { + self.dispatch_pointerdown_action(input_id, pointer_down_action); + + // Step 10. If subtype is "pointerDown", append a copy of action + // object with the subtype property changed to "pointerUp" to + // input state's input cancel list. + self.session().unwrap().input_cancel_list.borrow_mut().push( + ActionItem::Pointer(PointerActionItem::Pointer(PointerAction::Up( + PointerUpAction { + button: pointer_down_action.button, + ..Default::default() + }, + ))), + ); + }, + ActionItem::Pointer(PointerActionItem::Pointer(PointerAction::Move( + pointer_move_action, + ))) => { + self.dispatch_pointermove_action(input_id, pointer_move_action, tick_duration)?; + }, + ActionItem::Pointer(PointerActionItem::Pointer(PointerAction::Up( + pointer_up_action, + ))) => { + self.dispatch_pointerup_action(input_id, pointer_up_action); + }, + ActionItem::Wheel(WheelActionItem::General(_)) => { + self.dispatch_general_action(input_id); + }, + ActionItem::Wheel(WheelActionItem::Wheel(WheelAction::Scroll(scroll_action))) => { + self.dispatch_scroll_action(scroll_action, tick_duration)?; + }, + _ => {}, + } + } + + Ok(()) + } + fn dispatch_general_action(&self, source_id: &str) { self.session() .unwrap() @@ -181,143 +263,7 @@ impl Handler { // Nothing to be done } - // https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions - fn dispatch_tick_actions( - &self, - tick_actions: &ActionSequence, - tick_duration: u64, - ) -> Result<(), ErrorStatus> { - let source_id = &tick_actions.id; - match &tick_actions.actions { - ActionsType::Null { actions } => { - for _action in actions.iter() { - self.dispatch_general_action(source_id); - } - }, - ActionsType::Key { actions } => { - for action in actions.iter() { - match action { - KeyActionItem::General(_action) => { - self.dispatch_general_action(source_id); - }, - KeyActionItem::Key(action) => { - self.session() - .unwrap() - .input_state_table - .borrow_mut() - .entry(source_id.to_string()) - .or_insert(InputSourceState::Key(KeyInputState::new())); - match action { - KeyAction::Down(action) => { - self.dispatch_keydown_action(source_id, action); - // Step 9. If subtype is "keyDown", append a copy of action - // object with the subtype property changed to "keyUp" to - // input state's input cancel list. - self.session().unwrap().input_cancel_list.borrow_mut().push( - ActionSequence { - id: source_id.into(), - actions: ActionsType::Key { - actions: vec![KeyActionItem::Key(KeyAction::Up( - KeyUpAction { - value: action.value.clone(), - }, - ))], - }, - }, - ); - }, - KeyAction::Up(action) => { - self.dispatch_keyup_action(source_id, action) - }, - }; - }, - } - } - }, - ActionsType::Pointer { - parameters, - actions, - } => { - for action in actions.iter() { - match action { - PointerActionItem::General(_action) => { - self.dispatch_general_action(source_id); - }, - PointerActionItem::Pointer(action) => { - self.session() - .unwrap() - .input_state_table - .borrow_mut() - .entry(source_id.to_string()) - .or_insert(InputSourceState::Pointer(PointerInputState::new( - ¶meters.pointer_type, - ))); - match action { - PointerAction::Cancel => (), - PointerAction::Down(action) => { - self.dispatch_pointerdown_action(source_id, action); - - // Step 10. If subtype is "pointerDown", append a copy of action - // object with the subtype property changed to "pointerUp" to - // input state's input cancel list. - self.session().unwrap().input_cancel_list.borrow_mut().push( - ActionSequence { - id: source_id.into(), - actions: ActionsType::Pointer { - parameters: PointerActionParameters { - pointer_type: parameters.pointer_type, - }, - actions: vec![PointerActionItem::Pointer( - PointerAction::Up(PointerUpAction { - button: action.button, - ..Default::default() - }), - )], - }, - }, - ); - }, - PointerAction::Move(action) => self.dispatch_pointermove_action( - source_id, - action, - tick_duration, - )?, - PointerAction::Up(action) => { - self.dispatch_pointerup_action(source_id, action) - }, - } - }, - } - } - }, - ActionsType::Wheel { actions } => { - for action in actions.iter() { - match action { - WheelActionItem::General(_action) => { - self.dispatch_general_action(source_id) - }, - WheelActionItem::Wheel(action) => { - self.session() - .unwrap() - .input_state_table - .borrow_mut() - .entry(source_id.to_string()) - .or_insert(InputSourceState::Wheel); - match action { - WheelAction::Scroll(action) => { - self.dispatch_scroll_action(action, tick_duration)? - }, - } - }, - } - } - }, - } - - Ok(()) - } - - // https://w3c.github.io/webdriver/#dfn-dispatch-a-keydown-action + /// fn dispatch_keydown_action(&self, source_id: &str, action: &KeyDownAction) { let session = self.session().unwrap(); @@ -340,7 +286,7 @@ impl Handler { .unwrap(); } - // https://w3c.github.io/webdriver/#dfn-dispatch-a-keyup-action + /// fn dispatch_keyup_action(&self, source_id: &str, action: &KeyUpAction) { let session = self.session().unwrap(); @@ -393,7 +339,7 @@ impl Handler { .unwrap(); } - // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerup-action + /// pub(crate) fn dispatch_pointerup_action(&self, source_id: &str, action: &PointerUpAction) { let session = self.session().unwrap(); @@ -423,7 +369,7 @@ impl Handler { .unwrap(); } - // https://w3c.github.io/webdriver/#dfn-dispatch-a-pointermove-action + /// pub(crate) fn dispatch_pointermove_action( &self, source_id: &str, @@ -738,4 +684,94 @@ impl Handler { None => Err(ErrorStatus::UnknownError), } } + + /// + pub(crate) fn extract_an_action_sequence(&self, params: ActionsParameters) -> ActionsByTick { + // Step 1. Let actions be the result of getting a property named "actions" from parameters. + // Step 2 (ignored because params is already validated earlier). If actions is not a list, + // return an error with status InvalidArgument. + let actions = params.actions; + + self.actions_by_tick_from_sequence(actions) + } + + pub(crate) fn actions_by_tick_from_sequence( + &self, + actions: Vec, + ) -> ActionsByTick { + // Step 3. Let actions by tick be an empty list. + let mut actions_by_tick: ActionsByTick = Vec::new(); + + // Step 4. For each value action sequence corresponding to an indexed property in actions + for action_sequence in actions { + // Store id before moving action_sequence + let id = action_sequence.id.clone(); + // Step 4.1. Let source actions be the result of trying to process an input source action sequence + let source_actions = self.process_an_input_source_action_sequence(action_sequence); + + // Step 4.2.2. Ensure we have enough ticks to hold all actions + while actions_by_tick.len() < source_actions.len() { + actions_by_tick.push(HashMap::new()); + } + + // Step 4.2.3. + for (tick_index, action_item) in source_actions.into_iter().enumerate() { + actions_by_tick[tick_index].insert(id.clone(), action_item); + } + } + + actions_by_tick + } + + /// + pub(crate) fn process_an_input_source_action_sequence( + &self, + action_sequence: ActionSequence, + ) -> Vec { + // Step 2. Let id be the value of the id property of action sequence. + let id = action_sequence.id.clone(); + + let mut input_state_table = self.session().unwrap().input_state_table.borrow_mut(); + + match action_sequence.actions { + ActionsType::Null { + actions: null_actions, + } => { + input_state_table + .entry(id) + .or_insert(InputSourceState::Null); + null_actions.into_iter().map(ActionItem::Null).collect() + }, + ActionsType::Key { + actions: key_actions, + } => { + input_state_table + .entry(id) + .or_insert(InputSourceState::Key(KeyInputState::new())); + key_actions.into_iter().map(ActionItem::Key).collect() + }, + ActionsType::Pointer { + parameters: _, + actions: pointer_actions, + } => { + input_state_table + .entry(id) + .or_insert(InputSourceState::Pointer(PointerInputState::new( + PointerType::Mouse, + ))); + pointer_actions + .into_iter() + .map(ActionItem::Pointer) + .collect() + }, + ActionsType::Wheel { + actions: wheel_actions, + } => { + input_state_table + .entry(id) + .or_insert(InputSourceState::Wheel); + wheel_actions.into_iter().map(ActionItem::Wheel).collect() + }, + } + } } diff --git a/components/webdriver_server/lib.rs b/components/webdriver_server/lib.rs index eb3e7bf17ec..8551080d347 100644 --- a/components/webdriver_server/lib.rs +++ b/components/webdriver_server/lib.rs @@ -64,7 +64,7 @@ use webdriver::response::{ }; use webdriver::server::{self, Session, SessionTeardownKind, WebDriverHandler}; -use crate::actions::{InputSourceState, PointerInputState}; +use crate::actions::{ActionItem, InputSourceState, PointerInputState}; #[derive(Default)] pub struct WebDriverMessageIdGenerator { @@ -171,7 +171,7 @@ pub struct WebDriverSession { input_state_table: RefCell>, /// - input_cancel_list: RefCell>, + input_cancel_list: RefCell>, } impl WebDriverSession { @@ -1480,20 +1480,22 @@ impl Handler { fn handle_perform_actions( &mut self, - parameters: &ActionsParameters, + parameters: ActionsParameters, ) -> WebDriverResult { - match self.dispatch_actions(¶meters.actions) { + // Step 5. Let actions by tick be the result of trying to extract an action sequence + let actions_by_tick = self.extract_an_action_sequence(parameters); + + // Step 6. Dispatch actions + match self.dispatch_actions(actions_by_tick) { Ok(_) => Ok(WebDriverResponse::Void), Err(error) => Err(WebDriverError::new(error, "")), } } + /// fn handle_release_actions(&mut self) -> WebDriverResult { - let input_cancel_list = self.session().unwrap().input_cancel_list.borrow(); - if let Err(error) = self.dispatch_actions(&input_cancel_list) { - return Err(WebDriverError::new(error, "")); - } - + // TODO: The previous implementation of this function was different from the spec. + // Need to re-implement this to match the spec. let session = self.session()?; session.input_state_table.borrow_mut().clear(); @@ -1655,7 +1657,7 @@ impl Handler { // Step 8.1 self.session_mut()?.input_state_table.borrow_mut().insert( id.clone(), - InputSourceState::Pointer(PointerInputState::new(&PointerType::Mouse)), + InputSourceState::Pointer(PointerInputState::new(PointerType::Mouse)), ); // Step 8.7. Construct a pointer move action. @@ -1702,7 +1704,11 @@ impl Handler { }, }; - let _ = self.dispatch_actions(&[action_sequence]); + let actions_by_tick = self.actions_by_tick_from_sequence(vec![action_sequence]); + + if let Err(e) = self.dispatch_actions(actions_by_tick) { + log::error!("handle_element_click: dispatch_actions failed: {:?}", e); + } // Step 8.17 Remove an input source with input state and input id. self.session_mut()? @@ -1940,7 +1946,9 @@ impl WebDriverHandler for Handler { self.handle_element_css(element, name) }, WebDriverCommand::GetPageSource => self.handle_get_page_source(), - WebDriverCommand::PerformActions(ref x) => self.handle_perform_actions(x), + WebDriverCommand::PerformActions(actions_parameters) => { + self.handle_perform_actions(actions_parameters) + }, WebDriverCommand::ReleaseActions => self.handle_release_actions(), WebDriverCommand::ExecuteScript(ref x) => self.handle_execute_script(x), WebDriverCommand::ExecuteAsyncScript(ref x) => self.handle_execute_async_script(x),