Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-06 16:45:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

Implement pointerMove webdriver action

Browse source
This commit is contained in:
George Roman 2019-07-31 00:18:27 +03:00
parent 2a9b2fe027
commit f064883e07
9 changed files with 306 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

20
components/compositing/compositor.rs
View file

@ -522,19 +522,21 @@ impl<Window: WindowMethods + ?Sized> IOCompositor<Window> {
Msg::WebDriverMouseButtonEvent(mouse_event_type, mouse_button, x, y),
ShutdownState::NotShuttingDown,
) => {
let dppx = self.device_pixels_per_page_px();
let point = dppx.transform_point(Point2D::new(x, y));
self.on_mouse_window_event_class(match mouse_event_type {
MouseEventType::Click => {
MouseWindowEvent::Click(mouse_button, DevicePoint::new(x, y))
},
MouseEventType::MouseDown => {
MouseWindowEvent::MouseDown(mouse_button, DevicePoint::new(x, y))
},
MouseEventType::MouseUp => {
MouseWindowEvent::MouseUp(mouse_button, DevicePoint::new(x, y))
},
MouseEventType::Click => MouseWindowEvent::Click(mouse_button, point),
MouseEventType::MouseDown => MouseWindowEvent::MouseDown(mouse_button, point),
MouseEventType::MouseUp => MouseWindowEvent::MouseUp(mouse_button, point),
});
},
(Msg::WebDriverMouseMoveEvent(x, y), ShutdownState::NotShuttingDown) => {
let dppx = self.device_pixels_per_page_px();
let point = dppx.transform_point(Point2D::new(x, y));
self.on_mouse_window_move_event_class(DevicePoint::new(point.x, point.y));
},
(Msg::PendingPaintMetric(pipeline_id, epoch), _) => {
self.pending_paint_metrics.insert(pipeline_id, epoch);
},

3
components/compositing/compositor_thread.rs
View file

@ -110,6 +110,8 @@ pub enum Msg {
LoadComplete(TopLevelBrowsingContextId),
/// WebDriver mouse button event
WebDriverMouseButtonEvent(MouseEventType, MouseButton, f32, f32),
/// WebDriver mouse move event
WebDriverMouseMoveEvent(f32, f32),
/// Get Window Informations size and position.
GetClientWindow(IpcSender<(DeviceIntSize, DeviceIntPoint)>),
@ -137,6 +139,7 @@ impl Debug for Msg {
Msg::PendingPaintMetric(..) => write!(f, "PendingPaintMetric"),
Msg::LoadComplete(..) => write!(f, "LoadComplete"),
Msg::WebDriverMouseButtonEvent(..) => write!(f, "WebDriverMouseButtonEvent"),
Msg::WebDriverMouseMoveEvent(..) => write!(f, "WebDriverMouseMoveEvent"),
Msg::GetClientWindow(..) => write!(f, "GetClientWindow"),
Msg::GetScreenSize(..) => write!(f, "GetScreenSize"),
Msg::GetScreenAvailSize(..) => write!(f, "GetScreenAvailSize"),

4
components/constellation/constellation.rs
View file

@ -3528,6 +3528,10 @@ where
y,
));
},
WebDriverCommandMsg::MouseMoveAction(x, y) => {
self.compositor_proxy
.send(ToCompositorMsg::WebDriverMouseMoveEvent(x, y));
},
WebDriverCommandMsg::TakeScreenshot(_, rect, reply) => {
self.compositor_proxy
.send(ToCompositorMsg::CreatePng(rect, reply));

8
components/script/script_thread.rs
View file

@ -2209,6 +2209,14 @@ impl ScriptThread {
WebDriverScriptCommand::GetElementText(node_id, reply) => {
webdriver_handlers::handle_get_text(&*documents, pipeline_id, node_id, reply)
},
WebDriverScriptCommand::GetElementInViewCenterPoint(node_id, reply) => {
webdriver_handlers::handle_get_element_in_view_center_point(
&*documents,
pipeline_id,
node_id,
reply,
)
},
WebDriverScriptCommand::GetBrowsingContextId(webdriver_frame_id, reply) => {
webdriver_handlers::handle_get_browsing_context_id(
&*documents,

51
components/script/webdriver_handlers.rs
View file

@ -53,6 +53,7 @@ use script_traits::webdriver_msg::{
WebDriverFrameId, WebDriverJSError, WebDriverJSResult, WebDriverJSValue,
};
use servo_url::ServoUrl;
use std::cmp;
use std::collections::HashMap;
use std::ffi::CString;
use webdriver::common::{WebElement, WebFrame, WebWindow};
@ -360,6 +361,56 @@ pub fn handle_get_browsing_context_id(
.unwrap();
}
// https://w3c.github.io/webdriver/#dfn-center-point
fn get_element_in_view_center_point(element: &Element) -> Option<Point2D<i64>> {
element
.GetClientRects()
.iter()
// Step 1
.next()
.map(|rectangle| {
let x = rectangle.X().round() as i64;
let y = rectangle.Y().round() as i64;
let width = rectangle.Width().round() as i64;
let height = rectangle.Height().round() as i64;
let window = window_from_node(element.upcast::<Node>());
let document = window.Document();
let document_element = document.upcast::<Node>().downcast::<Element>().unwrap();
let clientWidth = document_element.ClientWidth() as i64;
let clientHeight = document_element.ClientHeight() as i64;
// Steps 2 - 5
let left = cmp::max(0, cmp::min(x, x + width));
let right = cmp::min(clientWidth, cmp::max(x, x + width));
let top = cmp::max(0, cmp::min(y, y + height));
let bottom = cmp::min(clientHeight, cmp::max(y, y + height));
// Steps 6 - 7
let x = (left + right) / 2;
let y = (top + bottom) / 2;
// Step 8
Point2D::new(x, y)
})
}
pub fn handle_get_element_in_view_center_point(
documents: &Documents,
pipeline: PipelineId,
element_id: String,
reply: IpcSender<Result<Option<(i64, i64)>, ErrorStatus>>,
) {
reply
.send(
find_node_by_unique_id(documents, pipeline, element_id).map(|node| {
get_element_in_view_center_point(node.downcast::<Element>().unwrap())
.map(|point| (point.x, point.y))
}),
)
.unwrap();
}
pub fn handle_find_element_css(
documents: &Documents,
pipeline: PipelineId,

2
components/script_traits/lib.rs
View file

@ -797,6 +797,8 @@ pub enum WebDriverCommandMsg {
KeyboardAction(BrowsingContextId, KeyboardEvent),
/// Act as if the mouse was clicked in the browsing context with the given ID.
MouseButtonAction(MouseEventType, MouseButton, f32, f32),
/// Act as if the mouse was moved in the browsing context with the given ID.
MouseMoveAction(f32, f32),
/// Set the window size.
SetWindowSize(
TopLevelBrowsingContextId,

1
components/script_traits/webdriver_msg.rs
View file

@ -75,6 +75,7 @@ pub enum WebDriverScriptCommand {
GetElementRect(String, IpcSender<Result<Rect<f64>, ErrorStatus>>),
GetElementTagName(String, IpcSender<Result<String, ErrorStatus>>),
GetElementText(String, IpcSender<Result<String, ErrorStatus>>),
GetElementInViewCenterPoint(String, IpcSender<Result<Option<(i64, i64)>, ErrorStatus>>),
GetBoundingClientRect(String, IpcSender<Result<Rect<f32>, ErrorStatus>>),
GetBrowsingContextId(
WebDriverFrameId,

233
components/webdriver_server/actions.rs
View file

@ -3,30 +3,40 @@
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::Handler;
use crossbeam_channel::Sender;
use ipc_channel::ipc;
use keyboard_types::webdriver::KeyInputState;
use script_traits::webdriver_msg::WebDriverScriptCommand;
use script_traits::{ConstellationMsg, MouseButton, MouseEventType, WebDriverCommandMsg};
use std::cmp;
use std::collections::HashSet;
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
use webdriver::actions::{ActionSequence, ActionsType, GeneralAction, NullActionItem};
use webdriver::actions::{KeyAction, KeyActionItem, KeyDownAction, KeyUpAction};
use webdriver::actions::{
PointerAction, PointerActionItem, PointerActionParameters, PointerDownAction,
};
use webdriver::actions::{PointerType, PointerUpAction};
use webdriver::actions::{PointerMoveAction, PointerOrigin, PointerType, PointerUpAction};
use webdriver::error::ErrorStatus;
// Interval between pointerMove increments in ms, based on common vsync
static POINTERMOVE_INTERVAL: u64 = 17;
// https://w3c.github.io/webdriver/#dfn-input-source-state
pub(crate) enum InputSourceState {
Null,
Key(KeyInputState),
Pointer(PointerInputState),
Pointer(Arc<Mutex<PointerInputState>>),
}
// https://w3c.github.io/webdriver/#dfn-pointer-input-source
pub(crate) struct PointerInputState {
subtype: PointerType,
pressed: HashSet<u64>,
x: u64,
y: u64,
x: i64,
y: i64,
}
impl PointerInputState {
@ -72,6 +82,73 @@ fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 {
duration
}
// https://w3c.github.io/webdriver/#dfn-perform-a-pointer-move
fn perform_pointer_move(
constellation_chan: Sender<ConstellationMsg>,
pointer_input_state: Arc<Mutex<PointerInputState>>,
duration: u64,
start_x: i64,
start_y: i64,
target_x: i64,
target_y: i64,
tick_start: Instant,
) {
let mut pointer_input_state = pointer_input_state.lock().unwrap();
loop {
// Step 1
let time_delta = tick_start.elapsed().as_millis();
// Step 2
let duration_ratio = if duration > 0 {
time_delta as f64 / duration as f64
} else {
1.0
};
// Step 3
let last = if 1.0 - duration_ratio < 0.001 {
true
} else {
false
};
// Step 4
let (x, y) = if last {
(target_x, target_y)
} else {
(
(duration_ratio * (target_x - start_x) as f64) as i64 + start_x,
(duration_ratio * (target_y - start_y) as f64) as i64 + start_y,
)
};
// Steps 5 - 6
let current_x = pointer_input_state.x;
let current_y = pointer_input_state.y;
// Step 7
if x != current_x || y != current_y {
// Step 7.2
let cmd_msg = WebDriverCommandMsg::MouseMoveAction(x as f32, y as f32);
constellation_chan
.send(ConstellationMsg::WebDriverCommand(cmd_msg))
.unwrap();
// Step 7.3
pointer_input_state.x = x;
pointer_input_state.y = y;
}
// Step 8
if last {
return;
}
// Step 9
thread::sleep(Duration::from_millis(POINTERMOVE_INTERVAL));
}
}
fn u64_to_mouse_button(button: u64) -> Option<MouseButton> {
if MouseButton::Left as u64 == button {
Some(MouseButton::Left)
@ -86,11 +163,15 @@ fn u64_to_mouse_button(button: u64) -> Option<MouseButton> {
impl Handler {
// https://w3c.github.io/webdriver/#dfn-dispatch-actions
pub(crate) fn dispatch_actions(&mut self, actions_by_tick: &[ActionSequence]) {
pub(crate) fn dispatch_actions(
&mut self,
actions_by_tick: &[ActionSequence],
) -> Result<(), ErrorStatus> {
for tick_actions in actions_by_tick.iter() {
let tick_duration = compute_tick_duration(&tick_actions);
self.dispatch_tick_actions(&tick_actions, tick_duration);
self.dispatch_tick_actions(&tick_actions, tick_duration)?;
}
Ok(())
}
fn dispatch_general_action(&mut self, source_id: &str) {
@ -104,7 +185,11 @@ impl Handler {
}
// https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions
fn dispatch_tick_actions(&mut self, tick_actions: &ActionSequence, _tick_duration: u64) {
fn dispatch_tick_actions(
&mut self,
tick_actions: &ActionSequence,
tick_duration: u64,
) -> Result<(), ErrorStatus> {
let source_id = &tick_actions.id;
match &tick_actions.actions {
ActionsType::Null { actions } => {
@ -150,15 +235,19 @@ impl Handler {
.unwrap()
.input_state_table
.entry(source_id.to_string())
.or_insert(InputSourceState::Pointer(PointerInputState::new(
&parameters.pointer_type,
)));
.or_insert(InputSourceState::Pointer(Arc::new(Mutex::new(
PointerInputState::new(&parameters.pointer_type),
))));
match action {
PointerAction::Cancel => (),
PointerAction::Down(action) => {
self.dispatch_pointerdown_action(&source_id, &action)
},
PointerAction::Move(_action) => (),
PointerAction::Move(action) => self.dispatch_pointermove_action(
&source_id,
&action,
tick_duration,
)?,
PointerAction::Up(action) => {
self.dispatch_pointerup_action(&source_id, &action)
},
@ -168,6 +257,8 @@ impl Handler {
}
},
}
Ok(())
}
// https://w3c.github.io/webdriver/#dfn-dispatch-a-keydown-action
@ -231,10 +322,10 @@ impl Handler {
fn dispatch_pointerdown_action(&mut self, source_id: &str, action: &PointerDownAction) {
let session = self.session.as_mut().unwrap();
let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
let mut pointer_input_state = match session.input_state_table.get(source_id).unwrap() {
InputSourceState::Null => unreachable!(),
InputSourceState::Key(_) => unreachable!(),
InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
InputSourceState::Pointer(pointer_input_state) => pointer_input_state.lock().unwrap(),
};
if pointer_input_state.pressed.contains(&action.button) {
@ -277,10 +368,10 @@ impl Handler {
fn dispatch_pointerup_action(&mut self, source_id: &str, action: &PointerUpAction) {
let session = self.session.as_mut().unwrap();
let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
let mut pointer_input_state = match session.input_state_table.get(source_id).unwrap() {
InputSourceState::Null => unreachable!(),
InputSourceState::Key(_) => unreachable!(),
InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
InputSourceState::Pointer(pointer_input_state) => pointer_input_state.lock().unwrap(),
};
if !pointer_input_state.pressed.contains(&action.button) {
@ -318,4 +409,116 @@ impl Handler {
.unwrap();
}
}
// https://w3c.github.io/webdriver/#dfn-dispatch-a-pointermove-action
fn dispatch_pointermove_action(
&mut self,
source_id: &str,
action: &PointerMoveAction,
tick_duration: u64,
) -> Result<(), ErrorStatus> {
let tick_start = Instant::now();
// Steps 1 - 2
let x_offset = action.x.unwrap_or(0);
let y_offset = action.y.unwrap_or(0);
// Steps 3 - 4
let (start_x, start_y) = match self
.session
.as_ref()
.unwrap()
.input_state_table
.get(source_id)
.unwrap()
{
InputSourceState::Null => unreachable!(),
InputSourceState::Key(_) => unreachable!(),
InputSourceState::Pointer(pointer_input_state) => {
let pointer_input_state = pointer_input_state.lock().unwrap();
(pointer_input_state.x, pointer_input_state.y)
},
};
// Step 5 - 6
let (x, y) = match action.origin {
PointerOrigin::Viewport => (x_offset, y_offset),
PointerOrigin::Pointer => (start_x + x_offset, start_y + y_offset),
PointerOrigin::Element(ref x) => {
let (sender, receiver) = ipc::channel().unwrap();
self.top_level_script_command(WebDriverScriptCommand::GetElementInViewCenterPoint(
x.to_string(),
sender,
))
.unwrap();
match receiver.recv().unwrap() {
Ok(point) => match point {
Some(point) => point,
None => return Err(ErrorStatus::UnknownError),
},
Err(_) => return Err(ErrorStatus::UnknownError),
}
},
};
let (sender, receiver) = ipc::channel().unwrap();
let cmd_msg = WebDriverCommandMsg::GetWindowSize(
self.session.as_ref().unwrap().top_level_browsing_context_id,
sender,
);
self.constellation_chan
.send(ConstellationMsg::WebDriverCommand(cmd_msg))
.unwrap();
// Steps 7 - 8
let viewport = receiver.recv().unwrap().initial_viewport;
if x < 0 || x as f32 > viewport.width || y < 0 || y as f32 > viewport.height {
return Err(ErrorStatus::MoveTargetOutOfBounds);
}
// Step 9
let duration = match action.duration {
Some(duration) => duration,
None => tick_duration,
};
// Step 10
if duration > 0 {
thread::sleep(Duration::from_millis(POINTERMOVE_INTERVAL));
}
let pointer_input_state = match self
.session
.as_ref()
.unwrap()
.input_state_table
.get(source_id)
.unwrap()
{
InputSourceState::Null => unreachable!(),
InputSourceState::Key(_) => unreachable!(),
InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
};
let constellation_chan = self.constellation_chan.clone();
let pointer_input_state = pointer_input_state.clone();
// Step 11
thread::spawn(move || {
perform_pointer_move(
constellation_chan,
pointer_input_state,
duration,
start_x,
start_y,
x,
y,
tick_start,
);
});
// Step 12
Ok(())
}
}

12
components/webdriver_server/lib.rs
View file

@ -1345,9 +1345,10 @@ impl Handler {
&mut self,
parameters: &ActionsParameters,
) -> WebDriverResult<WebDriverResponse> {
self.dispatch_actions(&parameters.actions);
Ok(WebDriverResponse::Void)
match self.dispatch_actions(&parameters.actions) {
Ok(_) => Ok(WebDriverResponse::Void),
Err(error) => Err(WebDriverError::new(error, "")),
}
}
fn handle_release_actions(&mut self) -> WebDriverResult<WebDriverResponse> {
@ -1356,7 +1357,10 @@ impl Handler {
session.input_cancel_list.reverse();
mem::replace(&mut session.input_cancel_list, Vec::new())
};
self.dispatch_actions(&input_cancel_list);
if let Err(error) = self.dispatch_actions(&input_cancel_list) {
return Err(WebDriverError::new(error, ""));
}
let session = self.session_mut()?;
session.input_state_table = HashMap::new();