Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-06 16:45:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

Synchronize dispatch_actions in WebDriver (#36932)

Browse source 
Implement missing synchronization in `dispatch_actions` of `WebDriver`.
https://w3c.github.io/webdriver/#dispatching-actions

> The user agent event loop has spun enough times to process the DOM
events generated by the last invocation of the >[dispatch tick
actions](https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions)
steps.

- Add a way for `ScriptThread` to notify `WebDriver` about the
completion of input commands.
- Add a `webdriver_id` field for `InputEvent`. `ScriptThread` uses it to
distinguish WebDriver events and sends notification.

Tests:
`./mach test-wpt --product servodriver -r
tests\wpt\tests\webdriver\tests\classic\element_click\events.py` pass if
`hit_testing` pass. Check
[issue](https://github.com/servo/servo/issues/36676#issuecomment-2882917136)

cc: @xiaochengh

---------

Signed-off-by: batu_hoang <longvatrong111@gmail.com>
Signed-off-by: Martin Robinson <mrobinson@igalia.com>
Co-authored-by: Martin Robinson <mrobinson@igalia.com>
This commit is contained in:
batu_hoang 2025-05-21 19:03:04 +08:00 • committed by GitHub
parent 3a527d784b
commit f52fa9b672
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 471 additions and 140 deletions
Download patch file Download diff file Expand all files Collapse all files

38
components/compositing/compositor.rs
View file

@ -620,29 +620,37 @@ impl IOCompositor {
}
},
CompositorMsg::WebDriverMouseButtonEvent(webview_id, action, button, x, y) => {
let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else {
warn!("Handling input event for unknown webview: {webview_id}");
return;
};
let dppx = webview_renderer.device_pixels_per_page_pixel();
let point = dppx.transform_point(Point2D::new(x, y));
webview_renderer.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent {
point,
CompositorMsg::WebDriverMouseButtonEvent(
webview_id,
action,
button,
}));
},
CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y) => {
x,
y,
message_id,
) => {
let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else {
warn!("Handling input event for unknown webview: {webview_id}");
return;
};
let dppx = webview_renderer.device_pixels_per_page_pixel();
let point = dppx.transform_point(Point2D::new(x, y));
webview_renderer
.dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point }));
webview_renderer.dispatch_input_event(
InputEvent::MouseButton(MouseButtonEvent::new(action, button, point))
.with_webdriver_message_id(Some(message_id)),
);
},
CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y, message_id) => {
let Some(webview_renderer) = self.webview_renderers.get_mut(webview_id) else {
warn!("Handling input event for unknown webview: {webview_id}");
return;
};
let dppx = webview_renderer.device_pixels_per_page_pixel();
let point = dppx.transform_point(Point2D::new(x, y));
webview_renderer.dispatch_input_event(
InputEvent::MouseMove(MouseMoveEvent::new(point))
.with_webdriver_message_id(Some(message_id)),
);
},
CompositorMsg::WebDriverWheelScrollEvent(webview_id, x, y, delta_x, delta_y) => {

14
components/compositing/webview_renderer.rs
View file

@ -687,17 +687,17 @@ impl WebViewRenderer {
/// <http://w3c.github.io/touch-events/#mouse-events>
fn simulate_mouse_click(&mut self, point: DevicePoint) {
let button = MouseButton::Left;
self.dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent { point }));
self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent {
self.dispatch_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));
self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Down,
button,
action: MouseButtonAction::Down,
point,
}));
self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent {
)));
self.dispatch_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Up,
button,
action: MouseButtonAction::Up,
point,
}));
)));
}
pub(crate) fn notify_scroll_event(

30
components/constellation/constellation.rs
View file

@ -132,7 +132,7 @@ use embedder_traits::{
FocusSequenceNumber, ImeEvent, InputEvent, JSValue, JavaScriptEvaluationError,
JavaScriptEvaluationId, MediaSessionActionType, MediaSessionEvent, MediaSessionPlaybackState,
MouseButton, MouseButtonAction, MouseButtonEvent, Theme, ViewportDetails, WebDriverCommandMsg,
WebDriverLoadStatus,
WebDriverCommandResponse, WebDriverLoadStatus,
};
use euclid::Size2D;
use euclid::default::Size2D as UntypedSize2D;
@ -532,6 +532,8 @@ pub struct InitialConstellationState {
struct WebDriverData {
load_channel: Option<(PipelineId, IpcSender<WebDriverLoadStatus>)>,
resize_channel: Option<IpcSender<Size2D<f32, CSSPixel>>>,
// Forward responses from the script thread to the webdriver server.
input_command_response_sender: Option<IpcSender<WebDriverCommandResponse>>,
}
impl WebDriverData {
@ -539,6 +541,7 @@ impl WebDriverData {
WebDriverData {
load_channel: None,
resize_channel: None,
input_command_response_sender: None,
}
}
}
@ -1867,6 +1870,18 @@ where
ScriptToConstellationMessage::FinishJavaScriptEvaluation(evaluation_id, result) => {
self.handle_finish_javascript_evaluation(evaluation_id, result)
},
ScriptToConstellationMessage::WebDriverInputComplete(msg_id) => {
if let Some(ref reply_sender) = self.webdriver.input_command_response_sender {
reply_sender
.send(WebDriverCommandResponse { id: msg_id })
.unwrap_or_else(|_| {
warn!("Failed to send WebDriverInputComplete {:?}", msg_id);
self.webdriver.input_command_response_sender = None;
});
} else {
warn!("No WebDriver input_command_response_sender");
}
},
}
}
@ -4836,7 +4851,11 @@ where
mouse_button,
x,
y,
msg_id,
response_sender,
) => {
self.webdriver.input_command_response_sender = Some(response_sender);
self.compositor_proxy
.send(CompositorMsg::WebDriverMouseButtonEvent(
webview_id,
@ -4844,11 +4863,16 @@ where
mouse_button,
x,
y,
msg_id,
));
},
WebDriverCommandMsg::MouseMoveAction(webview_id, x, y) => {
WebDriverCommandMsg::MouseMoveAction(webview_id, x, y, msg_id, response_sender) => {
self.webdriver.input_command_response_sender = Some(response_sender);
self.compositor_proxy
.send(CompositorMsg::WebDriverMouseMoveEvent(webview_id, x, y));
.send(CompositorMsg::WebDriverMouseMoveEvent(
webview_id, x, y, msg_id,
));
},
WebDriverCommandMsg::WheelScrollAction(webview, x, y, delta_x, delta_y) => {
self.compositor_proxy

1
components/constellation/tracing.rs
View file

@ -177,6 +177,7 @@ mod from_script {
Self::TitleChanged(..) => target!("TitleChanged"),
Self::IFrameSizes(..) => target!("IFrameSizes"),
Self::ReportMemory(..) => target!("ReportMemory"),
Self::WebDriverInputComplete(..) => target!("WebDriverInputComplete"),
Self::FinishJavaScriptEvaluation(..) => target!("FinishJavaScriptEvaluation"),
}
}

38
components/script/script_thread.rs
View file

@ -1081,6 +1081,10 @@ impl ScriptThread {
for event in document.take_pending_input_events().into_iter() {
document.update_active_keyboard_modifiers(event.active_keyboard_modifiers);
// We do this now, because the event is consumed below, but the order doesn't really
// matter as the event will be handled before any new ScriptThread messages are processed.
self.notify_webdriver_input_event_completed(pipeline_id, &event.event);
match event.event {
InputEvent::MouseButton(mouse_button_event) => {
document.handle_mouse_button_event(
@ -1144,6 +1148,19 @@ impl ScriptThread {
ScriptThread::set_user_interacting(false);
}
fn notify_webdriver_input_event_completed(&self, pipeline_id: PipelineId, event: &InputEvent) {
let Some(id) = event.webdriver_message_id() else {
return;
};
if let Err(error) = self.senders.pipeline_to_constellation_sender.send((
pipeline_id,
ScriptToConstellationMessage::WebDriverInputComplete(id),
)) {
warn!("ScriptThread failed to send WebDriverInputComplete {id:?}: {error:?}",);
}
}
/// <https://html.spec.whatwg.org/multipage/#update-the-rendering>
///
/// Attempt to update the rendering and then do a microtask checkpoint if rendering was actually
@ -3420,7 +3437,7 @@ impl ScriptThread {
// the pointer, when the user presses down and releases the primary pointer button"
// Servo-specific: Trigger if within 10px of the down point
if let InputEvent::MouseButton(mouse_button_event) = event.event {
if let InputEvent::MouseButton(mouse_button_event) = &event.event {
if let MouseButton::Left = mouse_button_event.button {
match mouse_button_event.action {
MouseButtonAction::Up => {
@ -3429,16 +3446,23 @@ impl ScriptThread {
let pixel_dist =
(pixel_dist.x * pixel_dist.x + pixel_dist.y * pixel_dist.y).sqrt();
if pixel_dist < 10.0 * document.window().device_pixel_ratio().get() {
document.note_pending_input_event(event.clone());
// Pass webdriver_id to the newly generated click event
document.note_pending_input_event(ConstellationInputEvent {
hit_test_result: event.hit_test_result.clone(),
pressed_mouse_buttons: event.pressed_mouse_buttons,
active_keyboard_modifiers: event.active_keyboard_modifiers,
event: event.event.clone().with_webdriver_message_id(None),
});
document.note_pending_input_event(ConstellationInputEvent {
hit_test_result: event.hit_test_result,
pressed_mouse_buttons: event.pressed_mouse_buttons,
active_keyboard_modifiers: event.active_keyboard_modifiers,
event: InputEvent::MouseButton(MouseButtonEvent {
action: MouseButtonAction::Click,
button: mouse_button_event.button,
point: mouse_button_event.point,
}),
event: InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Click,
mouse_button_event.button,
mouse_button_event.point,
))
.with_webdriver_message_id(event.event.webdriver_message_id()),
});
return;
}

12
components/shared/compositing/lib.rs
View file

@ -10,6 +10,7 @@ use base::id::{PipelineId, WebViewId};
use crossbeam_channel::Sender;
use embedder_traits::{
AnimationState, EventLoopWaker, MouseButton, MouseButtonAction, TouchEventResult,
WebDriverMessageId,
};
use euclid::Rect;
use ipc_channel::ipc::IpcSender;
@ -101,9 +102,16 @@ pub enum CompositorMsg {
/// The load of a page has completed
LoadComplete(WebViewId),
/// WebDriver mouse button event
WebDriverMouseButtonEvent(WebViewId, MouseButtonAction, MouseButton, f32, f32),
WebDriverMouseButtonEvent(
WebViewId,
MouseButtonAction,
MouseButton,
f32,
f32,
WebDriverMessageId,
),
/// WebDriver mouse move event
WebDriverMouseMoveEvent(WebViewId, f32, f32),
WebDriverMouseMoveEvent(WebViewId, f32, f32, WebDriverMessageId),
// Webdriver wheel scroll event
WebDriverWheelScrollEvent(WebViewId, f32, f32, f64, f64),

3
components/shared/constellation/from_script_message.rs
View file

@ -17,6 +17,7 @@ use devtools_traits::{DevtoolScriptControlMsg, ScriptToDevtoolsControlMsg, Worke
use embedder_traits::{
AnimationState, EmbedderMsg, FocusSequenceNumber, JSValue, JavaScriptEvaluationError,
JavaScriptEvaluationId, MediaSessionEvent, TouchEventResult, ViewportDetails,
WebDriverMessageId,
};
use euclid::default::Size2D as UntypedSize2D;
use http::{HeaderMap, Method};
@ -652,6 +653,8 @@ pub enum ScriptToConstellationMessage {
JavaScriptEvaluationId,
Result<JSValue, JavaScriptEvaluationError>,
),
/// Notify the completion of a webdriver command.
WebDriverInputComplete(WebDriverMessageId),
}
impl fmt::Debug for ScriptToConstellationMessage {

56
components/shared/embedder/input_events.rs
View file

@ -8,6 +8,8 @@ use malloc_size_of_derive::MallocSizeOf;
use serde::{Deserialize, Serialize};
use webrender_api::units::DevicePoint;
use crate::WebDriverMessageId;
/// An input event that is sent from the embedder to Servo.
#[derive(Clone, Debug, Deserialize, Serialize)]
pub enum InputEvent {
@ -42,6 +44,38 @@ impl InputEvent {
InputEvent::Wheel(event) => Some(event.point),
}
}
pub fn webdriver_message_id(&self) -> Option<WebDriverMessageId> {
match self {
InputEvent::EditingAction(..) => None,
InputEvent::Gamepad(..) => None,
InputEvent::Ime(..) => None,
InputEvent::Keyboard(..) => None,
InputEvent::MouseButton(event) => event.webdriver_id,
InputEvent::MouseMove(event) => event.webdriver_id,
InputEvent::Touch(..) => None,
InputEvent::Wheel(..) => None,
}
}
pub fn with_webdriver_message_id(self, webdriver_id: Option<WebDriverMessageId>) -> Self {
match self {
InputEvent::EditingAction(..) => {},
InputEvent::Gamepad(..) => {},
InputEvent::Ime(..) => {},
InputEvent::Keyboard(..) => {},
InputEvent::MouseButton(mut event) => {
event.webdriver_id = webdriver_id;
},
InputEvent::MouseMove(mut event) => {
event.webdriver_id = webdriver_id;
},
InputEvent::Touch(..) => {},
InputEvent::Wheel(..) => {},
};
self
}
}
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
@ -49,6 +83,18 @@ pub struct MouseButtonEvent {
pub action: MouseButtonAction,
pub button: MouseButton,
pub point: DevicePoint,
webdriver_id: Option<WebDriverMessageId>,
}
impl MouseButtonEvent {
pub fn new(action: MouseButtonAction, button: MouseButton, point: DevicePoint) -> Self {
Self {
action,
button,
point,
webdriver_id: None,
}
}
}
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
@ -102,6 +148,16 @@ pub enum MouseButtonAction {
#[derive(Clone, Copy, Debug, Deserialize, Serialize)]
pub struct MouseMoveEvent {
pub point: DevicePoint,
webdriver_id: Option<WebDriverMessageId>,
}
impl MouseMoveEvent {
pub fn new(point: DevicePoint) -> Self {
Self {
point,
webdriver_id: None,
}
}
}
/// The type of input represented by a multi-touch event.

26
components/shared/embedder/webdriver.rs
View file

@ -24,6 +24,9 @@ use webrender_api::units::DeviceIntSize;
use crate::{MouseButton, MouseButtonAction};
#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)]
pub struct WebDriverMessageId(pub usize);
/// Messages to the constellation originating from the WebDriver server.
#[derive(Debug, Deserialize, Serialize)]
pub enum WebDriverCommandMsg {
@ -41,9 +44,23 @@ pub enum WebDriverCommandMsg {
/// Act as if keys were pressed or release in the browsing context with the given ID.
KeyboardAction(BrowsingContextId, KeyboardEvent),
/// Act as if the mouse was clicked in the browsing context with the given ID.
MouseButtonAction(WebViewId, MouseButtonAction, MouseButton, f32, f32),
MouseButtonAction(
WebViewId,
MouseButtonAction,
MouseButton,
f32,
f32,
WebDriverMessageId,
IpcSender<WebDriverCommandResponse>,
),
/// Act as if the mouse was moved in the browsing context with the given ID.
MouseMoveAction(WebViewId, f32, f32),
MouseMoveAction(
WebViewId,
f32,
f32,
WebDriverMessageId,
IpcSender<WebDriverCommandResponse>,
),
/// Act as if the mouse wheel is scrolled in the browsing context given the given ID.
WheelScrollAction(WebViewId, f32, f32, f64, f64),
/// Set the window size.
@ -188,6 +205,11 @@ pub enum WebDriverFrameId {
Parent,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct WebDriverCommandResponse {
pub id: WebDriverMessageId,
}
#[derive(Debug, Deserialize, Serialize)]
pub enum WebDriverLoadStatus {
Complete,

260
components/webdriver_server/actions.rs
View file

@ -98,20 +98,83 @@ fn compute_tick_duration(tick_actions: &ActionSequence) -> u64 {
impl Handler {
// https://w3c.github.io/webdriver/#dfn-dispatch-actions
pub(crate) fn dispatch_actions(
&mut self,
&self,
actions_by_tick: &[ActionSequence],
) -> Result<(), ErrorStatus> {
for tick_actions in actions_by_tick.iter() {
let tick_duration = compute_tick_duration(tick_actions);
self.dispatch_tick_actions(tick_actions, tick_duration)?;
// Step 1. Wait for an action queue token with input state.
let new_token = self.id_generator.next();
assert!(self.current_action_id.get().is_none());
self.current_action_id.set(Some(new_token));
// Step 2. Let actions result be the result of dispatch actions inner.
let res = self.dispatch_actions_inner(actions_by_tick);
// Step 3. Dequeue input state's actions queue.
self.current_action_id.set(None);
// Step 4. Return actions result.
res
}
// https://w3c.github.io/webdriver/#dfn-dispatch-actions-inner
fn dispatch_actions_inner(
&self,
actions_by_tick: &[ActionSequence],
) -> Result<(), ErrorStatus> {
// Step 1. For each item tick actions in actions by tick
for tick_actions in actions_by_tick.iter() {
// Step 1.2. Let tick duration be the result of
// computing the tick duration with argument tick actions.
let tick_duration = compute_tick_duration(tick_actions);
// Step 1.3. Try to dispatch tick actions
self.dispatch_tick_actions(tick_actions, tick_duration)?;
// Step 1.4. Wait for
// The user agent event loop has spun enough times to process the DOM events
// generated by the last invocation of the dispatch tick actions steps.
//
// To ensure we wait for all events to be processed, only the last event in
// this tick action step holds the message id.
// Whenever a new event is generated, the message id is passed to it.
//
// TO-DO: remove the first match after webdriver_id is implemented in all commands
match tick_actions.actions {
ActionsType::Key { .. } | ActionsType::Wheel { .. } | ActionsType::Null { .. } => {
return Ok(());
},
_ => {},
}
match self.constellation_receiver.recv() {
Ok(response) => {
let current_waiting_id = self
.current_action_id
.get()
.expect("Current id should be set before dispat_actions_inner is called");
if current_waiting_id != response.id {
dbg!("Dispatch actions completed with wrong id in response");
return Err(ErrorStatus::UnknownError);
}
},
Err(error) => {
dbg!("Dispatch actions completed with IPC error: {:?}", error);
return Err(ErrorStatus::UnknownError);
},
};
}
// Step 2. Return success with data null.
dbg!("Dispatch actions completed successfully");
Ok(())
}
fn dispatch_general_action(&mut self, source_id: &str) {
self.session_mut()
fn dispatch_general_action(&self, source_id: &str) {
self.session()
.unwrap()
.input_state_table
.borrow_mut()
.entry(source_id.to_string())
.or_insert(InputSourceState::Null);
// https://w3c.github.io/webdriver/#dfn-dispatch-a-pause-action
@ -120,7 +183,7 @@ impl Handler {
// https://w3c.github.io/webdriver/#dfn-dispatch-tick-actions
fn dispatch_tick_actions(
&mut self,
&self,
tick_actions: &ActionSequence,
tick_duration: u64,
) -> Result<(), ErrorStatus> {
@ -138,9 +201,10 @@ impl Handler {
self.dispatch_general_action(source_id);
},
KeyActionItem::Key(action) => {
self.session_mut()
self.session()
.unwrap()
.input_state_table
.borrow_mut()
.entry(source_id.to_string())
.or_insert(InputSourceState::Key(KeyInputState::new()));
match action {
@ -149,7 +213,7 @@ impl Handler {
// Step 9. If subtype is "keyDown", append a copy of action
// object with the subtype property changed to "keyUp" to
// input state's input cancel list.
self.session_mut().unwrap().input_cancel_list.push(
self.session().unwrap().input_cancel_list.borrow_mut().push(
ActionSequence {
id: source_id.into(),
actions: ActionsType::Key {
@ -180,9 +244,10 @@ impl Handler {
self.dispatch_general_action(source_id);
},
PointerActionItem::Pointer(action) => {
self.session_mut()
self.session()
.unwrap()
.input_state_table
.borrow_mut()
.entry(source_id.to_string())
.or_insert(InputSourceState::Pointer(PointerInputState::new(
&parameters.pointer_type,
@ -195,7 +260,7 @@ impl Handler {
// Step 10. If subtype is "pointerDown", append a copy of action
// object with the subtype property changed to "pointerUp" to
// input state's input cancel list.
self.session_mut().unwrap().input_cancel_list.push(
self.session().unwrap().input_cancel_list.borrow_mut().push(
ActionSequence {
id: source_id.into(),
actions: ActionsType::Pointer {
@ -232,9 +297,10 @@ impl Handler {
self.dispatch_general_action(source_id)
},
WheelActionItem::Wheel(action) => {
self.session_mut()
self.session()
.unwrap()
.input_state_table
.borrow_mut()
.entry(source_id.to_string())
.or_insert(InputSourceState::Wheel);
match action {
@ -252,12 +318,25 @@ impl Handler {
}
// https://w3c.github.io/webdriver/#dfn-dispatch-a-keydown-action
fn dispatch_keydown_action(&mut self, source_id: &str, action: &KeyDownAction) {
// Step 1
let raw_key = action.value.chars().next().unwrap();
let key_input_state = self.get_key_input_state_mut(source_id);
fn dispatch_keydown_action(&self, source_id: &str, action: &KeyDownAction) {
let session = self.session().unwrap();
let raw_key = action.value.chars().next().unwrap();
let mut input_state_table = session.input_state_table.borrow_mut();
let key_input_state = match input_state_table.get_mut(source_id).unwrap() {
InputSourceState::Key(key_input_state) => key_input_state,
_ => unreachable!(),
};
session.input_cancel_list.borrow_mut().push(ActionSequence {
id: source_id.into(),
actions: ActionsType::Key {
actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction {
value: action.value.clone(),
}))],
},
});
// Step 2 - 11. Done by `keyboard-types` crate.
let keyboard_event = key_input_state.dispatch_keydown(raw_key);
// Step 12
@ -271,12 +350,25 @@ impl Handler {
}
// https://w3c.github.io/webdriver/#dfn-dispatch-a-keyup-action
fn dispatch_keyup_action(&mut self, source_id: &str, action: &KeyUpAction) {
// Step 1
let raw_key = action.value.chars().next().unwrap();
let key_input_state = self.get_key_input_state_mut(source_id);
fn dispatch_keyup_action(&self, source_id: &str, action: &KeyUpAction) {
let session = self.session().unwrap();
let raw_key = action.value.chars().next().unwrap();
let mut input_state_table = session.input_state_table.borrow_mut();
let key_input_state = match input_state_table.get_mut(source_id).unwrap() {
InputSourceState::Key(key_input_state) => key_input_state,
_ => unreachable!(),
};
session.input_cancel_list.borrow_mut().push(ActionSequence {
id: source_id.into(),
actions: ActionsType::Key {
actions: vec![KeyActionItem::Key(KeyAction::Up(KeyUpAction {
value: action.value.clone(),
}))],
},
});
// Step 2 - 11. Done by `keyboard-types` crate.
if let Some(keyboard_event) = key_input_state.dispatch_keyup(raw_key) {
// Step 12
let cmd_msg = WebDriverCommandMsg::KeyboardAction(
@ -289,44 +381,49 @@ impl Handler {
}
}
fn get_pointer_input_state_mut(&mut self, source_id: &str) -> &mut PointerInputState {
let session = self.session_mut().unwrap();
let pointer_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
/// <https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerdown-action>
pub(crate) fn dispatch_pointerdown_action(&self, source_id: &str, action: &PointerDownAction) {
let session = self.session().unwrap();
let mut input_state_table = session.input_state_table.borrow_mut();
let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() {
InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
_ => unreachable!(),
};
pointer_input_state
}
fn get_key_input_state_mut(&mut self, source_id: &str) -> &mut KeyInputState {
let session = self.session_mut().unwrap();
let key_input_state = match session.input_state_table.get_mut(source_id).unwrap() {
InputSourceState::Key(key_input_state) => key_input_state,
_ => unreachable!(),
};
key_input_state
}
// https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerdown-action
pub(crate) fn dispatch_pointerdown_action(
&mut self,
source_id: &str,
action: &PointerDownAction,
) {
let webview_id = self.session().unwrap().webview_id;
let pointer_input_state = self.get_pointer_input_state_mut(source_id);
if pointer_input_state.pressed.contains(&action.button) {
return;
}
pointer_input_state.pressed.insert(action.button);
session.input_cancel_list.borrow_mut().push(ActionSequence {
id: source_id.into(),
actions: ActionsType::Pointer {
parameters: PointerActionParameters {
pointer_type: match pointer_input_state.subtype {
PointerType::Mouse => PointerType::Mouse,
PointerType::Pen => PointerType::Pen,
PointerType::Touch => PointerType::Touch,
},
},
actions: vec![PointerActionItem::Pointer(PointerAction::Up(
PointerUpAction {
button: action.button,
..Default::default()
},
))],
},
});
let msg_id = self.current_action_id.get().unwrap();
let cmd_msg = WebDriverCommandMsg::MouseButtonAction(
webview_id,
session.webview_id,
MouseButtonAction::Down,
action.button.into(),
pointer_input_state.x as f32,
pointer_input_state.y as f32,
msg_id,
self.constellation_sender.clone(),
);
self.constellation_chan
.send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
@ -334,21 +431,48 @@ impl Handler {
}
// https://w3c.github.io/webdriver/#dfn-dispatch-a-pointerup-action
pub(crate) fn dispatch_pointerup_action(&mut self, source_id: &str, action: &PointerUpAction) {
let webview_id = self.session().unwrap().webview_id;
let pointer_input_state = self.get_pointer_input_state_mut(source_id);
pub(crate) fn dispatch_pointerup_action(&self, source_id: &str, action: &PointerUpAction) {
let session = self.session().unwrap();
let mut input_state_table = session.input_state_table.borrow_mut();
let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() {
InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
_ => unreachable!(),
};
if !pointer_input_state.pressed.contains(&action.button) {
return;
}
pointer_input_state.pressed.remove(&action.button);
session.input_cancel_list.borrow_mut().push(ActionSequence {
id: source_id.into(),
actions: ActionsType::Pointer {
parameters: PointerActionParameters {
pointer_type: match pointer_input_state.subtype {
PointerType::Mouse => PointerType::Mouse,
PointerType::Pen => PointerType::Pen,
PointerType::Touch => PointerType::Touch,
},
},
actions: vec![PointerActionItem::Pointer(PointerAction::Down(
PointerDownAction {
button: action.button,
..Default::default()
},
))],
},
});
let msg_id = self.current_action_id.get().unwrap();
let cmd_msg = WebDriverCommandMsg::MouseButtonAction(
webview_id,
session.webview_id,
MouseButtonAction::Up,
action.button.into(),
pointer_input_state.x as f32,
pointer_input_state.y as f32,
msg_id,
self.constellation_sender.clone(),
);
self.constellation_chan
.send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
@ -357,7 +481,7 @@ impl Handler {
// https://w3c.github.io/webdriver/#dfn-dispatch-a-pointermove-action
pub(crate) fn dispatch_pointermove_action(
&mut self,
&self,
source_id: &str,
action: &PointerMoveAction,
tick_duration: u64,
@ -370,10 +494,10 @@ impl Handler {
// Steps 3 - 4
let (start_x, start_y) = match self
.session
.as_ref()
.session()
.unwrap()
.input_state_table
.borrow_mut()
.get(source_id)
.unwrap()
{
@ -416,7 +540,7 @@ impl Handler {
/// <https://w3c.github.io/webdriver/#dfn-perform-a-pointer-move>
#[allow(clippy::too_many_arguments)]
fn perform_pointer_move(
&mut self,
&self,
source_id: &str,
duration: u64,
start_x: f64,
@ -425,9 +549,13 @@ impl Handler {
target_y: f64,
tick_start: Instant,
) {
let webview_id = self.session().unwrap().webview_id;
let constellation_chan = self.constellation_chan.clone();
let pointer_input_state = self.get_pointer_input_state_mut(source_id);
let session = self.session().unwrap();
let mut input_state_table = session.input_state_table.borrow_mut();
let pointer_input_state = match input_state_table.get_mut(source_id).unwrap() {
InputSourceState::Pointer(pointer_input_state) => pointer_input_state,
_ => unreachable!(),
};
loop {
// Step 1
let time_delta = tick_start.elapsed().as_millis();
@ -459,9 +587,15 @@ impl Handler {
// Step 7
if x != current_x || y != current_y {
// Step 7.2
let cmd_msg = WebDriverCommandMsg::MouseMoveAction(webview_id, x as f32, y as f32);
//TODO: Need Synchronization here before updating `pointer_input_state`
constellation_chan
let msg_id = self.current_action_id.get().unwrap();
let cmd_msg = WebDriverCommandMsg::MouseMoveAction(
session.webview_id,
x as f32,
y as f32,
msg_id,
self.constellation_sender.clone(),
);
self.constellation_chan
.send(EmbedderToConstellationMessage::WebDriverCommand(cmd_msg))
.unwrap();
// Step 7.3
@ -481,7 +615,7 @@ impl Handler {
/// <https://w3c.github.io/webdriver/#dfn-dispatch-a-scroll-action>
fn dispatch_scroll_action(
&mut self,
&self,
action: &WheelScrollAction,
tick_duration: u64,
) -> Result<(), ErrorStatus> {
@ -546,7 +680,7 @@ impl Handler {
/// <https://w3c.github.io/webdriver/#dfn-perform-a-scroll>
#[allow(clippy::too_many_arguments)]
fn perform_scroll(
&mut self,
&self,
duration: u64,
x: i64,
y: i64,
@ -556,7 +690,7 @@ impl Handler {
mut curr_delta_y: i64,
tick_start: Instant,
) {
let session = self.session_mut().unwrap();
let session = self.session().unwrap();
// Step 1
let time_delta = tick_start.elapsed().as_millis();

111
components/webdriver_server/lib.rs
View file

@ -10,11 +10,12 @@ mod actions;
mod capabilities;
use std::borrow::ToOwned;
use std::cell::{Cell, RefCell};
use std::collections::{BTreeMap, HashMap};
use std::io::Cursor;
use std::net::{SocketAddr, SocketAddrV4};
use std::time::Duration;
use std::{env, fmt, mem, process, thread};
use std::{env, fmt, process, thread};
use base::id::{BrowsingContextId, WebViewId};
use base64::Engine;
@ -23,8 +24,9 @@ use constellation_traits::{EmbedderToConstellationMessage, TraversalDirection};
use cookie::{CookieBuilder, Expiration};
use crossbeam_channel::{Receiver, Sender, after, select, unbounded};
use embedder_traits::{
MouseButton, WebDriverCommandMsg, WebDriverCookieError, WebDriverFrameId, WebDriverJSError,
WebDriverJSResult, WebDriverJSValue, WebDriverLoadStatus, WebDriverScriptCommand,
MouseButton, WebDriverCommandMsg, WebDriverCommandResponse, WebDriverCookieError,
WebDriverFrameId, WebDriverJSError, WebDriverJSResult, WebDriverJSValue, WebDriverLoadStatus,
WebDriverMessageId, WebDriverScriptCommand,
};
use euclid::{Rect, Size2D};
use http::method::Method;
@ -43,8 +45,8 @@ use servo_url::ServoUrl;
use style_traits::CSSPixel;
use uuid::Uuid;
use webdriver::actions::{
ActionSequence, PointerDownAction, PointerMoveAction, PointerOrigin, PointerType,
PointerUpAction,
ActionSequence, ActionsType, PointerAction, PointerActionItem, PointerActionParameters,
PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction,
};
use webdriver::capabilities::CapabilitiesMatching;
use webdriver::command::{
@ -64,6 +66,26 @@ use webdriver::server::{self, Session, SessionTeardownKind, WebDriverHandler};
use crate::actions::{InputSourceState, PointerInputState};
#[derive(Default)]
pub struct WebDriverMessageIdGenerator {
counter: Cell<usize>,
}
impl WebDriverMessageIdGenerator {
pub fn new() -> Self {
Self {
counter: Cell::new(0),
}
}
/// Returns a unique ID.
pub fn next(&self) -> WebDriverMessageId {
let id = self.counter.get();
self.counter.set(id + 1);
WebDriverMessageId(id)
}
}
fn extension_routes() -> Vec<(Method, &'static str, ServoExtensionRoute)> {
vec![
(
@ -145,10 +167,11 @@ pub struct WebDriverSession {
unhandled_prompt_behavior: String,
// https://w3c.github.io/webdriver/#dfn-input-state-table
input_state_table: HashMap<String, InputSourceState>,
// https://w3c.github.io/webdriver/#dfn-input-cancel-list
input_cancel_list: Vec<ActionSequence>,
/// <https://w3c.github.io/webdriver/#dfn-input-state-map>
input_state_table: RefCell<HashMap<String, InputSourceState>>,
/// <https://w3c.github.io/webdriver/#dfn-input-cancel-list>
input_cancel_list: RefCell<Vec<ActionSequence>>,
}
impl WebDriverSession {
@ -172,8 +195,8 @@ impl WebDriverSession {
strict_file_interactability: false,
unhandled_prompt_behavior: "dismiss and notify".to_string(),
input_state_table: HashMap::new(),
input_cancel_list: Vec::new(),
input_state_table: RefCell::new(HashMap::new()),
input_cancel_list: RefCell::new(Vec::new()),
}
}
}
@ -187,8 +210,22 @@ struct Handler {
/// for it to send us a load-status. Messages sent on it
/// will be forwarded to the load_status_receiver.
load_status_sender: IpcSender<WebDriverLoadStatus>,
session: Option<WebDriverSession>,
/// The channel for sending Webdriver messages to the constellation.
constellation_chan: Sender<EmbedderToConstellationMessage>,
/// The IPC sender which we can clone and pass along to the constellation
constellation_sender: IpcSender<WebDriverCommandResponse>,
/// Receiver notification from the constellation when a command is completed
constellation_receiver: IpcReceiver<WebDriverCommandResponse>,
id_generator: WebDriverMessageIdGenerator,
current_action_id: Cell<Option<WebDriverMessageId>>,
resize_timeout: u32,
}
@ -409,11 +446,18 @@ impl Handler {
let (load_status_sender, receiver) = ipc::channel().unwrap();
let (sender, load_status_receiver) = unbounded();
ROUTER.route_ipc_receiver_to_crossbeam_sender(receiver, sender);
let (constellation_sender, constellation_receiver) = ipc::channel().unwrap();
Handler {
load_status_sender,
load_status_receiver,
session: None,
constellation_chan,
constellation_sender,
constellation_receiver,
id_generator: WebDriverMessageIdGenerator::new(),
current_action_id: Cell::new(None),
resize_timeout: 500,
}
}
@ -1445,18 +1489,13 @@ impl Handler {
}
fn handle_release_actions(&mut self) -> WebDriverResult<WebDriverResponse> {
let input_cancel_list = {
let session = self.session_mut()?;
session.input_cancel_list.reverse();
mem::take(&mut session.input_cancel_list)
};
let input_cancel_list = self.session().unwrap().input_cancel_list.borrow();
if let Err(error) = self.dispatch_actions(&input_cancel_list) {
return Err(WebDriverError::new(error, ""));
}
let session = self.session_mut()?;
session.input_state_table = HashMap::new();
let session = self.session()?;
session.input_state_table.borrow_mut().clear();
Ok(WebDriverResponse::Void)
}
@ -1614,7 +1653,7 @@ impl Handler {
let id = Uuid::new_v4().to_string();
// Step 8.1
self.session_mut()?.input_state_table.insert(
self.session_mut()?.input_state_table.borrow_mut().insert(
id.clone(),
InputSourceState::Pointer(PointerInputState::new(&PointerType::Mouse)),
);
@ -1645,19 +1684,31 @@ impl Handler {
..Default::default()
};
// Step 8.16 Dispatch a list of actions with input state,
// actions, session's current browsing context, and actions options.
if let Err(error) =
self.dispatch_pointermove_action(&id, &pointer_move_action, 0)
{
return Err(WebDriverError::new(error, ""));
}
let action_sequence = ActionSequence {
id: id.clone(),
actions: ActionsType::Pointer {
parameters: PointerActionParameters {
pointer_type: PointerType::Mouse,
},
actions: vec![
PointerActionItem::Pointer(PointerAction::Move(
pointer_move_action,
)),
PointerActionItem::Pointer(PointerAction::Down(
pointer_down_action,
)),
PointerActionItem::Pointer(PointerAction::Up(pointer_up_action)),
],
},
};
self.dispatch_pointerdown_action(&id, &pointer_down_action);
self.dispatch_pointerup_action(&id, &pointer_up_action);
let _ = self.dispatch_actions(&[action_sequence]);
// Step 8.17 Remove an input source with input state and input id.
self.session_mut()?.input_state_table.remove(&id);
self.session_mut()?
.input_state_table
.borrow_mut()
.remove(&id);
// Step 13
Ok(WebDriverResponse::Void)

8
ports/servoshell/desktop/headed_window.rs
View file

@ -262,11 +262,11 @@ impl Window {
ElementState::Released => MouseButtonAction::Up,
};
webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
webview.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
action,
button: mouse_button,
mouse_button,
point,
}));
)));
}
/// Handle key events before sending them to Servo.
@ -563,7 +563,7 @@ impl WindowPortsMethods for Window {
point.y -= (self.toolbar_height() * self.hidpi_scale_factor()).0;
self.webview_relative_mouse_point.set(point);
webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent { point }));
webview.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(point)));
},
WindowEvent::MouseWheel { delta, phase, .. } => {
let (mut dx, mut dy, mode) = match delta {

32
ports/servoshell/egl/app_state.rs
View file

@ -537,31 +537,31 @@ impl RunningAppState {
/// Register a mouse movement.
pub fn mouse_move(&self, x: f32, y: f32) {
self.active_webview()
.notify_input_event(InputEvent::MouseMove(MouseMoveEvent {
point: Point2D::new(x, y),
}));
.notify_input_event(InputEvent::MouseMove(MouseMoveEvent::new(Point2D::new(
x, y,
))));
self.perform_updates();
}
/// Register a mouse button press.
pub fn mouse_down(&self, x: f32, y: f32, button: MouseButton) {
self.active_webview()
.notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
action: MouseButtonAction::Down,
.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Down,
button,
point: Point2D::new(x, y),
}));
Point2D::new(x, y),
)));
self.perform_updates();
}
/// Register a mouse button release.
pub fn mouse_up(&self, x: f32, y: f32, button: MouseButton) {
self.active_webview()
.notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
action: MouseButtonAction::Up,
.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Up,
button,
point: Point2D::new(x, y),
}));
Point2D::new(x, y),
)));
self.perform_updates();
}
@ -589,11 +589,11 @@ impl RunningAppState {
/// Perform a click.
pub fn click(&self, x: f32, y: f32) {
self.active_webview()
.notify_input_event(InputEvent::MouseButton(MouseButtonEvent {
action: MouseButtonAction::Click,
button: MouseButton::Left,
point: Point2D::new(x, y),
}));
.notify_input_event(InputEvent::MouseButton(MouseButtonEvent::new(
MouseButtonAction::Click,
MouseButton::Left,
Point2D::new(x, y),
)));
self.perform_updates();
}