From cae8d22823e5ed94336d477d9479e38fc071045e Mon Sep 17 00:00:00 2001 From: Kenzie Raditya Tirtarahardja Date: Fri, 22 Aug 2025 13:20:54 +0800 Subject: [PATCH] webdriver: Element Send keys use dispatch actions for KeyboardEvent (#38444) Previously we immediately passed the KeyboardEvent to embedder. Now we make element send keys go through the dispatch action which required by spec. CompositionEvent still immediately passed through embedder Testing: Should make `./tests/wpt/tests/webdriver/tests/classic/element_send_keys/` more stable. Fixes: https://github.com/servo/servo/issues/38354 Fixes: https://github.com/servo/servo/issues/38442 --------- Signed-off-by: PotatoCP Co-authored-by: Euclid Ye --- components/constellation/constellation.rs | 2 +- components/shared/embedder/webdriver.rs | 7 +- components/webdriver_server/lib.rs | 138 ++++++++++++++++-- ports/servoshell/desktop/app.rs | 24 +-- .../classic/element_clear/user_prompts.py.ini | 54 ------- .../classic/element_send_keys/events.py.ini | 6 - .../element_send_keys/form_controls.py.ini | 21 --- .../element_send_keys/interactability.py.ini | 8 +- .../element_send_keys/user_prompts.py.ini | 15 -- 9 files changed, 138 insertions(+), 137 deletions(-) delete mode 100644 tests/wpt/meta/webdriver/tests/classic/element_clear/user_prompts.py.ini delete mode 100644 tests/wpt/meta/webdriver/tests/classic/element_send_keys/events.py.ini delete mode 100644 tests/wpt/meta/webdriver/tests/classic/element_send_keys/form_controls.py.ini delete mode 100644 tests/wpt/meta/webdriver/tests/classic/element_send_keys/user_prompts.py.ini diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 745c35f0864..d265678d13a 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -4537,7 +4537,7 @@ where WebDriverCommandMsg::MaximizeWebView(..) | WebDriverCommandMsg::LoadUrl(..) | WebDriverCommandMsg::Refresh(..) | - WebDriverCommandMsg::SendKeys(..) | + WebDriverCommandMsg::DispatchComposition(..) | WebDriverCommandMsg::KeyboardAction(..) | WebDriverCommandMsg::MouseButtonAction(..) | WebDriverCommandMsg::MouseMoveAction(..) | diff --git a/components/shared/embedder/webdriver.rs b/components/shared/embedder/webdriver.rs index 7bd11a1d163..13c426ac4f4 100644 --- a/components/shared/embedder/webdriver.rs +++ b/components/shared/embedder/webdriver.rs @@ -12,8 +12,7 @@ use euclid::default::Rect as UntypedRect; use euclid::{Rect, Size2D}; use hyper_serde::Serde; use ipc_channel::ipc::IpcSender; -use keyboard_types::KeyboardEvent; -use keyboard_types::webdriver::Event as WebDriverInputEvent; +use keyboard_types::{CompositionEvent, KeyboardEvent}; use pixels::RasterImage; use serde::{Deserialize, Serialize}; use servo_geometry::DeviceIndependentIntRect; @@ -92,8 +91,8 @@ pub enum WebDriverCommandMsg { /// Pass a webdriver command to the script thread of the current pipeline /// of a browsing context. ScriptCommand(BrowsingContextId, WebDriverScriptCommand), - /// Act as if keys were pressed in the browsing context with the given ID. - SendKeys(WebViewId, Vec), + /// Dispatch composition event from element send keys command. + DispatchComposition(WebViewId, CompositionEvent), /// Act as if keys were pressed or release in the browsing context with the given ID. KeyboardAction( WebViewId, diff --git a/components/webdriver_server/lib.rs b/components/webdriver_server/lib.rs index 5b8ffd1ea81..1a6cb9493ef 100644 --- a/components/webdriver_server/lib.rs +++ b/components/webdriver_server/lib.rs @@ -36,8 +36,9 @@ use http::method::Method; use image::{DynamicImage, ImageFormat, RgbaImage}; use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use ipc_channel::router::ROUTER; -use keyboard_types::webdriver::send_keys; -use log::{debug, info}; +use keyboard_types::webdriver::{Event as DispatchStringEvent, KeyInputState, send_keys}; +use keyboard_types::{Code, Key, KeyState, KeyboardEvent, Location, NamedKey}; +use log::{debug, error, info}; use pixels::PixelFormat; use serde::de::{Deserializer, MapAccess, Visitor}; use serde::ser::Serializer; @@ -50,8 +51,9 @@ use style_traits::CSSPixel; use time::OffsetDateTime; use uuid::Uuid; use webdriver::actions::{ - ActionSequence, ActionsType, PointerAction, PointerActionItem, PointerActionParameters, - PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, + ActionSequence, ActionsType, KeyAction, KeyActionItem, KeyDownAction, KeyUpAction, + PointerAction, PointerActionItem, PointerActionParameters, PointerDownAction, + PointerMoveAction, PointerOrigin, PointerType, PointerUpAction, }; use webdriver::capabilities::CapabilitiesMatching; use webdriver::command::{ @@ -2073,7 +2075,7 @@ impl Handler { /// fn handle_element_send_keys( - &self, + &mut self, element: &WebElement, keys: &SendKeysParameters, ) -> WebDriverResult { @@ -2099,14 +2101,54 @@ impl Handler { return Ok(WebDriverResponse::Void); } - let input_events = send_keys(&keys.text); - // TODO: there's a race condition caused by the focus command and the // send keys command being two separate messages, // so the constellation may have changed state between them. - // TODO: We should use `dispatch_action` to send the keys. - let cmd_msg = WebDriverCommandMsg::SendKeys(self.webview_id()?, input_events); - self.send_message_to_embedder(cmd_msg)?; + + // Step 10. Let input id be a the result of generating a UUID. + let id = Uuid::new_v4().to_string(); + + // Step 12. Add an input source + self.session_mut()? + .input_state_table_mut() + .insert(id.clone(), InputSourceState::Key(KeyInputState::new())); + + // Step 13. dispatch actions for a string + // https://w3c.github.io/webdriver/#dfn-dispatch-actions-for-a-string + let input_events = send_keys(&keys.text); + + for event in input_events { + match event { + DispatchStringEvent::Keyboard(event) => { + let raw_string = convert_keyboard_event_to_string(&event); + let key_action = match event.state { + KeyState::Down => KeyAction::Down(KeyDownAction { value: raw_string }), + KeyState::Up => KeyAction::Up(KeyUpAction { value: raw_string }), + }; + let action_sequence = ActionSequence { + id: id.clone(), + actions: ActionsType::Key { + actions: vec![KeyActionItem::Key(key_action)], + }, + }; + + let actions_by_tick = self.actions_by_tick_from_sequence(vec![action_sequence]); + if let Err(e) = + self.dispatch_actions(actions_by_tick, self.browsing_context_id()?) + { + log::error!("handle_element_send_keys: dispatch_actions failed: {:?}", e); + } + }, + DispatchStringEvent::Composition(event) => { + let cmd_msg = + WebDriverCommandMsg::DispatchComposition(self.webview_id()?, event); + self.send_message_to_embedder(cmd_msg)?; + }, + } + } + + // Step 14. Remove an input source with input state and input id. + self.session_mut()?.input_state_table_mut().remove(&id); Ok(WebDriverResponse::Void) } @@ -2591,3 +2633,79 @@ fn unwrap_first_element_response(res: WebDriverResponse) -> WebDriverResult String { + let key = &event.key; + let named_key = match key { + Key::Character(s) => return s.to_string(), + Key::Named(named_key) => named_key, + }; + + match event.location { + Location::Left | Location::Standard => match named_key { + NamedKey::Unidentified => '\u{E000}'.to_string(), + NamedKey::Cancel => '\u{E001}'.to_string(), + NamedKey::Help => '\u{E002}'.to_string(), + NamedKey::Backspace => '\u{E003}'.to_string(), + NamedKey::Tab => '\u{E004}'.to_string(), + NamedKey::Clear => '\u{E005}'.to_string(), + NamedKey::Enter => match event.code { + Code::NumpadEnter => '\u{E007}'.to_string(), + _ => '\u{E006}'.to_string(), + }, + NamedKey::Shift => '\u{E008}'.to_string(), + NamedKey::Control => '\u{E009}'.to_string(), + NamedKey::Alt => '\u{E00A}'.to_string(), + NamedKey::Pause => '\u{E00B}'.to_string(), + NamedKey::Escape => '\u{E00C}'.to_string(), + NamedKey::PageUp => '\u{E00E}'.to_string(), + NamedKey::PageDown => '\u{E00F}'.to_string(), + NamedKey::End => '\u{E010}'.to_string(), + NamedKey::Home => '\u{E011}'.to_string(), + NamedKey::ArrowLeft => '\u{E012}'.to_string(), + NamedKey::ArrowUp => '\u{E013}'.to_string(), + NamedKey::ArrowRight => '\u{E014}'.to_string(), + NamedKey::ArrowDown => '\u{E015}'.to_string(), + NamedKey::Insert => '\u{E016}'.to_string(), + NamedKey::Delete => '\u{E017}'.to_string(), + NamedKey::F1 => '\u{E031}'.to_string(), + NamedKey::F2 => '\u{E032}'.to_string(), + NamedKey::F3 => '\u{E033}'.to_string(), + NamedKey::F4 => '\u{E034}'.to_string(), + NamedKey::F5 => '\u{E035}'.to_string(), + NamedKey::F6 => '\u{E036}'.to_string(), + NamedKey::F7 => '\u{E037}'.to_string(), + NamedKey::F8 => '\u{E038}'.to_string(), + NamedKey::F9 => '\u{E039}'.to_string(), + NamedKey::F10 => '\u{E03A}'.to_string(), + NamedKey::F11 => '\u{E03B}'.to_string(), + NamedKey::F12 => '\u{E03C}'.to_string(), + NamedKey::Meta => '\u{E03D}'.to_string(), + NamedKey::ZenkakuHankaku => '\u{E040}'.to_string(), + _ => { + error!("Unexpected NamedKey on send_keys"); + '\u{E000}'.to_string() + }, + }, + Location::Right | Location::Numpad => match named_key { + NamedKey::Shift => '\u{E050}'.to_string(), + NamedKey::Control => '\u{E051}'.to_string(), + NamedKey::Alt => '\u{E052}'.to_string(), + NamedKey::Meta => '\u{E053}'.to_string(), + NamedKey::PageUp => '\u{E054}'.to_string(), + NamedKey::PageDown => '\u{E055}'.to_string(), + NamedKey::End => '\u{E056}'.to_string(), + NamedKey::Home => '\u{E057}'.to_string(), + NamedKey::ArrowLeft => '\u{E058}'.to_string(), + NamedKey::ArrowUp => '\u{E059}'.to_string(), + NamedKey::ArrowRight => '\u{E05A}'.to_string(), + NamedKey::ArrowDown => '\u{E05B}'.to_string(), + NamedKey::Insert => '\u{E05C}'.to_string(), + NamedKey::Delete => '\u{E05D}'.to_string(), + _ => { + error!("Unexpected NamedKey on send_keys"); + '\u{E000}'.to_string() + }, + }, + } +} diff --git a/ports/servoshell/desktop/app.rs b/ports/servoshell/desktop/app.rs index 4a4abe1d2fa..ccfd2158852 100644 --- a/ports/servoshell/desktop/app.rs +++ b/ports/servoshell/desktop/app.rs @@ -16,7 +16,6 @@ use constellation_traits::EmbedderToConstellationMessage; use crossbeam_channel::unbounded; use euclid::{Point2D, Vector2D}; use ipc_channel::ipc; -use keyboard_types::webdriver::Event as WebDriverInputEvent; use log::{info, trace, warn}; use net::protocols::ProtocolRegistry; use servo::config::opts::Opts; @@ -486,24 +485,11 @@ impl App { } }, // Key events don't need hit test so can be forwarded to constellation for now - WebDriverCommandMsg::SendKeys(webview_id, webdriver_input_events) => { - let Some(webview) = running_state.webview_by_id(webview_id) else { - continue; - }; - - for event in webdriver_input_events { - match event { - WebDriverInputEvent::Keyboard(event) => { - webview.notify_input_event(InputEvent::Keyboard( - KeyboardEvent::new(event), - )); - }, - WebDriverInputEvent::Composition(event) => { - webview.notify_input_event(InputEvent::Ime(ImeEvent::Composition( - event, - ))); - }, - } + WebDriverCommandMsg::DispatchComposition(webview_id, composition_event) => { + if let Some(webview) = running_state.webview_by_id(webview_id) { + webview.notify_input_event(InputEvent::Ime(ImeEvent::Composition( + composition_event, + ))); } }, WebDriverCommandMsg::KeyboardAction(webview_id, key_event, msg_id) => { diff --git a/tests/wpt/meta/webdriver/tests/classic/element_clear/user_prompts.py.ini b/tests/wpt/meta/webdriver/tests/classic/element_clear/user_prompts.py.ini deleted file mode 100644 index a383edda800..00000000000 --- a/tests/wpt/meta/webdriver/tests/classic/element_clear/user_prompts.py.ini +++ /dev/null @@ -1,54 +0,0 @@ -[user_prompts.py] - [test_accept[alert-None\]] - expected: FAIL - - [test_accept[confirm-True\]] - expected: FAIL - - [test_accept[prompt-\]] - expected: FAIL - - [test_accept_and_notify[alert-None\]] - expected: FAIL - - [test_accept_and_notify[confirm-True\]] - expected: FAIL - - [test_accept_and_notify[prompt-\]] - expected: FAIL - - [test_dismiss[alert-None\]] - expected: FAIL - - [test_dismiss[confirm-False\]] - expected: FAIL - - [test_dismiss[prompt-None\]] - expected: FAIL - - [test_dismiss_and_notify[alert-None\]] - expected: FAIL - - [test_dismiss_and_notify[confirm-False\]] - expected: FAIL - - [test_dismiss_and_notify[prompt-None\]] - expected: FAIL - - [test_ignore[alert\]] - expected: FAIL - - [test_ignore[confirm\]] - expected: FAIL - - [test_ignore[prompt\]] - expected: FAIL - - [test_default[alert-None\]] - expected: FAIL - - [test_default[confirm-False\]] - expected: FAIL - - [test_default[prompt-None\]] - expected: FAIL diff --git a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/events.py.ini b/tests/wpt/meta/webdriver/tests/classic/element_send_keys/events.py.ini deleted file mode 100644 index 5e4543ef0e9..00000000000 --- a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/events.py.ini +++ /dev/null @@ -1,6 +0,0 @@ -[events.py] - [test_form_control_send_text[input\]] - expected: FAIL - - [test_form_control_send_text[textarea\]] - expected: FAIL diff --git a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/form_controls.py.ini b/tests/wpt/meta/webdriver/tests/classic/element_send_keys/form_controls.py.ini deleted file mode 100644 index 52ffe7e88d9..00000000000 --- a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/form_controls.py.ini +++ /dev/null @@ -1,21 +0,0 @@ -[form_controls.py] - [test_input] - expected: FAIL - - [test_textarea] - expected: FAIL - - [test_input_append] - expected: FAIL - - [test_input_insert_when_focused] - expected: FAIL - - [test_textarea_insert_when_focused] - expected: FAIL - - [test_date] - expected: FAIL - - [test_textarea_append] - expected: FAIL diff --git a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/interactability.py.ini b/tests/wpt/meta/webdriver/tests/classic/element_send_keys/interactability.py.ini index 4975e9d64a1..f1758c1c54f 100644 --- a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/interactability.py.ini +++ b/tests/wpt/meta/webdriver/tests/classic/element_send_keys/interactability.py.ini @@ -11,11 +11,5 @@ [test_hidden] expected: FAIL - [test_iframe_is_interactable] - expected: FAIL - - [test_transparent_element] - expected: FAIL - - [test_obscured_element] + [test_readonly_element] expected: FAIL diff --git a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/user_prompts.py.ini b/tests/wpt/meta/webdriver/tests/classic/element_send_keys/user_prompts.py.ini deleted file mode 100644 index 52d8130f726..00000000000 --- a/tests/wpt/meta/webdriver/tests/classic/element_send_keys/user_prompts.py.ini +++ /dev/null @@ -1,15 +0,0 @@ -[user_prompts.py] - [test_accept[confirm-True\]] - expected: FAIL - - [test_accept[prompt-\]] - expected: FAIL - - [test_dismiss[prompt-None\]] - expected: FAIL - - [test_accept[alert-None\]] - expected: FAIL - - [test_dismiss[confirm-False\]] - expected: FAIL