webdriver: Element Send keys use dispatch actions for KeyboardEvent (#38444)

Previously we immediately passed the KeyboardEvent to embedder. Now we
make element send keys go through the dispatch action which required by
spec. CompositionEvent still immediately passed through embedder

Testing: Should make
`./tests/wpt/tests/webdriver/tests/classic/element_send_keys/` more
stable.
Fixes: https://github.com/servo/servo/issues/38354
Fixes: https://github.com/servo/servo/issues/38442

---------

Signed-off-by: PotatoCP <Kenzie.Raditya.Tirtarahardja@huawei.com>
Co-authored-by: Euclid Ye <euclid.ye@huawei.com>

components/constellation/constellation.rs

@@ -4537,7 +4537,7 @@ where
             WebDriverCommandMsg::MaximizeWebView(..) |
             WebDriverCommandMsg::LoadUrl(..) |
             WebDriverCommandMsg::Refresh(..) |
-            WebDriverCommandMsg::SendKeys(..) |
+            WebDriverCommandMsg::DispatchComposition(..) |
             WebDriverCommandMsg::KeyboardAction(..) |
             WebDriverCommandMsg::MouseButtonAction(..) |
             WebDriverCommandMsg::MouseMoveAction(..) |

components/shared/embedder/webdriver.rs

@@ -12,8 +12,7 @@ use euclid::default::Rect as UntypedRect;
 use euclid::{Rect, Size2D};
 use hyper_serde::Serde;
 use ipc_channel::ipc::IpcSender;
-use keyboard_types::KeyboardEvent;
+use keyboard_types::{CompositionEvent, KeyboardEvent};
-use keyboard_types::webdriver::Event as WebDriverInputEvent;
 use pixels::RasterImage;
 use serde::{Deserialize, Serialize};
 use servo_geometry::DeviceIndependentIntRect;
@@ -92,8 +91,8 @@ pub enum WebDriverCommandMsg {
     /// Pass a webdriver command to the script thread of the current pipeline
     /// of a browsing context.
     ScriptCommand(BrowsingContextId, WebDriverScriptCommand),
-    /// Act as if keys were pressed in the browsing context with the given ID.
+    /// Dispatch composition event from element send keys command.
-    SendKeys(WebViewId, Vec<WebDriverInputEvent>),
+    DispatchComposition(WebViewId, CompositionEvent),
     /// Act as if keys were pressed or release in the browsing context with the given ID.
     KeyboardAction(
         WebViewId,

components/webdriver_server/lib.rs

@@ -36,8 +36,9 @@ use http::method::Method;
 use image::{DynamicImage, ImageFormat, RgbaImage};
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use ipc_channel::router::ROUTER;
-use keyboard_types::webdriver::send_keys;
+use keyboard_types::webdriver::{Event as DispatchStringEvent, KeyInputState, send_keys};
-use log::{debug, info};
+use keyboard_types::{Code, Key, KeyState, KeyboardEvent, Location, NamedKey};
+use log::{debug, error, info};
 use pixels::PixelFormat;
 use serde::de::{Deserializer, MapAccess, Visitor};
 use serde::ser::Serializer;
@@ -50,8 +51,9 @@ use style_traits::CSSPixel;
 use time::OffsetDateTime;
 use uuid::Uuid;
 use webdriver::actions::{
-    ActionSequence, ActionsType, PointerAction, PointerActionItem, PointerActionParameters,
+    ActionSequence, ActionsType, KeyAction, KeyActionItem, KeyDownAction, KeyUpAction,
-    PointerDownAction, PointerMoveAction, PointerOrigin, PointerType, PointerUpAction,
+    PointerAction, PointerActionItem, PointerActionParameters, PointerDownAction,
+    PointerMoveAction, PointerOrigin, PointerType, PointerUpAction,
 };
 use webdriver::capabilities::CapabilitiesMatching;
 use webdriver::command::{
@@ -2073,7 +2075,7 @@ impl Handler {
 
     /// <https://w3c.github.io/webdriver/#dfn-element-send-keys>
     fn handle_element_send_keys(
-        &self,
+        &mut self,
         element: &WebElement,
         keys: &SendKeysParameters,
     ) -> WebDriverResult<WebDriverResponse> {
@@ -2099,14 +2101,54 @@ impl Handler {
             return Ok(WebDriverResponse::Void);
         }
 
-        let input_events = send_keys(&keys.text);
-
         // TODO: there's a race condition caused by the focus command and the
         // send keys command being two separate messages,
         // so the constellation may have changed state between them.
-        // TODO: We should use `dispatch_action` to send the keys.
+
-        let cmd_msg = WebDriverCommandMsg::SendKeys(self.webview_id()?, input_events);
+        // Step 10. Let input id be a the result of generating a UUID.
-        self.send_message_to_embedder(cmd_msg)?;
+        let id = Uuid::new_v4().to_string();
+
+        // Step 12. Add an input source
+        self.session_mut()?
+            .input_state_table_mut()
+            .insert(id.clone(), InputSourceState::Key(KeyInputState::new()));
+
+        // Step 13. dispatch actions for a string
+        // https://w3c.github.io/webdriver/#dfn-dispatch-actions-for-a-string
+        let input_events = send_keys(&keys.text);
+
+        for event in input_events {
+            match event {
+                DispatchStringEvent::Keyboard(event) => {
+                    let raw_string = convert_keyboard_event_to_string(&event);
+                    let key_action = match event.state {
+                        KeyState::Down => KeyAction::Down(KeyDownAction { value: raw_string }),
+                        KeyState::Up => KeyAction::Up(KeyUpAction { value: raw_string }),
+                    };
+                    let action_sequence = ActionSequence {
+                        id: id.clone(),
+                        actions: ActionsType::Key {
+                            actions: vec![KeyActionItem::Key(key_action)],
+                        },
+                    };
+
+                    let actions_by_tick = self.actions_by_tick_from_sequence(vec![action_sequence]);
+                    if let Err(e) =
+                        self.dispatch_actions(actions_by_tick, self.browsing_context_id()?)
+                    {
+                        log::error!("handle_element_send_keys: dispatch_actions failed: {:?}", e);
+                    }
+                },
+                DispatchStringEvent::Composition(event) => {
+                    let cmd_msg =
+                        WebDriverCommandMsg::DispatchComposition(self.webview_id()?, event);
+                    self.send_message_to_embedder(cmd_msg)?;
+                },
+            }
+        }
+
+        // Step 14. Remove an input source with input state and input id.
+        self.session_mut()?.input_state_table_mut().remove(&id);
 
         Ok(WebDriverResponse::Void)
     }
@@ -2591,3 +2633,79 @@ fn unwrap_first_element_response(res: WebDriverResponse) -> WebDriverResult<WebD
         unreachable!()
     }
 }
+
+fn convert_keyboard_event_to_string(event: &KeyboardEvent) -> String {
+    let key = &event.key;
+    let named_key = match key {
+        Key::Character(s) => return s.to_string(),
+        Key::Named(named_key) => named_key,
+    };
+
+    match event.location {
+        Location::Left | Location::Standard => match named_key {
+            NamedKey::Unidentified => '\u{E000}'.to_string(),
+            NamedKey::Cancel => '\u{E001}'.to_string(),
+            NamedKey::Help => '\u{E002}'.to_string(),
+            NamedKey::Backspace => '\u{E003}'.to_string(),
+            NamedKey::Tab => '\u{E004}'.to_string(),
+            NamedKey::Clear => '\u{E005}'.to_string(),
+            NamedKey::Enter => match event.code {
+                Code::NumpadEnter => '\u{E007}'.to_string(),
+                _ => '\u{E006}'.to_string(),
+            },
+            NamedKey::Shift => '\u{E008}'.to_string(),
+            NamedKey::Control => '\u{E009}'.to_string(),
+            NamedKey::Alt => '\u{E00A}'.to_string(),
+            NamedKey::Pause => '\u{E00B}'.to_string(),
+            NamedKey::Escape => '\u{E00C}'.to_string(),
+            NamedKey::PageUp => '\u{E00E}'.to_string(),
+            NamedKey::PageDown => '\u{E00F}'.to_string(),
+            NamedKey::End => '\u{E010}'.to_string(),
+            NamedKey::Home => '\u{E011}'.to_string(),
+            NamedKey::ArrowLeft => '\u{E012}'.to_string(),
+            NamedKey::ArrowUp => '\u{E013}'.to_string(),
+            NamedKey::ArrowRight => '\u{E014}'.to_string(),
+            NamedKey::ArrowDown => '\u{E015}'.to_string(),
+            NamedKey::Insert => '\u{E016}'.to_string(),
+            NamedKey::Delete => '\u{E017}'.to_string(),
+            NamedKey::F1 => '\u{E031}'.to_string(),
+            NamedKey::F2 => '\u{E032}'.to_string(),
+            NamedKey::F3 => '\u{E033}'.to_string(),
+            NamedKey::F4 => '\u{E034}'.to_string(),
+            NamedKey::F5 => '\u{E035}'.to_string(),
+            NamedKey::F6 => '\u{E036}'.to_string(),
+            NamedKey::F7 => '\u{E037}'.to_string(),
+            NamedKey::F8 => '\u{E038}'.to_string(),
+            NamedKey::F9 => '\u{E039}'.to_string(),
+            NamedKey::F10 => '\u{E03A}'.to_string(),
+            NamedKey::F11 => '\u{E03B}'.to_string(),
+            NamedKey::F12 => '\u{E03C}'.to_string(),
+            NamedKey::Meta => '\u{E03D}'.to_string(),
+            NamedKey::ZenkakuHankaku => '\u{E040}'.to_string(),
+            _ => {
+                error!("Unexpected NamedKey on send_keys");
+                '\u{E000}'.to_string()
+            },
+        },
+        Location::Right | Location::Numpad => match named_key {
+            NamedKey::Shift => '\u{E050}'.to_string(),
+            NamedKey::Control => '\u{E051}'.to_string(),
+            NamedKey::Alt => '\u{E052}'.to_string(),
+            NamedKey::Meta => '\u{E053}'.to_string(),
+            NamedKey::PageUp => '\u{E054}'.to_string(),
+            NamedKey::PageDown => '\u{E055}'.to_string(),
+            NamedKey::End => '\u{E056}'.to_string(),
+            NamedKey::Home => '\u{E057}'.to_string(),
+            NamedKey::ArrowLeft => '\u{E058}'.to_string(),
+            NamedKey::ArrowUp => '\u{E059}'.to_string(),
+            NamedKey::ArrowRight => '\u{E05A}'.to_string(),
+            NamedKey::ArrowDown => '\u{E05B}'.to_string(),
+            NamedKey::Insert => '\u{E05C}'.to_string(),
+            NamedKey::Delete => '\u{E05D}'.to_string(),
+            _ => {
+                error!("Unexpected NamedKey on send_keys");
+                '\u{E000}'.to_string()
+            },
+        },
+    }
+}

ports/servoshell/desktop/app.rs

@@ -16,7 +16,6 @@ use constellation_traits::EmbedderToConstellationMessage;
 use crossbeam_channel::unbounded;
 use euclid::{Point2D, Vector2D};
 use ipc_channel::ipc;
-use keyboard_types::webdriver::Event as WebDriverInputEvent;
 use log::{info, trace, warn};
 use net::protocols::ProtocolRegistry;
 use servo::config::opts::Opts;
@@ -486,24 +485,11 @@ impl App {
                     }
                 },
                 // Key events don't need hit test so can be forwarded to constellation for now
-                WebDriverCommandMsg::SendKeys(webview_id, webdriver_input_events) => {
+                WebDriverCommandMsg::DispatchComposition(webview_id, composition_event) => {
-                    let Some(webview) = running_state.webview_by_id(webview_id) else {
+                    if let Some(webview) = running_state.webview_by_id(webview_id) {
-                        continue;
+                        webview.notify_input_event(InputEvent::Ime(ImeEvent::Composition(
-                    };
+                            composition_event,
-
+                        )));
-                    for event in webdriver_input_events {
-                        match event {
-                            WebDriverInputEvent::Keyboard(event) => {
-                                webview.notify_input_event(InputEvent::Keyboard(
-                                    KeyboardEvent::new(event),
-                                ));
-                            },
-                            WebDriverInputEvent::Composition(event) => {
-                                webview.notify_input_event(InputEvent::Ime(ImeEvent::Composition(
-                                    event,
-                                )));
-                            },
-                        }
                     }
                 },
                 WebDriverCommandMsg::KeyboardAction(webview_id, key_event, msg_id) => {

tests/wpt/meta/webdriver/tests/classic/element_clear/user_prompts.py.ini

@@ -1,54 +0,0 @@
-[user_prompts.py]
-  [test_accept[alert-None\]]
-    expected: FAIL
-
-  [test_accept[confirm-True\]]
-    expected: FAIL
-
-  [test_accept[prompt-\]]
-    expected: FAIL
-
-  [test_accept_and_notify[alert-None\]]
-    expected: FAIL
-
-  [test_accept_and_notify[confirm-True\]]
-    expected: FAIL
-
-  [test_accept_and_notify[prompt-\]]
-    expected: FAIL
-
-  [test_dismiss[alert-None\]]
-    expected: FAIL
-
-  [test_dismiss[confirm-False\]]
-    expected: FAIL
-
-  [test_dismiss[prompt-None\]]
-    expected: FAIL
-
-  [test_dismiss_and_notify[alert-None\]]
-    expected: FAIL
-
-  [test_dismiss_and_notify[confirm-False\]]
-    expected: FAIL
-
-  [test_dismiss_and_notify[prompt-None\]]
-    expected: FAIL
-
-  [test_ignore[alert\]]
-    expected: FAIL
-
-  [test_ignore[confirm\]]
-    expected: FAIL
-
-  [test_ignore[prompt\]]
-    expected: FAIL
-
-  [test_default[alert-None\]]
-    expected: FAIL
-
-  [test_default[confirm-False\]]
-    expected: FAIL
-
-  [test_default[prompt-None\]]
-    expected: FAIL

tests/wpt/meta/webdriver/tests/classic/element_send_keys/events.py.ini

@@ -1,6 +0,0 @@
-[events.py]
-  [test_form_control_send_text[input\]]
-    expected: FAIL
-
-  [test_form_control_send_text[textarea\]]
-    expected: FAIL

tests/wpt/meta/webdriver/tests/classic/element_send_keys/form_controls.py.ini

@@ -1,21 +0,0 @@
-[form_controls.py]
-  [test_input]
-    expected: FAIL
-
-  [test_textarea]
-    expected: FAIL
-
-  [test_input_append]
-    expected: FAIL
-
-  [test_input_insert_when_focused]
-    expected: FAIL
-
-  [test_textarea_insert_when_focused]
-    expected: FAIL
-
-  [test_date]
-    expected: FAIL
-
-  [test_textarea_append]
-    expected: FAIL

tests/wpt/meta/webdriver/tests/classic/element_send_keys/interactability.py.ini

@@ -11,11 +11,5 @@
   [test_hidden]
     expected: FAIL
 
-  [test_iframe_is_interactable]
+  [test_readonly_element]
-    expected: FAIL
-
-  [test_transparent_element]
-    expected: FAIL
-
-  [test_obscured_element]
     expected: FAIL

tests/wpt/meta/webdriver/tests/classic/element_send_keys/user_prompts.py.ini

@@ -1,15 +0,0 @@
-[user_prompts.py]
-  [test_accept[confirm-True\]]
-    expected: FAIL
-
-  [test_accept[prompt-\]]
-    expected: FAIL
-
-  [test_dismiss[prompt-None\]]
-    expected: FAIL
-
-  [test_accept[alert-None\]]
-    expected: FAIL
-
-  [test_dismiss[confirm-False\]]
-    expected: FAIL