Webdriver: Send Keys use webview::notify_input_event (#37911)

Previously, we SendKeys will be forwarded to constellation by the
embedder. Now we use webview.notify_input_event, which will send
WebDriverCommandMsg::ForwardInputEvent for the KeyboardEvent and
CompositionEvent.

Fixes: part of https://github.com/servo/servo/issues/37370

---------

Signed-off-by: PotatoCP <kenzieradityatirtarahardja18@gmail.com>

components/constellation/constellation.rs

@@ -131,10 +131,10 @@ use embedder_traits::resources::{self, Resource};
 use embedder_traits::user_content_manager::UserContentManager;
 use embedder_traits::{
     AnimationState, CompositorHitTestResult, Cursor, EmbedderMsg, EmbedderProxy,
-    FocusSequenceNumber, ImeEvent, InputEvent, JSValue, JavaScriptEvaluationError,
+    FocusSequenceNumber, InputEvent, JSValue, JavaScriptEvaluationError, JavaScriptEvaluationId,
-    JavaScriptEvaluationId, KeyboardEvent, MediaSessionActionType, MediaSessionEvent,
+    KeyboardEvent, MediaSessionActionType, MediaSessionEvent, MediaSessionPlaybackState,
-    MediaSessionPlaybackState, MouseButton, MouseButtonAction, MouseButtonEvent, Theme,
+    MouseButton, MouseButtonAction, MouseButtonEvent, Theme, ViewportDetails, WebDriverCommandMsg,
-    ViewportDetails, WebDriverCommandMsg, WebDriverCommandResponse, WebDriverLoadStatus,
+    WebDriverCommandResponse, WebDriverLoadStatus,
 };
 use euclid::Size2D;
 use euclid::default::Size2D as UntypedSize2D;
@@ -142,7 +142,6 @@ use fonts::SystemFontServiceProxy;
 use ipc_channel::Error as IpcError;
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use ipc_channel::router::ROUTER;
-use keyboard_types::webdriver::Event as WebDriverInputEvent;
 use keyboard_types::{Key, KeyState, Modifiers};
 use layout_api::{LayoutFactory, ScriptThreadFactory};
 use log::{debug, error, info, trace, warn};
@@ -4594,36 +4593,8 @@ where
                     self.handle_send_error(pipeline_id, e);
                 }
             },
-            WebDriverCommandMsg::SendKeys(browsing_context_id, cmd) => {
+            WebDriverCommandMsg::SendKeys(..) => {
-                let pipeline_id = self
+                unreachable!("This command should be send directly to the embedder.");
-                    .browsing_contexts
-                    .get(&browsing_context_id)
-                    .expect("SendKeys: Browsing context must exist at this point")
-                    .pipeline_id;
-                let event_loop = match self.pipelines.get(&pipeline_id) {
-                    Some(pipeline) => pipeline.event_loop.clone(),
-                    None => return warn!("{}: SendKeys after closure", pipeline_id),
-                };
-                for event in cmd {
-                    let event = match event {
-                        WebDriverInputEvent::Keyboard(event) => ConstellationInputEvent {
-                            pressed_mouse_buttons: self.pressed_mouse_buttons,
-                            active_keyboard_modifiers: event.modifiers,
-                            hit_test_result: None,
-                            event: InputEvent::Keyboard(KeyboardEvent::new(event)),
-                        },
-                        WebDriverInputEvent::Composition(event) => ConstellationInputEvent {
-                            pressed_mouse_buttons: self.pressed_mouse_buttons,
-                            active_keyboard_modifiers: self.active_keyboard_modifiers,
-                            hit_test_result: None,
-                            event: InputEvent::Ime(ImeEvent::Composition(event)),
-                        },
-                    };
-                    let control_msg = ScriptThreadMessage::SendInputEvent(pipeline_id, event);
-                    if let Err(e) = event_loop.send(control_msg) {
-                        return self.handle_send_error(pipeline_id, e);
-                    }
-                }
             },
             WebDriverCommandMsg::KeyboardAction(..) => {
                 unreachable!("This command should be send directly to the embedder.");

components/shared/embedder/webdriver.rs

@@ -56,7 +56,7 @@ pub enum WebDriverCommandMsg {
     /// of a browsing context.
     ScriptCommand(BrowsingContextId, WebDriverScriptCommand),
     /// Act as if keys were pressed in the browsing context with the given ID.
-    SendKeys(BrowsingContextId, Vec<WebDriverInputEvent>),
+    SendKeys(WebViewId, Vec<WebDriverInputEvent>),
     /// Act as if keys were pressed or release in the browsing context with the given ID.
     KeyboardAction(
         WebViewId,

components/webdriver_server/lib.rs

@@ -1842,20 +1842,15 @@ impl Handler {
         element: &WebElement,
         keys: &SendKeysParameters,
     ) -> WebDriverResult<WebDriverResponse> {
-        let browsing_context_id = self.session()?.browsing_context_id;
+        // Step 3-8
-        // Step 3. If session's current browsing context is no longer open,
-        // return error with error code no such window.
-        self.verify_browsing_context_is_open(browsing_context_id)?;
         let (sender, receiver) = ipc::channel().unwrap();
-
         let cmd = WebDriverScriptCommand::WillSendKeys(
             element.to_string(),
             keys.text.to_string(),
             self.session()?.strict_file_interactability,
             sender,
         );
-        let cmd_msg = WebDriverCommandMsg::ScriptCommand(browsing_context_id, cmd);
+        self.browsing_context_script_command(cmd, VerifyBrowsingContextIsOpen::Yes)?;
-        self.send_message_to_embedder(cmd_msg)?;
 
         // TODO: distinguish the not found and not focusable cases
         // File input and non-typeable form control should have
@@ -1869,7 +1864,8 @@ impl Handler {
         // TODO: there's a race condition caused by the focus command and the
         // send keys command being two separate messages,
         // so the constellation may have changed state between them.
-        let cmd_msg = WebDriverCommandMsg::SendKeys(browsing_context_id, input_events);
+        // TODO: We should use `dispatch_action` to send the keys.
+        let cmd_msg = WebDriverCommandMsg::SendKeys(self.session()?.webview_id, input_events);
         self.send_message_to_embedder(cmd_msg)?;
 
         Ok(WebDriverResponse::Void)

ports/servoshell/desktop/app.rs

@@ -16,6 +16,7 @@ use constellation_traits::EmbedderToConstellationMessage;
 use crossbeam_channel::unbounded;
 use euclid::{Point2D, Vector2D};
 use ipc_channel::ipc;
+use keyboard_types::webdriver::Event as WebDriverInputEvent;
 use log::{info, trace, warn};
 use net::protocols::ProtocolRegistry;
 use servo::config::opts::Opts;
@@ -26,7 +27,7 @@ use servo::user_content_manager::{UserContentManager, UserScript};
 use servo::webrender_api::ScrollLocation;
 use servo::webrender_api::units::DeviceIntSize;
 use servo::{
-    EventLoopWaker, InputEvent, KeyboardEvent, MouseButtonEvent, MouseMoveEvent,
+    EventLoopWaker, ImeEvent, InputEvent, KeyboardEvent, MouseButtonEvent, MouseMoveEvent,
     WebDriverCommandMsg, WebDriverScriptCommand, WebDriverUserPromptAction, WheelDelta, WheelEvent,
     WheelMode,
 };
@@ -476,8 +477,25 @@ impl App {
                     }
                 },
                 // Key events don't need hit test so can be forwarded to constellation for now
-                WebDriverCommandMsg::SendKeys(..) => {
+                WebDriverCommandMsg::SendKeys(webview_id, webdriver_input_events) => {
-                    running_state.forward_webdriver_command(msg);
+                    let Some(webview) = running_state.webview_by_id(webview_id) else {
+                        continue;
+                    };
+
+                    for event in webdriver_input_events {
+                        match event {
+                            WebDriverInputEvent::Keyboard(event) => {
+                                webview.notify_input_event(InputEvent::Keyboard(
+                                    KeyboardEvent::new(event),
+                                ));
+                            },
+                            WebDriverInputEvent::Composition(event) => {
+                                webview.notify_input_event(InputEvent::Ime(ImeEvent::Composition(
+                                    event,
+                                )));
+                            },
+                        }
+                    }
                 },
                 WebDriverCommandMsg::KeyboardAction(webview_id, key_event, msg_id) => {
                     // TODO: We should do processing like in `headed_window:handle_keyboard_input`.