Auto merge of #22224 - pyfisch:composition-webdriver, r=nox

Forward WebDriver CompositionEvent

Dispatch composition events in JS.
Insert characters from composition events to text input.

CompositionEvents currently can only be
created by WebDriver and not by embedders.

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [ ] These changes fix #__ (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [ ] These changes do not require tests because _____

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22224)
<!-- Reviewable:end -->

components/atoms/static_atoms.txt

@@ -13,6 +13,9 @@ click
 close
 color
 complete
+compositionend
+compositionstart
+compositionupdate
 controllerchange
 cursive
 date

components/constellation/constellation.rs

@@ -119,6 +119,7 @@ use gfx_traits::Epoch;
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use ipc_channel::router::ROUTER;
 use ipc_channel::Error as IpcError;
+use keyboard_types::webdriver::Event as WebDriverInputEvent;
 use keyboard_types::KeyboardEvent;
 use layout_traits::LayoutThreadFactory;
 use log::{Level, LevelFilter, Log, Metadata, Record};
@@ -3035,7 +3036,14 @@ where
                     None => return warn!("Pipeline {} SendKeys after closure.", pipeline_id),
                 };
                 for event in cmd {
-                    let event = CompositorEvent::KeyboardEvent(event);
+                    let event = match event {
+                        WebDriverInputEvent::Keyboard(event) => {
+                            CompositorEvent::KeyboardEvent(event)
+                        },
+                        WebDriverInputEvent::Composition(event) => {
+                            CompositorEvent::CompositionEvent(event)
+                        },
+                    };
                     let control_msg = ConstellationControlMsg::SendEvent(pipeline_id, event);
                     if let Err(e) = event_loop.send(control_msg) {
                         return self.handle_send_error(pipeline_id, e);

components/script/Cargo.toml

@@ -64,7 +64,7 @@ image = "0.20"
 ipc-channel = "0.11"
 itertools = "0.7.6"
 jstraceable_derive = {path = "../jstraceable_derive"}
-keyboard-types = "0.4.3"
+keyboard-types = "0.4.4"
 lazy_static = "1"
 libc = "0.2"
 log = "0.4"

components/script/dom/compositionevent.rs

@@ -59,6 +59,10 @@ impl CompositionEvent {
         );
         Ok(event)
     }
+
+    pub fn data(&self) -> &str {
+        &*self.data
+    }
 }
 
 impl CompositionEventMethods for CompositionEvent {

components/script/dom/document.rs

@@ -38,6 +38,7 @@ use crate::dom::bindings::xmlname::{
 };
 use crate::dom::closeevent::CloseEvent;
 use crate::dom::comment::Comment;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::cssstylesheet::CSSStyleSheet;
 use crate::dom::customelementregistry::CustomElementDefinition;
 use crate::dom::customevent::CustomEvent;
@@ -1465,6 +1466,37 @@ impl Document {
         self.window.reflow(ReflowGoal::Full, ReflowReason::KeyEvent);
     }
 
+    pub fn dispatch_composition_event(
+        &self,
+        composition_event: ::keyboard_types::CompositionEvent,
+    ) {
+        // spec: https://w3c.github.io/uievents/#compositionstart
+        // spec: https://w3c.github.io/uievents/#compositionupdate
+        // spec: https://w3c.github.io/uievents/#compositionend
+        // > Event.target : focused element processing the composition
+        let focused = self.get_focused_element();
+        let target = if let Some(elem) = &focused {
+            elem.upcast()
+        } else {
+            // Event is only dispatched if there is a focused element.
+            return;
+        };
+
+        let cancelable = composition_event.state == keyboard_types::CompositionState::Start;
+
+        let compositionevent = CompositionEvent::new(
+            &self.window,
+            DOMString::from(composition_event.state.to_string()),
+            true,
+            cancelable,
+            Some(&self.window),
+            0,
+            DOMString::from(composition_event.data),
+        );
+        let event = compositionevent.upcast::<Event>();
+        event.fire(target);
+    }
+
     // https://dom.spec.whatwg.org/#converting-nodes-into-a-node
     pub fn node_from_nodes_and_strings(
         &self,

components/script/dom/htmlinputelement.rs

@@ -17,6 +17,7 @@ use crate::dom::bindings::inheritance::Castable;
 use crate::dom::bindings::reflector::DomObject;
 use crate::dom::bindings::root::{Dom, DomRoot, LayoutDom, MutNullableDom, RootedReference};
 use crate::dom::bindings::str::DOMString;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::document::Document;
 use crate::dom::element::{
     AttributeMutation, Element, LayoutElementHelpers, RawLayoutElementHelpers,
@@ -1526,6 +1527,23 @@ impl VirtualMethods for HTMLInputElement {
                         &window,
                     );
             }
+        } else if (event.type_() == atom!("compositionstart") ||
+            event.type_() == atom!("compositionupdate") ||
+            event.type_() == atom!("compositionend")) &&
+            self.input_type().is_textual_or_password()
+        {
+            // TODO: Update DOM on start and continue
+            // and generally do proper CompositionEvent handling.
+            if let Some(compositionevent) = event.downcast::<CompositionEvent>() {
+                if event.type_() == atom!("compositionend") {
+                    let _ = self
+                        .textinput
+                        .borrow_mut()
+                        .handle_compositionend(compositionevent);
+                    self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
+                }
+                event.mark_as_handled();
+            }
         }
     }
 }

components/script/dom/htmltextareaelement.rs

@@ -13,6 +13,7 @@ use crate::dom::bindings::error::ErrorResult;
 use crate::dom::bindings::inheritance::Castable;
 use crate::dom::bindings::root::{DomRoot, LayoutDom, MutNullableDom};
 use crate::dom::bindings::str::DOMString;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::document::Document;
 use crate::dom::element::RawLayoutElementHelpers;
 use crate::dom::element::{AttributeMutation, Element};
@@ -576,6 +577,22 @@ impl VirtualMethods for HTMLTextAreaElement {
                         &window,
                     );
             }
+        } else if event.type_() == atom!("compositionstart") ||
+            event.type_() == atom!("compositionupdate") ||
+            event.type_() == atom!("compositionend")
+        {
+            // TODO: Update DOM on start and continue
+            // and generally do proper CompositionEvent handling.
+            if let Some(compositionevent) = event.downcast::<CompositionEvent>() {
+                if event.type_() == atom!("compositionend") {
+                    let _ = self
+                        .textinput
+                        .borrow_mut()
+                        .handle_compositionend(compositionevent);
+                    self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
+                }
+                event.mark_as_handled();
+            }
         }
     }
 

components/script/script_thread.rs

@@ -113,7 +113,7 @@ use profile_traits::time::{self as profile_time, profile, ProfilerCategory};
 use script_layout_interface::message::{self, Msg, NewLayoutThreadInfo, ReflowGoal};
 use script_traits::webdriver_msg::WebDriverScriptCommand;
 use script_traits::CompositorEvent::{
-    KeyboardEvent, MouseButtonEvent, MouseMoveEvent, ResizeEvent, TouchEvent,
+    CompositionEvent, KeyboardEvent, MouseButtonEvent, MouseMoveEvent, ResizeEvent, TouchEvent,
 };
 use script_traits::{CompositorEvent, ConstellationControlMsg};
 use script_traits::{DiscardBrowsingContext, DocumentActivity, EventResult};
@@ -2881,6 +2881,14 @@ impl ScriptThread {
                 };
                 document.dispatch_key_event(key_event);
             },
+
+            CompositionEvent(composition_event) => {
+                let document = match { self.documents.borrow().find_document(pipeline_id) } {
+                    Some(document) => document,
+                    None => return warn!("Message sent to closed pipeline {}.", pipeline_id),
+                };
+                document.dispatch_composition_event(composition_event);
+            },
         }
     }
 

components/script/textinput.rs

@@ -6,6 +6,7 @@
 
 use crate::clipboard_provider::ClipboardProvider;
 use crate::dom::bindings::str::DOMString;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::keyboardevent::KeyboardEvent;
 use keyboard_types::{Key, KeyState, Modifiers, ShortcutMatcher};
 use std::borrow::ToOwned;
@@ -831,6 +832,11 @@ impl<T: ClipboardProvider> TextInput<T> {
             .unwrap()
     }
 
+    pub fn handle_compositionend(&mut self, event: &CompositionEvent) -> KeyReaction {
+        self.insert_string(event.data());
+        KeyReaction::DispatchInput
+    }
+
     /// Whether the content is empty.
     pub fn is_empty(&self) -> bool {
         self.lines.len() <= 1 && self.lines.get(0).map_or(true, |line| line.is_empty())

components/script_traits/lib.rs

@@ -30,7 +30,8 @@ use http::HeaderMap;
 use hyper::Method;
 use ipc_channel::ipc::{IpcReceiver, IpcSender};
 use ipc_channel::Error as IpcError;
-use keyboard_types::KeyboardEvent;
+use keyboard_types::webdriver::Event as WebDriverInputEvent;
+use keyboard_types::{CompositionEvent, KeyboardEvent};
 use libc::c_void;
 use msg::constellation_msg::{BrowsingContextId, HistoryStateId, PipelineId};
 use msg::constellation_msg::{PipelineNamespaceId, TopLevelBrowsingContextId, TraversalDirection};
@@ -461,6 +462,8 @@ pub enum CompositorEvent {
     ),
     /// A key was pressed.
     KeyboardEvent(KeyboardEvent),
+    /// An event from the IME is dispatched.
+    CompositionEvent(CompositionEvent),
 }
 
 /// Requests a TimerEvent-Message be sent after the given duration.
@@ -691,7 +694,7 @@ pub enum WebDriverCommandMsg {
     /// of a browsing context.
     ScriptCommand(BrowsingContextId, WebDriverScriptCommand),
     /// Act as if keys were pressed in the browsing context with the given ID.
-    SendKeys(BrowsingContextId, Vec<KeyboardEvent>),
+    SendKeys(BrowsingContextId, Vec<WebDriverInputEvent>),
     /// Set the window size.
     SetWindowSize(
         TopLevelBrowsingContextId,

components/webdriver_server/lib.rs

@@ -17,7 +17,7 @@ use euclid::TypedSize2D;
 use hyper::Method;
 use image::{DynamicImage, ImageFormat, RgbImage};
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
-use keyboard_types::webdriver::{send_keys, Event as KeyEvent};
+use keyboard_types::webdriver::send_keys;
 use msg::constellation_msg::{BrowsingContextId, TopLevelBrowsingContextId, TraversalDirection};
 use net_traits::image::base::PixelFormat;
 use regex::Captures;
@@ -1036,19 +1036,12 @@ impl Handler {
             ))
         })?;
 
-        // FIXME: Don't discard composition events.
-        let keys = send_keys(&keys.text)
-            .drain(..)
-            .filter_map(|event| match event {
-                KeyEvent::Keyboard(v) => Some(v),
-                _ => None,
-            })
-            .collect();
+        let input_events = send_keys(&keys.text);
 
         // TODO: there's a race condition caused by the focus command and the
         // send keys command being two separate messages,
         // so the constellation may have changed state between them.
-        let cmd_msg = WebDriverCommandMsg::SendKeys(browsing_context_id, keys);
+        let cmd_msg = WebDriverCommandMsg::SendKeys(browsing_context_id, input_events);
         self.constellation_chan
             .send(ConstellationMsg::WebDriverCommand(cmd_msg))
             .unwrap();