Auto merge of #22224 - pyfisch:composition-webdriver, r=nox

Forward WebDriver CompositionEvent

Dispatch composition events in JS.
Insert characters from composition events to text input.

CompositionEvents currently can only be
created by WebDriver and not by embedders.

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [ ] These changes fix #__ (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [ ] These changes do not require tests because _____

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22224)
<!-- Reviewable:end -->

Cargo.lock

@@ -539,7 +539,7 @@ dependencies = [
  "gleam 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "msg 0.0.1",
@@ -576,7 +576,7 @@ dependencies = [
  "gfx_traits 0.0.1",
  "http 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "layout_traits 0.0.1",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "metrics 0.0.1",
@@ -959,7 +959,7 @@ version = "0.0.1"
 dependencies = [
  "crossbeam-channel 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "msg 0.0.1",
@@ -1913,7 +1913,7 @@ dependencies = [
 
 [[package]]
 name = "keyboard-types"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2235,7 +2235,7 @@ dependencies = [
  "hashglobe 0.1.0",
  "hyper 0.12.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper_serde 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "mozjs 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "selectors 0.20.0",
  "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3226,7 +3226,7 @@ dependencies = [
  "ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "jstraceable_derive 0.0.1",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3329,7 +3329,7 @@ name = "script_tests"
 version = "0.0.1"
 dependencies = [
  "euclid 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "script 0.0.1",
  "servo_url 0.0.1",
 ]
@@ -3350,7 +3350,7 @@ dependencies = [
  "hyper 0.12.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper_serde 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)",
  "malloc_size_of 0.0.1",
  "malloc_size_of_derive 0.0.1",
@@ -3433,7 +3433,7 @@ dependencies = [
  "gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "gleam 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "glutin 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libservo 0.0.1",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -4496,7 +4496,7 @@ dependencies = [
  "hyper 0.12.12 (registry+https://github.com/rust-lang/crates.io-index)",
  "image 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "msg 0.0.1",
  "net_traits 0.0.1",
@@ -4951,7 +4951,7 @@ dependencies = [
 "checksum jni-sys 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
 "checksum jpeg-decoder 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "0dfe27a6c0dabd772d0f9b9f8701c4ca12c4d1eebcadf2be1f6f70396f6a1434"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
-"checksum keyboard-types 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "823bf0e5ec01b80424a318e79a0d1375725281acf311c47543ab3413f704dc25"
+"checksum keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "53b536dc22c0dabb295e85dbd0c062023885b12b8db24e1d86833f4e50ea7959"
 "checksum khronos_api 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "62237e6d326bd5871cd21469323bf096de81f1618cd82cbaf5d87825335aeb49"
 "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
 "checksum lazycell 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d33a48d0365c96081958cc663eef834975cb1e8d8bea3378513fc72bdbf11e50"

components/atoms/static_atoms.txt

@@ -13,6 +13,9 @@ click
 close
 color
 complete
+compositionend
+compositionstart
+compositionupdate
 controllerchange
 cursive
 date

components/constellation/constellation.rs

@@ -119,6 +119,7 @@ use gfx_traits::Epoch;
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use ipc_channel::router::ROUTER;
 use ipc_channel::Error as IpcError;
+use keyboard_types::webdriver::Event as WebDriverInputEvent;
 use keyboard_types::KeyboardEvent;
 use layout_traits::LayoutThreadFactory;
 use log::{Level, LevelFilter, Log, Metadata, Record};
@@ -3035,7 +3036,14 @@ where
                     None => return warn!("Pipeline {} SendKeys after closure.", pipeline_id),
                 };
                 for event in cmd {
-                    let event = CompositorEvent::KeyboardEvent(event);
+                    let event = match event {
+                        WebDriverInputEvent::Keyboard(event) => {
+                            CompositorEvent::KeyboardEvent(event)
+                        },
+                        WebDriverInputEvent::Composition(event) => {
+                            CompositorEvent::CompositionEvent(event)
+                        },
+                    };
                     let control_msg = ConstellationControlMsg::SendEvent(pipeline_id, event);
                     if let Err(e) = event_loop.send(control_msg) {
                         return self.handle_send_error(pipeline_id, e);

components/script/Cargo.toml

@@ -64,7 +64,7 @@ image = "0.20"
 ipc-channel = "0.11"
 itertools = "0.7.6"
 jstraceable_derive = {path = "../jstraceable_derive"}
-keyboard-types = "0.4.3"
+keyboard-types = "0.4.4"
 lazy_static = "1"
 libc = "0.2"
 log = "0.4"

components/script/dom/compositionevent.rs

@@ -59,6 +59,10 @@ impl CompositionEvent {
         );
         Ok(event)
     }
+
+    pub fn data(&self) -> &str {
+        &*self.data
+    }
 }
 
 impl CompositionEventMethods for CompositionEvent {

components/script/dom/document.rs

@@ -38,6 +38,7 @@ use crate::dom::bindings::xmlname::{
 };
 use crate::dom::closeevent::CloseEvent;
 use crate::dom::comment::Comment;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::cssstylesheet::CSSStyleSheet;
 use crate::dom::customelementregistry::CustomElementDefinition;
 use crate::dom::customevent::CustomEvent;
@@ -1465,6 +1466,37 @@ impl Document {
         self.window.reflow(ReflowGoal::Full, ReflowReason::KeyEvent);
     }
 
+    pub fn dispatch_composition_event(
+        &self,
+        composition_event: ::keyboard_types::CompositionEvent,
+    ) {
+        // spec: https://w3c.github.io/uievents/#compositionstart
+        // spec: https://w3c.github.io/uievents/#compositionupdate
+        // spec: https://w3c.github.io/uievents/#compositionend
+        // > Event.target : focused element processing the composition
+        let focused = self.get_focused_element();
+        let target = if let Some(elem) = &focused {
+            elem.upcast()
+        } else {
+            // Event is only dispatched if there is a focused element.
+            return;
+        };
+
+        let cancelable = composition_event.state == keyboard_types::CompositionState::Start;
+
+        let compositionevent = CompositionEvent::new(
+            &self.window,
+            DOMString::from(composition_event.state.to_string()),
+            true,
+            cancelable,
+            Some(&self.window),
+            0,
+            DOMString::from(composition_event.data),
+        );
+        let event = compositionevent.upcast::<Event>();
+        event.fire(target);
+    }
+
     // https://dom.spec.whatwg.org/#converting-nodes-into-a-node
     pub fn node_from_nodes_and_strings(
         &self,

components/script/dom/htmlinputelement.rs

@@ -17,6 +17,7 @@ use crate::dom::bindings::inheritance::Castable;
 use crate::dom::bindings::reflector::DomObject;
 use crate::dom::bindings::root::{Dom, DomRoot, LayoutDom, MutNullableDom, RootedReference};
 use crate::dom::bindings::str::DOMString;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::document::Document;
 use crate::dom::element::{
     AttributeMutation, Element, LayoutElementHelpers, RawLayoutElementHelpers,
@@ -1526,6 +1527,23 @@ impl VirtualMethods for HTMLInputElement {
                         &window,
                     );
             }
+        } else if (event.type_() == atom!("compositionstart") ||
+            event.type_() == atom!("compositionupdate") ||
+            event.type_() == atom!("compositionend")) &&
+            self.input_type().is_textual_or_password()
+        {
+            // TODO: Update DOM on start and continue
+            // and generally do proper CompositionEvent handling.
+            if let Some(compositionevent) = event.downcast::<CompositionEvent>() {
+                if event.type_() == atom!("compositionend") {
+                    let _ = self
+                        .textinput
+                        .borrow_mut()
+                        .handle_compositionend(compositionevent);
+                    self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
+                }
+                event.mark_as_handled();
+            }
         }
     }
 }

components/script/dom/htmltextareaelement.rs

@@ -13,6 +13,7 @@ use crate::dom::bindings::error::ErrorResult;
 use crate::dom::bindings::inheritance::Castable;
 use crate::dom::bindings::root::{DomRoot, LayoutDom, MutNullableDom};
 use crate::dom::bindings::str::DOMString;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::document::Document;
 use crate::dom::element::RawLayoutElementHelpers;
 use crate::dom::element::{AttributeMutation, Element};
@@ -576,6 +577,22 @@ impl VirtualMethods for HTMLTextAreaElement {
                         &window,
                     );
             }
+        } else if event.type_() == atom!("compositionstart") ||
+            event.type_() == atom!("compositionupdate") ||
+            event.type_() == atom!("compositionend")
+        {
+            // TODO: Update DOM on start and continue
+            // and generally do proper CompositionEvent handling.
+            if let Some(compositionevent) = event.downcast::<CompositionEvent>() {
+                if event.type_() == atom!("compositionend") {
+                    let _ = self
+                        .textinput
+                        .borrow_mut()
+                        .handle_compositionend(compositionevent);
+                    self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
+                }
+                event.mark_as_handled();
+            }
         }
     }
 

components/script/script_thread.rs

@@ -113,7 +113,7 @@ use profile_traits::time::{self as profile_time, profile, ProfilerCategory};
 use script_layout_interface::message::{self, Msg, NewLayoutThreadInfo, ReflowGoal};
 use script_traits::webdriver_msg::WebDriverScriptCommand;
 use script_traits::CompositorEvent::{
-    KeyboardEvent, MouseButtonEvent, MouseMoveEvent, ResizeEvent, TouchEvent,
+    CompositionEvent, KeyboardEvent, MouseButtonEvent, MouseMoveEvent, ResizeEvent, TouchEvent,
 };
 use script_traits::{CompositorEvent, ConstellationControlMsg};
 use script_traits::{DiscardBrowsingContext, DocumentActivity, EventResult};
@@ -2881,6 +2881,14 @@ impl ScriptThread {
                 };
                 document.dispatch_key_event(key_event);
             },
+
+            CompositionEvent(composition_event) => {
+                let document = match { self.documents.borrow().find_document(pipeline_id) } {
+                    Some(document) => document,
+                    None => return warn!("Message sent to closed pipeline {}.", pipeline_id),
+                };
+                document.dispatch_composition_event(composition_event);
+            },
         }
     }
 

components/script/textinput.rs

@@ -6,6 +6,7 @@
 
 use crate::clipboard_provider::ClipboardProvider;
 use crate::dom::bindings::str::DOMString;
+use crate::dom::compositionevent::CompositionEvent;
 use crate::dom::keyboardevent::KeyboardEvent;
 use keyboard_types::{Key, KeyState, Modifiers, ShortcutMatcher};
 use std::borrow::ToOwned;
@@ -831,6 +832,11 @@ impl<T: ClipboardProvider> TextInput<T> {
             .unwrap()
     }
 
+    pub fn handle_compositionend(&mut self, event: &CompositionEvent) -> KeyReaction {
+        self.insert_string(event.data());
+        KeyReaction::DispatchInput
+    }
+
     /// Whether the content is empty.
     pub fn is_empty(&self) -> bool {
         self.lines.len() <= 1 && self.lines.get(0).map_or(true, |line| line.is_empty())

components/script_traits/lib.rs

@@ -30,7 +30,8 @@ use http::HeaderMap;
 use hyper::Method;
 use ipc_channel::ipc::{IpcReceiver, IpcSender};
 use ipc_channel::Error as IpcError;
-use keyboard_types::KeyboardEvent;
+use keyboard_types::webdriver::Event as WebDriverInputEvent;
+use keyboard_types::{CompositionEvent, KeyboardEvent};
 use libc::c_void;
 use msg::constellation_msg::{BrowsingContextId, HistoryStateId, PipelineId};
 use msg::constellation_msg::{PipelineNamespaceId, TopLevelBrowsingContextId, TraversalDirection};
@@ -461,6 +462,8 @@ pub enum CompositorEvent {
     ),
     /// A key was pressed.
     KeyboardEvent(KeyboardEvent),
+    /// An event from the IME is dispatched.
+    CompositionEvent(CompositionEvent),
 }
 
 /// Requests a TimerEvent-Message be sent after the given duration.
@@ -691,7 +694,7 @@ pub enum WebDriverCommandMsg {
     /// of a browsing context.
     ScriptCommand(BrowsingContextId, WebDriverScriptCommand),
     /// Act as if keys were pressed in the browsing context with the given ID.
-    SendKeys(BrowsingContextId, Vec<KeyboardEvent>),
+    SendKeys(BrowsingContextId, Vec<WebDriverInputEvent>),
     /// Set the window size.
     SetWindowSize(
         TopLevelBrowsingContextId,

components/webdriver_server/lib.rs

@@ -17,7 +17,7 @@ use euclid::TypedSize2D;
 use hyper::Method;
 use image::{DynamicImage, ImageFormat, RgbImage};
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
-use keyboard_types::webdriver::{send_keys, Event as KeyEvent};
+use keyboard_types::webdriver::send_keys;
 use msg::constellation_msg::{BrowsingContextId, TopLevelBrowsingContextId, TraversalDirection};
 use net_traits::image::base::PixelFormat;
 use regex::Captures;
@@ -1036,19 +1036,12 @@ impl Handler {
             ))
         })?;
 
-        // FIXME: Don't discard composition events.
+        let input_events = send_keys(&keys.text);
-        let keys = send_keys(&keys.text)
-            .drain(..)
-            .filter_map(|event| match event {
-                KeyEvent::Keyboard(v) => Some(v),
-                _ => None,
-            })
-            .collect();
 
         // TODO: there's a race condition caused by the focus command and the
         // send keys command being two separate messages,
         // so the constellation may have changed state between them.
-        let cmd_msg = WebDriverCommandMsg::SendKeys(browsing_context_id, keys);
+        let cmd_msg = WebDriverCommandMsg::SendKeys(browsing_context_id, input_events);
         self.constellation_chan
             .send(ConstellationMsg::WebDriverCommand(cmd_msg))
             .unwrap();