UWP: support virtual keyboard

components/embedder_traits/lib.rs

@@ -19,7 +19,7 @@ use keyboard_types::KeyboardEvent;
 use msg::constellation_msg::{InputMethodType, PipelineId, TopLevelBrowsingContextId};
 use servo_url::ServoUrl;
 use std::fmt::{Debug, Error, Formatter};
-use webrender_api::units::{DeviceIntPoint, DeviceIntSize};
+use webrender_api::units::{DeviceIntPoint, DeviceIntRect, DeviceIntSize};
 
 pub use webxr_api::MainThreadWaker as EventLoopWaker;
 
@@ -197,7 +197,7 @@ pub enum EmbedderMsg {
     /// Open interface to request permission specified by prompt.
     PromptPermission(PermissionPrompt, IpcSender<PermissionRequest>),
     /// Request to present an IME to the user when an editable element is focused.
-    ShowIME(InputMethodType),
+    ShowIME(InputMethodType, Option<String>, DeviceIntRect),
     /// Request to hide the IME when the editable element is blurred.
     HideIME,
     /// Servo has shut down

components/script/dom/document.rs

@@ -15,6 +15,8 @@ use crate::dom::bindings::codegen::Bindings::DocumentBinding::{
 };
 use crate::dom::bindings::codegen::Bindings::EventBinding::EventBinding::EventMethods;
 use crate::dom::bindings::codegen::Bindings::HTMLIFrameElementBinding::HTMLIFrameElementBinding::HTMLIFrameElementMethods;
+use crate::dom::bindings::codegen::Bindings::HTMLInputElementBinding::HTMLInputElementMethods;
+use crate::dom::bindings::codegen::Bindings::HTMLTextAreaElementBinding::HTMLTextAreaElementMethods;
 use crate::dom::bindings::codegen::Bindings::NodeBinding::NodeMethods;
 use crate::dom::bindings::codegen::Bindings::NodeFilterBinding::NodeFilter;
 use crate::dom::bindings::codegen::Bindings::PerformanceBinding::PerformanceMethods;
@@ -67,7 +69,9 @@ use crate::dom::htmlheadelement::HTMLHeadElement;
 use crate::dom::htmlhtmlelement::HTMLHtmlElement;
 use crate::dom::htmliframeelement::HTMLIFrameElement;
 use crate::dom::htmlimageelement::HTMLImageElement;
+use crate::dom::htmlinputelement::HTMLInputElement;
 use crate::dom::htmlscriptelement::{HTMLScriptElement, ScriptResult};
+use crate::dom::htmltextareaelement::HTMLTextAreaElement;
 use crate::dom::htmltitleelement::HTMLTitleElement;
 use crate::dom::keyboardevent::KeyboardEvent;
 use crate::dom::location::Location;
@@ -113,7 +117,7 @@ use devtools_traits::ScriptToDevtoolsControlMsg;
 use dom_struct::dom_struct;
 use embedder_traits::EmbedderMsg;
 use encoding_rs::{Encoding, UTF_8};
-use euclid::default::Point2D;
+use euclid::default::{Point2D, Rect, Size2D};
 use html5ever::{LocalName, Namespace, QualName};
 use hyper_serde::Serde;
 use ipc_channel::ipc::{self, IpcSender};
@@ -168,6 +172,7 @@ use style::stylesheet_set::DocumentStylesheetSet;
 use style::stylesheets::{Origin, OriginSet, Stylesheet};
 use url::Host;
 use uuid::Uuid;
+use webrender_api::units::DeviceIntRect;
 
 /// The number of times we are allowed to see spurious `requestAnimationFrame()` calls before
 /// falling back to fake ones.
@@ -1105,7 +1110,23 @@ impl Document {
 
             // Notify the embedder to display an input method.
             if let Some(kind) = elem.input_method_type() {
-                self.send_to_embedder(EmbedderMsg::ShowIME(kind));
+                let rect = elem.upcast::<Node>().bounding_content_box_or_zero();
+                let rect = Rect::new(
+                    Point2D::new(rect.origin.x.to_px(), rect.origin.y.to_px()),
+                    Size2D::new(rect.size.width.to_px(), rect.size.height.to_px()),
+                );
+                let text = if let Some(input) = elem.downcast::<HTMLInputElement>() {
+                    Some((&input.Value()).to_string())
+                } else if let Some(textarea) = elem.downcast::<HTMLTextAreaElement>() {
+                    Some((&textarea.Value()).to_string())
+                } else {
+                    None
+                };
+                self.send_to_embedder(EmbedderMsg::ShowIME(
+                    kind,
+                    text,
+                    DeviceIntRect::from_untyped(&rect),
+                ));
             }
         }
     }

ports/libmlservo/src/lib.rs

@@ -14,10 +14,11 @@ use rust_webvr::api::MagicLeapVRService;
 use servo::euclid::Scale;
 use servo::keyboard_types::Key;
 use servo::servo_url::ServoUrl;
-use servo::webrender_api::units::{DevicePixel, DevicePoint, LayoutPixel};
+use servo::webrender_api::units::{DeviceIntRect, DevicePixel, DevicePoint, LayoutPixel};
 use simpleservo::{self, deinit, gl_glue, MouseButton, ServoGlue, SERVO};
 use simpleservo::{
-    Coordinates, EventLoopWaker, HostTrait, InitOptions, PromptResult, VRInitOptions,
+    Coordinates, EventLoopWaker, HostTrait, InitOptions, InputMethodType, PromptResult,
+    VRInitOptions,
 };
 use smallvec::SmallVec;
 use std::cell::Cell;
@@ -412,9 +413,20 @@ impl HostTrait for HostCallbacks {
         self.shut_down_complete.set(true);
     }
 
-    fn on_ime_state_changed(&self, show: bool) {
+    fn on_ime_show(
+        &self,
+        _input_type: InputMethodType,
+        _text: Option<String>,
+        _bounds: DeviceIntRect,
+    ) {
         if let Some(keyboard) = self.keyboard.0 {
-            keyboard(self.app, show)
+            keyboard(self.app, true)
+        }
+    }
+
+    fn on_ime_hide(&self) {
+        if let Some(keyboard) = self.keyboard.0 {
+            keyboard(self.app, false)
         }
     }
 

ports/libsimpleservo/api/src/lib.rs

@@ -11,7 +11,9 @@ pub use servo::config::prefs::{add_user_prefs, PrefValue};
 pub use servo::embedder_traits::{
     ContextMenuResult, MediaSessionPlaybackState, PermissionPrompt, PermissionRequest, PromptResult,
 };
+pub use servo::msg::constellation_msg::InputMethodType;
 pub use servo::script_traits::{MediaSessionActionType, MouseButton};
+pub use servo::webrender_api::units::DeviceIntRect;
 
 use getopts::Options;
 use ipc_channel::ipc::IpcSender;
@@ -133,7 +135,9 @@ pub trait HostTrait {
     /// Servo finished shutting down.
     fn on_shutdown_complete(&self);
     /// A text input is focused.
-    fn on_ime_state_changed(&self, show: bool);
+    fn on_ime_show(&self, input_type: InputMethodType, text: Option<String>, bounds: DeviceIntRect);
+    /// Input lost focus
+    fn on_ime_hide(&self);
     /// Gets sytem clipboard contents.
     fn get_clipboard_contents(&self) -> Option<String>;
     /// Sets system clipboard contents.
@@ -721,11 +725,13 @@ impl ServoGlue {
 
                     let _ = sender.send(result);
                 },
-                EmbedderMsg::ShowIME(..) => {
+                EmbedderMsg::ShowIME(kind, text, bounds) => {
-                    self.callbacks.host_callbacks.on_ime_state_changed(true);
+                    self.callbacks
+                        .host_callbacks
+                        .on_ime_show(kind, text, bounds);
                 },
                 EmbedderMsg::HideIME => {
-                    self.callbacks.host_callbacks.on_ime_state_changed(false);
+                    self.callbacks.host_callbacks.on_ime_hide();
                 },
                 EmbedderMsg::MediaSessionEvent(event) => {
                     match event {

ports/libsimpleservo/capi/src/lib.rs

@@ -20,8 +20,8 @@ use keyboard_types::Key;
 use log::LevelFilter;
 use simpleservo::{self, gl_glue, ServoGlue, SERVO};
 use simpleservo::{
-    ContextMenuResult, Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionActionType,
+    ContextMenuResult, Coordinates, DeviceIntRect, EventLoopWaker, HostTrait, InitOptions,
-    MediaSessionPlaybackState, MouseButton, PromptResult,
+    InputMethodType, MediaSessionActionType, MediaSessionPlaybackState, MouseButton, PromptResult,
 };
 use std::ffi::{CStr, CString};
 #[cfg(target_os = "windows")]
@@ -212,7 +212,8 @@ pub struct CHostCallbacks {
     pub on_history_changed: extern "C" fn(can_go_back: bool, can_go_forward: bool),
     pub on_animating_changed: extern "C" fn(animating: bool),
     pub on_shutdown_complete: extern "C" fn(),
-    pub on_ime_state_changed: extern "C" fn(show: bool),
+    pub on_ime_show: extern "C" fn(text: *const c_char, x: i32, y: i32, width: i32, height: i32),
+    pub on_ime_hide: extern "C" fn(),
     pub get_clipboard_contents: extern "C" fn() -> *const c_char,
     pub set_clipboard_contents: extern "C" fn(contents: *const c_char),
     pub on_media_session_metadata:
@@ -834,9 +835,30 @@ impl HostTrait for HostCallbacks {
         (self.0.on_shutdown_complete)();
     }
 
-    fn on_ime_state_changed(&self, show: bool) {
+    fn on_ime_show(
-        debug!("on_ime_state_changed");
+        &self,
-        (self.0.on_ime_state_changed)(show);
+        _input_type: InputMethodType,
+        text: Option<String>,
+        bounds: DeviceIntRect,
+    ) {
+        debug!("on_ime_show");
+        let text = text.and_then(|s| CString::new(s).ok());
+        let text_ptr = text
+            .as_ref()
+            .map(|cstr| cstr.as_ptr())
+            .unwrap_or(std::ptr::null());
+        (self.0.on_ime_show)(
+            text_ptr,
+            bounds.origin.x,
+            bounds.origin.y,
+            bounds.size.width,
+            bounds.size.height,
+        );
+    }
+
+    fn on_ime_hide(&self) {
+        debug!("on_ime_hide");
+        (self.0.on_ime_hide)();
     }
 
     fn get_clipboard_contents(&self) -> Option<String> {

ports/libsimpleservo/jniapi/src/lib.rs

@@ -14,10 +14,11 @@ use jni::sys::{jboolean, jfloat, jint, jstring, JNI_TRUE};
 use jni::{errors, JNIEnv, JavaVM};
 use libc::{dup2, pipe, read};
 use log::Level;
+use simpleservo::{self, deinit, gl_glue, MouseButton, ServoGlue, SERVO};
 use simpleservo::{self, gl_glue, ServoGlue, SERVO};
 use simpleservo::{
-    Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionPlaybackState, PromptResult,
+    Coordinates, DeviceIntRect, EventLoopWaker, HostTrait, InitOptions, InputMethodType,
-    VRInitOptions,
+    MediaSessionPlaybackState, PromptResult, VRInitOptions,
 };
 use std::os::raw::{c_char, c_int, c_void};
 use std::ptr::{null, null_mut};
@@ -529,7 +530,8 @@ impl HostTrait for HostCallbacks {
         .unwrap();
     }
 
-    fn on_ime_state_changed(&self, _show: bool) {}
+    fn on_ime_show(&self, _type: InputEncoding, _text: Option<String>, _rect: DeviceIntRect) {}
+    fn on_ime_hide(&self) {}
 
     fn get_clipboard_contents(&self) -> Option<String> {
         None

ports/winit/browser.rs

@@ -498,7 +498,7 @@ where
                     let permission_state = prompt_user(prompt);
                     let _ = sender.send(permission_state);
                 }
-                EmbedderMsg::ShowIME(_kind) => {
+                EmbedderMsg::ShowIME(_kind, _text, _rect) => {
                     debug!("ShowIME received");
                 },
                 EmbedderMsg::HideIME => {

support/hololens/ServoApp/ServoControl/Servo.cpp

@@ -43,10 +43,17 @@ void on_panic(const char *backtrace) {
   throw hresult_error(E_FAIL, char2hstring(backtrace));
 }
 
-void on_ime_state_changed(bool aShow) {
+void on_ime_show(const char *text, int32_t x, int32_t y, int32_t width,
-  sServo->Delegate().OnServoIMEStateChanged(aShow);
+                 int32_t height) {
+  hstring htext = L"";
+  if (text != nullptr) {
+    htext = char2hstring(text);
+  }
+  sServo->Delegate().OnServoIMEShow(htext, x, y, width, height);
 }
 
+void on_ime_hide() { sServo->Delegate().OnServoIMEHide(); }
+
 void set_clipboard_contents(const char *) {
   // FIXME
 }
@@ -256,7 +263,8 @@ Servo::Servo(hstring args, GLsizei width, GLsizei height,
   c.on_animating_changed = &on_animating_changed;
   c.on_shutdown_complete = &on_shutdown_complete;
   c.on_allow_navigation = &on_allow_navigation;
-  c.on_ime_state_changed = &on_ime_state_changed;
+  c.on_ime_show = &on_ime_show;
+  c.on_ime_hide = &on_ime_hide;
   c.get_clipboard_contents = &get_clipboard_contents;
   c.set_clipboard_contents = &set_clipboard_contents;
   c.on_media_session_metadata = &on_media_session_metadata;

support/hololens/ServoApp/ServoControl/Servo.h

@@ -109,7 +109,9 @@ public:
   virtual void OnServoURLChanged(hstring) = 0;
   virtual bool OnServoAllowNavigation(hstring) = 0;
   virtual void OnServoAnimatingChanged(bool) = 0;
-  virtual void OnServoIMEStateChanged(bool) = 0;
+  virtual void OnServoIMEShow(hstring text, int32_t x, int32_t y, int32_t width,
+                              int32_t height) = 0;
+  virtual void OnServoIMEHide() = 0;
   virtual void OnServoDevtoolsStarted(bool, const unsigned int, hstring) = 0;
   virtual void OnServoMediaSessionMetadata(hstring, hstring, hstring) = 0;
   virtual void OnServoMediaSessionPlaybackStateChange(int) = 0;

support/hololens/ServoApp/ServoControl/ServoControl.cpp

@@ -94,26 +94,37 @@ void ServoControl::InitializeTextController() {
   auto manager = CoreTextServicesManager::GetForCurrentView();
   mEditContext = manager.CreateEditContext();
   mEditContext->InputPaneDisplayPolicy(CoreTextInputPaneDisplayPolicy::Manual);
-  mEditContext->InputScope(CoreTextInputScope::Text);
+
   mEditContext->TextRequested([=](const auto &, const auto &e) {
-    e.Request().Text(L"");
+    e.Request().Text(*mFocusedInputText);
-    CoreTextRange sel;
+  });
-    sel.StartCaretPosition = 0;
+
-    sel.EndCaretPosition = 0;
+  mEditContext->SelectionRequested([=](const auto &, const auto &) {});
-    e.Request().Range() = sel;
+
+  mEditContext->LayoutRequested([=](const auto &, const auto &e) {
+    // Necessary to show the preview
+    e.Request().LayoutBounds().TextBounds(*mFocusedInputRect);
+    e.Request().LayoutBounds().ControlBounds(*mFocusedInputRect);
   });
 
   mEditContext->TextUpdating([=](const auto &, const auto &e) {
     RunOnGLThread([=] {
-      auto keystr = *hstring2char((e.Text()));
+      auto text = *hstring2char(e.Text());
-      mServo->KeyDown(keystr);
+      size_t size = strlen(text);
+      for (int i = 0; i < size; i++) {
+        char letter[2];
+        memcpy(letter, &text[i], 1);
+        letter[1] = '\0';
+        mServo->KeyDown(letter);
+        mServo->KeyUp(letter);
+      }
     });
+    e.Result(CoreTextTextUpdatingResult::Succeeded);
   });
 
-  mEditContext->SelectionRequested([](const auto &, const auto &) {});
-
   GotFocus(
       [=](const auto &, const auto &) { mEditContext->NotifyFocusEnter(); });
+
   LostFocus(
       [=](const auto &, const auto &) { mEditContext->NotifyFocusLeave(); });
 
@@ -126,6 +137,7 @@ void ServoControl::InitializeTextController() {
       });
     }
   });
+
   PreviewKeyUp([=](const auto &, const auto &e) {
     auto keystr = KeyToString(e.Key());
     if (keystr.has_value()) {
@@ -489,13 +501,24 @@ void ServoControl::OnServoAnimatingChanged(bool animating) {
   WakeConditionVariable(&mGLCondVar);
 }
 
-void ServoControl::OnServoIMEStateChanged(bool focused) {
+void ServoControl::OnServoIMEHide() {
+  RunOnUIThread([=] { mInputPane->TryHide(); });
+}
+
+void ServoControl::OnServoIMEShow(hstring text, int32_t x, int32_t y,
+                                  int32_t width, int32_t height) {
   RunOnUIThread([=] {
-    if (focused) {
+    mEditContext->NotifyFocusEnter();
-      mInputPane->TryShow();
+    // FIXME: The simpleservo on_ime_show callback comes with a input method
-    } else {
+    // type parameter that could be used to set the input scope here.
-      mInputPane->TryHide();
+    mEditContext->InputScope(CoreTextInputScope::Text);
-    }
+    // offset of the Servo SwapChainPanel.
+    auto transform = Panel().TransformToVisual(Window::Current().Content());
+    auto offset = transform.TransformPoint(Point(0, 0));
+    mFocusedInputRect =
+        Rect(x + offset.X, y + offset.Y, (float)width, (float)height);
+    mFocusedInputText = text;
+    mInputPane->TryShow();
   });
 }
 

support/hololens/ServoApp/ServoControl/ServoControl.h

@@ -173,7 +173,8 @@ struct ServoControl : ServoControlT<ServoControl>, public servo::ServoDelegate {
   virtual void OnServoURLChanged(winrt::hstring);
   virtual bool OnServoAllowNavigation(winrt::hstring);
   virtual void OnServoAnimatingChanged(bool);
-  virtual void OnServoIMEStateChanged(bool);
+  virtual void OnServoIMEHide();
+  virtual void OnServoIMEShow(hstring text, int32_t, int32_t, int32_t, int32_t);
   virtual void OnServoMediaSessionMetadata(winrt::hstring, winrt::hstring,
                                            winrt::hstring);
   virtual void OnServoMediaSessionPlaybackStateChange(int);
@@ -281,6 +282,9 @@ private:
 
   std::optional<Windows::UI::Text::Core::CoreTextEditContext> mEditContext;
   std::optional<Windows::UI::ViewManagement::InputPane> mInputPane;
+
+  std::optional<Windows::Foundation::Rect> mFocusedInputRect;
+  std::optional<hstring> mFocusedInputText;
 };
 } // namespace winrt::ServoApp::implementation