Auto merge of #27114 - paulrouget:key_events, r=jdm

[UWP] Key events

This is the initial work required for proper keyboard events.
The text controller implementation is very basic, just enough to show the virtual keyboard when it's needed, and have basic key events.

Cargo.lock

@@ -2736,9 +2736,9 @@ dependencies = [
 
 [[package]]
 name = "keyboard-types"
-version = "0.4.4"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53b536dc22c0dabb295e85dbd0c062023885b12b8db24e1d86833f4e50ea7959"
+checksum = "a989afac88279b0482f402d234b5fbd405bf1ad051308595b58de4e6de22346b"
 dependencies = [
  "bitflags",
  "serde",
@@ -5224,6 +5224,7 @@ dependencies = [
  "backtrace",
  "cbindgen",
  "env_logger",
+ "keyboard-types",
  "lazy_static",
  "libc",
  "log",

components/compositing/Cargo.toml

@@ -24,7 +24,7 @@ gfx_traits = { path = "../gfx_traits" }
 gleam = { version = "0.11", optional = true }
 image = "0.23"
 ipc-channel = "0.14"
-keyboard-types = "0.4.3"
+keyboard-types = "0.5"
 libc = "0.2"
 log = "0.4"
 msg = { path = "../msg" }

components/constellation/Cargo.toml

@@ -25,7 +25,7 @@ gfx = { path = "../gfx" }
 gfx_traits = { path = "../gfx_traits" }
 http = "0.1"
 ipc-channel = "0.14"
-keyboard-types = "0.4.3"
+keyboard-types = "0.5"
 layout_traits = { path = "../layout_traits" }
 log = "0.4"
 media = { path = "../media" }

components/embedder_traits/Cargo.toml

@@ -13,7 +13,7 @@ path = "lib.rs"
 [dependencies]
 crossbeam-channel = "0.4"
 ipc-channel = "0.14"
-keyboard-types = "0.4.3"
+keyboard-types = "0.5"
 lazy_static = "1"
 log = "0.4"
 msg = { path = "../msg" }

components/malloc_size_of/Cargo.toml

@@ -36,7 +36,7 @@ euclid = "0.20"
 hashglobe = { path = "../hashglobe" }
 hyper = { version = "0.12", optional = true }
 hyper_serde = { version = "0.11", optional = true }
-keyboard-types = { version = "0.4.3", optional = true }
+keyboard-types = { version = "0.5", optional = true }
 selectors = { path = "../selectors" }
 serde = { version = "1.0.27", optional = true }
 serde_bytes = { version = "0.11", optional = true }

components/script/Cargo.toml

@@ -64,7 +64,7 @@ ipc-channel = "0.14"
 itertools = "0.8"
 js = { package = "mozjs", git = "https://github.com/servo/rust-mozjs" }
 jstraceable_derive = { path = "../jstraceable_derive" }
-keyboard-types = "0.4.4"
+keyboard-types = "0.5"
 lazy_static = "1"
 libc = "0.2"
 log = "0.4"

components/script_traits/Cargo.toml

@@ -24,7 +24,7 @@ http = "0.1"
 hyper = "0.12"
 hyper_serde = "0.11"
 ipc-channel = "0.14"
-keyboard-types = "0.4.3"
+keyboard-types = "0.5"
 libc = "0.2"
 log = "0.4"
 malloc_size_of = { path = "../malloc_size_of" }

components/servo/Cargo.toml

@@ -57,7 +57,7 @@ gfx_traits = { path = "../gfx_traits" }
 gleam = "0.11"
 gstreamer = { version = "0.15", optional = true }
 ipc-channel = "0.14"
-keyboard-types = "0.4"
+keyboard-types = "0.5"
 layout_thread_2013 = { path = "../layout_thread", optional = true }
 layout_thread_2020 = { path = "../layout_thread_2020", optional = true }
 log = "0.4"

components/webdriver_server/Cargo.toml

@@ -19,7 +19,7 @@ euclid = "0.20"
 hyper = "0.12"
 image = "0.23"
 ipc-channel = "0.14"
-keyboard-types = "0.4.3"
+keyboard-types = "0.5"
 log = "0.4"
 msg = { path = "../msg" }
 net_traits = { path = "../net_traits" }

ports/libsimpleservo/capi/Cargo.toml

@@ -19,6 +19,7 @@ lazy_static = "1"
 log = "0.4"
 simpleservo = { path = "../api" }
 surfman = "0.3"
+keyboard-types = "0.5"
 
 [target.'cfg(target_os = "windows")'.dependencies]
 libc = "0.2"

ports/libsimpleservo/capi/src/lib.rs

@@ -16,6 +16,7 @@ mod vslogger;
 use backtrace::Backtrace;
 #[cfg(not(target_os = "windows"))]
 use env_logger;
+use keyboard_types::Key;
 use log::LevelFilter;
 use simpleservo::{self, gl_glue, ServoGlue, SERVO};
 use simpleservo::{
@@ -717,6 +718,37 @@ pub extern "C" fn click(x: f32, y: f32) {
     });
 }
 
+#[no_mangle]
+pub extern "C" fn key_down(name: *const c_char) {
+    debug!("key_up");
+    key_event(name, false);
+}
+
+#[no_mangle]
+pub extern "C" fn key_up(name: *const c_char) {
+    debug!("key_up");
+    key_event(name, true);
+}
+
+fn key_event(name: *const c_char, up: bool) {
+    catch_any_panic(|| {
+        let name = unsafe { CStr::from_ptr(name) };
+        let name = match name.to_str() {
+            Ok(name) => name,
+            Err(..) => {
+                warn!("Couldn't not read str");
+                return;
+            },
+        };
+        let key = Key::from_str(&name);
+        if let Ok(key) = key {
+            call(|s| if up { s.key_up(key) } else { s.key_down(key) });
+        } else {
+            warn!("Received unknown keys");
+        }
+    });
+}
+
 #[no_mangle]
 pub extern "C" fn media_session_action(action: CMediaSessionActionType) {
     catch_any_panic(|| {

ports/winit/Cargo.toml

@@ -50,7 +50,7 @@ backtrace = "0.3"
 clipboard = "0.5"
 euclid = "0.20"
 getopts = "0.2.11"
-keyboard-types = "0.4.3"
+keyboard-types = "0.5"
 lazy_static = "1"
 libc = "0.2"
 libservo = { path = "../../components/servo" }

support/hololens/ServoApp/ServoApp.vcxproj

@@ -120,6 +120,7 @@
   </ItemDefinitionGroup>
   <ItemGroup>
     <ClInclude Include="Devtools\Client.h" />
+    <ClInclude Include="ServoControl\Keys.h" />
     <ClInclude Include="strutils.h" />
     <ClInclude Include="pch.h" />
     <ClInclude Include="App.h">

support/hololens/ServoApp/ServoApp.vcxproj.filters

@@ -43,6 +43,9 @@
     <ClInclude Include="Devtools\Client.h">
       <Filter>Devtools</Filter>
     </ClInclude>
+    <ClInclude Include="ServoControl\Keys.h">
+      <Filter>ServoControl</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Image Include="Assets\Wide310x150Logo.scale-200.png">

support/hololens/ServoApp/ServoControl/Keys.h

@@ -0,0 +1,130 @@
+using namespace winrt::Windows::System;
+
+namespace winrt {
+std::optional<const char *> KeyToString(Windows::System::VirtualKey key) {
+  switch (key) {
+  case VirtualKey::F1:
+    return "F1";
+  case VirtualKey::F2:
+    return "F2";
+  case VirtualKey::F3:
+    return "F3";
+  case VirtualKey::F4:
+    return "F4";
+  case VirtualKey::F5:
+    return "F5";
+  case VirtualKey::F6:
+    return "F6";
+  case VirtualKey::F7:
+    return "F7";
+  case VirtualKey::F8:
+    return "F8";
+  case VirtualKey::F9:
+    return "F9";
+  case VirtualKey::F10:
+    return "F10";
+  case VirtualKey::F11:
+    return "F11";
+  case VirtualKey::F12:
+    return "F12";
+  case VirtualKey::Shift:
+    return "Shift";
+  case VirtualKey::LeftShift:
+    return "Shift";
+  case VirtualKey::RightShift:
+    return "Shift";
+  case VirtualKey::Control:
+    return "Control";
+  case VirtualKey::LeftControl:
+    return "Control";
+  case VirtualKey::RightControl:
+    return "Control";
+  case VirtualKey::Escape:
+    return "Escape";
+  case VirtualKey::Enter:
+    return "Enter";
+  case VirtualKey::Tab:
+    return "Tab";
+  case VirtualKey::Delete:
+    return "Delete";
+  case VirtualKey::Back:
+    return "Backspace";
+  case VirtualKey::GoForward:
+    return "BrowserForward";
+  case VirtualKey::GoBack:
+    return "BrowserBack";
+  case VirtualKey::GoHome:
+    return "BrowserHome";
+  case VirtualKey::Favorites:
+    return "BrowserFavorites";
+  case VirtualKey::Search:
+    return "BrowserSearch";
+  case VirtualKey::Stop:
+    return "BrowserStop";
+  case VirtualKey::Menu:
+    return "Alt";
+  case VirtualKey::RightMenu:
+    return "Alt";
+  case VirtualKey::LeftMenu:
+    return "Alt";
+  case VirtualKey::CapitalLock:
+    return "CapsLock";
+  case VirtualKey::LeftWindows:
+    return "Meta";
+  case VirtualKey::RightWindows:
+    return "Meta";
+  case VirtualKey::NumberKeyLock:
+    return "NumLock";
+  case VirtualKey::Scroll:
+    return "ScrollLock";
+  case VirtualKey::Down:
+    return "ArrowDown";
+  case VirtualKey::Up:
+    return "ArrowUp";
+  case VirtualKey::Left:
+    return "ArrowLeft";
+  case VirtualKey::Right:
+    return "ArrowRight";
+  case VirtualKey::End:
+    return "End";
+  case VirtualKey::Home:
+    return "Home";
+  case VirtualKey::PageDown:
+    return "PageHome";
+  case VirtualKey::PageUp:
+    return "PageUp";
+  case VirtualKey::Clear:
+    return "Clear";
+  case VirtualKey::Insert:
+    return "Insert";
+  case VirtualKey::Accept:
+    return "Accept";
+  case VirtualKey::Cancel:
+    return "Cancel";
+  case VirtualKey::Execute:
+    return "Execute";
+  case VirtualKey::Help:
+    return "Help";
+  case VirtualKey::Pause:
+    return "Pause";
+  case VirtualKey::Select:
+    return "Select";
+  case VirtualKey::Print:
+    return "Print";
+  case VirtualKey::Convert:
+    return "Convert";
+  case VirtualKey::NonConvert:
+    return "NonConvert";
+  case VirtualKey::ModeChange:
+    return "ModeChange";
+  case VirtualKey::Hangul:
+    return "HangulMode";
+  case VirtualKey::Hanja:
+    return "HanjaMode";
+  case VirtualKey::Junja:
+    return "JunjaMode";
+  default:
+    return {};
+  }
+}
+} // namespace winrt

support/hololens/ServoApp/ServoControl/Servo.h

@@ -62,6 +62,8 @@ public:
   void TouchMove(float x, float y, int32_t id) { touch_move(x, y, id); }
   void TouchCancel(float x, float y, int32_t id) { touch_cancel(x, y, id); }
   void MouseMove(float x, float y) { mouse_move(x, y); }
+  void KeyDown(const char *k) { key_down(k); }
+  void KeyUp(const char *k) { key_up(k); }
 
   void Reload() { reload(); }
   void Stop() { stop(); }

support/hololens/ServoApp/ServoControl/ServoControl.cpp

@@ -4,6 +4,7 @@
 #include "ServoControl.g.cpp"
 #include "Pref.g.cpp"
 #include <stdlib.h>
+#include "Keys.h"
 
 using namespace std::placeholders;
 using namespace winrt::Windows::ApplicationModel::Resources;
@@ -11,6 +12,7 @@ using namespace winrt::Windows::Graphics::Display;
 using namespace winrt::Windows::UI::Xaml;
 using namespace winrt::Windows::UI::Popups;
 using namespace winrt::Windows::UI::Core;
+using namespace winrt::Windows::UI::Text::Core;
 using namespace winrt::Windows::Foundation;
 using namespace winrt::Windows::System;
 using namespace winrt::Windows::Devices::Input;
@@ -66,21 +68,19 @@ void ServoControl::OnLoaded(IInspectable const &, RoutedEventArgs const &) {
       std::bind(&ServoControl::OnSurfacePointerMoved, this, _1, _2));
   panel.PointerWheelChanged(
       std::bind(&ServoControl::OnSurfaceWheelChanged, this, _1, _2));
-  panel.ManipulationStarted(
-      [=](IInspectable const &,
-          Input::ManipulationStartedRoutedEventArgs const &e) {
-        mOnCaptureGesturesStartedEvent();
-        e.Handled(true);
-      });
-  panel.ManipulationCompleted(
-      [=](IInspectable const &,
-          Input::ManipulationCompletedRoutedEventArgs const &e) {
-        mOnCaptureGesturesEndedEvent();
-        e.Handled(true);
-      });
+  panel.ManipulationStarted([=](const auto &, const auto &e) {
+    mOnCaptureGesturesStartedEvent();
+    e.Handled(true);
+  });
+  panel.ManipulationCompleted([=](const auto &, const auto &e) {
+    mOnCaptureGesturesEndedEvent();
+    e.Handled(true);
+  });
   panel.ManipulationDelta(
       std::bind(&ServoControl::OnSurfaceManipulationDelta, this, _1, _2));
-  Panel().SizeChanged(std::bind(&ServoControl::OnSurfaceResized, this, _1, _2));
+  panel.SizeChanged(std::bind(&ServoControl::OnSurfaceResized, this, _1, _2));
+
+  InitializeTextController();
   InitializeConditionVariable(&mGLCondVar);
   InitializeCriticalSection(&mGLLock);
   InitializeConditionVariable(&mDialogCondVar);
@@ -89,6 +89,54 @@ void ServoControl::OnLoaded(IInspectable const &, RoutedEventArgs const &) {
   StartRenderLoop();
 }
 
+void ServoControl::InitializeTextController() {
+  mInputPane = Windows::UI::ViewManagement::InputPane::GetForCurrentView();
+  auto manager = CoreTextServicesManager::GetForCurrentView();
+  mEditContext = manager.CreateEditContext();
+  mEditContext->InputPaneDisplayPolicy(CoreTextInputPaneDisplayPolicy::Manual);
+  mEditContext->InputScope(CoreTextInputScope::Text);
+  mEditContext->TextRequested([=](const auto &, const auto &e) {
+    e.Request().Text(L"");
+    CoreTextRange sel;
+    sel.StartCaretPosition = 0;
+    sel.EndCaretPosition = 0;
+    e.Request().Range() = sel;
+  });
+
+  mEditContext->TextUpdating([=](const auto &, const auto &e) {
+    RunOnGLThread([=] {
+      auto keystr = *hstring2char((e.Text()));
+      mServo->KeyDown(keystr);
+    });
+  });
+
+  mEditContext->SelectionRequested([](const auto &, const auto &) {});
+
+  GotFocus(
+      [=](const auto &, const auto &) { mEditContext->NotifyFocusEnter(); });
+  LostFocus(
+      [=](const auto &, const auto &) { mEditContext->NotifyFocusLeave(); });
+
+  PreviewKeyDown([=](const auto &, const auto &e) {
+    auto keystr = KeyToString(e.Key());
+    if (keystr.has_value()) {
+      RunOnGLThread([=] {
+        auto keyname = *keystr;
+        mServo->KeyDown(keyname);
+      });
+    }
+  });
+  PreviewKeyUp([=](const auto &, const auto &e) {
+    auto keystr = KeyToString(e.Key());
+    if (keystr.has_value()) {
+      RunOnGLThread([=] {
+        auto keyname = *keystr;
+        mServo->KeyUp(keyname);
+      });
+    }
+  });
+}
+
 Controls::SwapChainPanel ServoControl::Panel() {
   return GetTemplateChild(L"swapChainPanel").as<Controls::SwapChainPanel>();
 }
@@ -128,6 +176,7 @@ void ServoControl::OnSurfaceManipulationDelta(
 
 void ServoControl::OnSurfaceTapped(IInspectable const &,
                                    Input::TappedRoutedEventArgs const &e) {
+  Focus(FocusState::Programmatic);
   if (e.PointerDeviceType() == PointerDeviceType::Mouse) {
     auto coords = e.GetPosition(Panel());
     auto x = coords.X * mDPI;
@@ -440,9 +489,14 @@ void ServoControl::OnServoAnimatingChanged(bool animating) {
   WakeConditionVariable(&mGLCondVar);
 }
 
-void ServoControl::OnServoIMEStateChanged(bool) {
-  // FIXME:
-  // https://docs.microsoft.com/en-us/windows/win32/winauto/uiauto-implementingtextandtextrange
+void ServoControl::OnServoIMEStateChanged(bool focused) {
+  RunOnUIThread([=] {
+    if (focused) {
+      mInputPane->TryShow();
+    } else {
+      mInputPane->TryHide();
+    }
+  });
 }
 
 void ServoControl::OnServoMediaSessionMetadata(hstring title, hstring artist,

support/hololens/ServoApp/ServoControl/ServoControl.h

@@ -264,6 +264,7 @@ private:
   void RunOnGLThread(std::function<void()>);
 
   void TryLoadUri(hstring);
+  void InitializeTextController();
 
   std::unique_ptr<servo::Servo> mServo;
   PropertySet mNativeWindowProperties;
@@ -277,6 +278,9 @@ private:
   hstring mArgs;
   std::optional<servo::Servo::MouseButton> mPressedMouseButton = {};
   std::unique_ptr<L10NStrings> mL10NStrings = nullptr;
+
+  std::optional<Windows::UI::Text::Core::CoreTextEditContext> mEditContext;
+  std::optional<Windows::UI::ViewManagement::InputPane> mInputPane;
 };
 } // namespace winrt::ServoApp::implementation
 

support/hololens/ServoApp/pch.h

@@ -60,6 +60,7 @@
 #include <winrt/Windows.UI.Xaml.Shapes.h>
 #include <winrt/Windows.UI.Notifications.h>
 #include <winrt/Windows.UI.Text.h>
+#include <winrt/Windows.UI.Text.Core.h>
 #include <winrt/Windows.Data.Xml.Dom.h>
 #include <winrt/Windows.Storage.Streams.h>
 #include <winrt/Windows.Data.Json.h>

tests/unit/script/Cargo.toml

@@ -11,6 +11,6 @@ path = "lib.rs"
 
 [dependencies]
 euclid = "0.20"
-keyboard-types = "0.4.3"
+keyboard-types = "0.5"
 script = {path = "../../../components/script"}
 servo_url = {path = "../../../components/url"}