Auto merge of #24927 - ferjm:hololens.mediasession, r=paulrouget

Basic Media Session UI for Hololens

- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes fix #24807

This PR implements a extremely basic Media Session UI. The media controls are part of the main window, which is not very nice, but it is what we can do until #24873 is done.

ports/libsimpleservo/api/src/lib.rs

@@ -7,7 +7,8 @@ extern crate log;
 
 pub mod gl_glue;
 
-pub use servo::script_traits::MouseButton;
+pub use servo::embedder_traits::MediaSessionPlaybackState;
+pub use servo::script_traits::{MediaSessionActionType, MouseButton};
 
 use getopts::Options;
 use servo::compositing::windowing::{
@@ -133,7 +134,7 @@ pub trait HostTrait {
     /// Called when we get the media session metadata/
     fn on_media_session_metadata(&self, title: String, artist: String, album: String);
     /// Called when the media session playback state changes.
-    fn on_media_session_playback_state_change(&self, state: i32);
+    fn on_media_session_playback_state_change(&self, state: MediaSessionPlaybackState);
     /// Called when the media session position state is set.
     fn on_media_session_set_position_state(&self, duration: f64, position: f64, playback_rate: f64);
 }
@@ -472,9 +473,12 @@ impl ServoGlue {
         self.process_event(WindowEvent::Keyboard(key_event))
     }
 
-    pub fn media_session_action(&mut self, action: i32) -> Result<(), &'static str> {
+    pub fn media_session_action(
+        &mut self,
+        action: MediaSessionActionType,
+    ) -> Result<(), &'static str> {
         info!("Media session action {:?}", action);
-        self.process_event(WindowEvent::MediaSessionAction(action.into()))
+        self.process_event(WindowEvent::MediaSessionAction(action))
     }
 
     fn process_event(&mut self, event: WindowEvent) -> Result<(), &'static str> {
@@ -595,7 +599,7 @@ impl ServoGlue {
                         MediaSessionEvent::PlaybackStateChange(state) => self
                             .callbacks
                             .host_callbacks
-                            .on_media_session_playback_state_change(state as i32),
+                            .on_media_session_playback_state_change(state),
                         MediaSessionEvent::SetPositionState(position_state) => self
                             .callbacks
                             .host_callbacks

ports/libsimpleservo/capi/src/lib.rs

@@ -17,7 +17,8 @@ use env_logger;
 use log::LevelFilter;
 use simpleservo::{self, gl_glue, ServoGlue, SERVO};
 use simpleservo::{
-    Coordinates, EventLoopWaker, HostTrait, InitOptions, MouseButton, VRInitOptions,
+    Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionActionType,
+    MediaSessionPlaybackState, MouseButton, VRInitOptions,
 };
 use std::ffi::{CStr, CString};
 #[cfg(target_os = "windows")]
@@ -218,7 +219,7 @@ pub struct CHostCallbacks {
     pub set_clipboard_contents: extern "C" fn(contents: *const c_char),
     pub on_media_session_metadata:
         extern "C" fn(title: *const c_char, album: *const c_char, artist: *const c_char),
-    pub on_media_session_playback_state_change: extern "C" fn(state: i32),
+    pub on_media_session_playback_state_change: extern "C" fn(state: CMediaSessionPlaybackState),
     pub on_media_session_set_position_state:
         extern "C" fn(duration: f64, position: f64, playback_rate: f64),
 }
@@ -254,6 +255,52 @@ impl CMouseButton {
     }
 }
 
+#[repr(C)]
+pub enum CMediaSessionPlaybackState {
+    None = 1,
+    Playing,
+    Paused,
+}
+
+impl From<MediaSessionPlaybackState> for CMediaSessionPlaybackState {
+    fn from(state: MediaSessionPlaybackState) -> Self {
+        match state {
+            MediaSessionPlaybackState::None_ => CMediaSessionPlaybackState::None,
+            MediaSessionPlaybackState::Playing => CMediaSessionPlaybackState::Playing,
+            MediaSessionPlaybackState::Paused => CMediaSessionPlaybackState::Paused,
+        }
+    }
+}
+
+#[repr(C)]
+pub enum CMediaSessionActionType {
+    Play = 1,
+    Pause,
+    SeekBackward,
+    SeekForward,
+    PreviousTrack,
+    NextTrack,
+    SkipAd,
+    Stop,
+    SeekTo,
+}
+
+impl CMediaSessionActionType {
+    pub fn convert(&self) -> MediaSessionActionType {
+        match self {
+            CMediaSessionActionType::Play => MediaSessionActionType::Play,
+            CMediaSessionActionType::Pause => MediaSessionActionType::Pause,
+            CMediaSessionActionType::SeekBackward => MediaSessionActionType::SeekBackward,
+            CMediaSessionActionType::SeekForward => MediaSessionActionType::SeekForward,
+            CMediaSessionActionType::PreviousTrack => MediaSessionActionType::PreviousTrack,
+            CMediaSessionActionType::NextTrack => MediaSessionActionType::NextTrack,
+            CMediaSessionActionType::SkipAd => MediaSessionActionType::SkipAd,
+            CMediaSessionActionType::Stop => MediaSessionActionType::Stop,
+            CMediaSessionActionType::SeekTo => MediaSessionActionType::SeekTo,
+        }
+    }
+}
+
 /// The returned string is not freed. This will leak.
 #[no_mangle]
 pub extern "C" fn servo_version() -> *const c_char {
@@ -607,6 +654,14 @@ pub extern "C" fn click(x: f32, y: f32) {
     });
 }
 
+#[no_mangle]
+pub extern "C" fn media_session_action(action: CMediaSessionActionType) {
+    catch_any_panic(|| {
+        debug!("media_session_action");
+        call(|s| s.media_session_action(action.convert()));
+    });
+}
+
 pub struct WakeupCallback(extern "C" fn());
 
 impl WakeupCallback {
@@ -725,9 +780,9 @@ impl HostTrait for HostCallbacks {
         (self.0.on_media_session_metadata)(title.as_ptr(), artist.as_ptr(), album.as_ptr());
     }
 
-    fn on_media_session_playback_state_change(&self, state: i32) {
+    fn on_media_session_playback_state_change(&self, state: MediaSessionPlaybackState) {
         debug!("on_media_session_playback_state_change {:?}", state);
-        (self.0.on_media_session_playback_state_change)(state);
+        (self.0.on_media_session_playback_state_change)(state.into());
     }
 
     fn on_media_session_set_position_state(

ports/libsimpleservo/jniapi/src/lib.rs

@@ -15,7 +15,9 @@ use jni::{errors, JNIEnv, JavaVM};
 use libc::{dup2, pipe, read};
 use log::Level;
 use simpleservo::{self, gl_glue, ServoGlue, SERVO};
-use simpleservo::{Coordinates, EventLoopWaker, HostTrait, InitOptions, VRInitOptions};
+use simpleservo::{
+    Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionPlaybackState, VRInitOptions,
+};
 use std::os::raw::{c_char, c_int, c_void};
 use std::ptr::{null, null_mut};
 use std::sync::Arc;
@@ -340,7 +342,7 @@ pub fn Java_org_mozilla_servoview_JNIServo_mediaSessionAction(
     action: jint,
 ) {
     debug!("mediaSessionAction");
-    call(&env, |s| s.media_session_action(action as i32));
+    call(&env, |s| s.media_session_action((action as i32).into()));
 }
 
 pub struct WakeupCallback {
@@ -548,9 +550,10 @@ impl HostTrait for HostCallbacks {
         .unwrap();
     }
 
-    fn on_media_session_playback_state_change(&self, state: i32) {
+    fn on_media_session_playback_state_change(&self, state: MediaSessionPlaybackState) {
         info!("on_media_session_playback_state_change {:?}", state);
         let env = self.jvm.get_env().unwrap();
+        let state = state as i32;
         let state = JValue::Int(state as jint);
         env.call_method(
             self.callbacks.as_obj(),

support/hololens/ServoApp/BrowserPage.cpp

@@ -70,6 +70,24 @@ void BrowserPage::BindServoEvents() {
   servoControl().OnCaptureGesturesEnded(
       [=] { navigationBar().IsHitTestVisible(true); });
   urlTextbox().GotFocus(std::bind(&BrowserPage::OnURLFocused, this, _1));
+  servoControl().OnMediaSessionMetadata(
+      [=](hstring title, hstring artist, hstring album) {});
+  servoControl().OnMediaSessionPlaybackStateChange(
+      [=](const auto &, int state) {
+        if (state == servo::Servo::MediaSessionPlaybackState::None) {
+          mediaControls().Visibility(Visibility::Collapsed);
+          return;
+        }
+        mediaControls().Visibility(Visibility::Visible);
+        playButton().Visibility(
+            state == servo::Servo::MediaSessionPlaybackState::Paused
+                ? Visibility::Visible
+                : Visibility::Collapsed);
+        pauseButton().Visibility(
+            state == servo::Servo::MediaSessionPlaybackState::Paused
+                ? Visibility::Collapsed
+                : Visibility::Visible);
+      });
 }
 
 void BrowserPage::OnURLFocused(Windows::Foundation::IInspectable const &) {
@@ -143,4 +161,17 @@ void BrowserPage::OnURLEdited(IInspectable const &,
   }
 }
 
+void BrowserPage::OnMediaControlsPlayClicked(
+    Windows::Foundation::IInspectable const &,
+    Windows::UI::Xaml::RoutedEventArgs const &) {
+  servoControl().SendMediaSessionAction(
+      static_cast<int32_t>(servo::Servo::MediaSessionActionType::Play));
+}
+void BrowserPage::OnMediaControlsPauseClicked(
+    Windows::Foundation::IInspectable const &,
+    Windows::UI::Xaml::RoutedEventArgs const &) {
+  servoControl().SendMediaSessionAction(
+      static_cast<int32_t>(servo::Servo::MediaSessionActionType::Pause));
+}
+
 } // namespace winrt::ServoApp::implementation

support/hololens/ServoApp/BrowserPage.h

@@ -41,6 +41,10 @@ public:
                                   Windows::UI::Xaml::RoutedEventArgs const &);
   void OnXRPkgWarningDismissClick(Windows::Foundation::IInspectable const &,
                                   Windows::UI::Xaml::RoutedEventArgs const &);
+  void OnMediaControlsPlayClicked(Windows::Foundation::IInspectable const &,
+                                  Windows::UI::Xaml::RoutedEventArgs const &);
+  void OnMediaControlsPauseClicked(Windows::Foundation::IInspectable const &,
+                                   Windows::UI::Xaml::RoutedEventArgs const &);
 
 private:
   void BindServoEvents();

support/hololens/ServoApp/BrowserPage.xaml

@@ -137,5 +137,10 @@
                 <Button Margin="0,10,10,0" Click="OnXRPkgWarningDismissClick">Dismiss</Button>
             </StackPanel>
         </StackPanel>
+        <CommandBar Grid.Row="3" x:Name="mediaControls" Visibility="Collapsed">
+            <AppBarButton Icon="Play" Label="Play" x:Name="playButton" Visibility="Collapsed" Click="OnMediaControlsPlayClicked"/>
+            <AppBarButton Icon="Pause" Label="Pause" x:Name="pauseButton" Click="OnMediaControlsPauseClicked"/>
+        </CommandBar>
     </Grid>
+
 </Page>

support/hololens/ServoApp/ServoControl/Servo.cpp

@@ -56,6 +56,17 @@ const char *get_clipboard_contents() {
   return nullptr;
 }
 
+void on_media_session_metadata(const char *title, const char *album,
+                               const char *artist) {
+  return sServo->Delegate().OnServoMediaSessionMetadata(
+      char2hstring(title), char2hstring(album), char2hstring(artist));
+}
+
+void on_media_session_playback_state_change(
+    const capi::CMediaSessionPlaybackState state) {
+  return sServo->Delegate().OnServoMediaSessionPlaybackStateChange(state);
+}
+
 Servo::Servo(hstring url, hstring args, GLsizei width, GLsizei height,
              float dpi, ServoDelegate &aDelegate)
     : mWindowHeight(height), mWindowWidth(width), mDelegate(aDelegate) {
@@ -110,6 +121,9 @@ Servo::Servo(hstring url, hstring args, GLsizei width, GLsizei height,
   c.on_ime_state_changed = &on_ime_state_changed;
   c.get_clipboard_contents = &get_clipboard_contents;
   c.set_clipboard_contents = &set_clipboard_contents;
+  c.on_media_session_metadata = &on_media_session_metadata;
+  c.on_media_session_playback_state_change =
+      &on_media_session_playback_state_change;
 
   capi::register_panic_handler(&on_panic);
 

support/hololens/ServoApp/ServoControl/Servo.h

@@ -36,6 +36,8 @@ public:
   virtual void OnServoIMEStateChanged(bool) = 0;
   virtual void Flush() = 0;
   virtual void MakeCurrent() = 0;
+  virtual void OnServoMediaSessionMetadata(hstring, hstring, hstring) = 0;
+  virtual void OnServoMediaSessionPlaybackStateChange(int) = 0;
 
 protected:
   virtual ~ServoDelegate(){};
@@ -48,6 +50,8 @@ public:
   ServoDelegate &Delegate() { return mDelegate; }
 
   typedef capi::CMouseButton MouseButton;
+  typedef capi::CMediaSessionActionType MediaSessionActionType;
+  typedef capi::CMediaSessionPlaybackState MediaSessionPlaybackState;
 
   void PerformUpdates() { capi::perform_updates(); }
   void DeInit() { capi::deinit(); }
@@ -86,6 +90,9 @@ public:
       capi::resize(mWindowWidth, mWindowHeight);
     }
   }
+  void SendMediaSessionAction(capi::CMediaSessionActionType action) {
+    capi::media_session_action(action);
+  }
 
 private:
   ServoDelegate &mDelegate;

support/hololens/ServoApp/ServoControl/ServoControl.cpp

@@ -271,6 +271,13 @@ hstring ServoControl::LoadURIOrSearch(hstring input) {
   return searchUri;
 }
 
+void ServoControl::SendMediaSessionAction(int32_t action) {
+  RunOnGLThread([=] {
+    mServo->SendMediaSessionAction(
+        static_cast<Servo::MediaSessionActionType>(action));
+  });
+}
+
 void ServoControl::TryLoadUri(hstring input) {
   if (!mLooping) {
     mInitialURL = input;
@@ -432,6 +439,15 @@ void ServoControl::OnServoIMEStateChanged(bool aShow) {
   // https://docs.microsoft.com/en-us/windows/win32/winauto/uiauto-implementingtextandtextrange
 }
 
+void ServoControl::OnServoMediaSessionMetadata(hstring title, hstring artist,
+                                               hstring album) {
+  RunOnUIThread([=] { mOnMediaSessionMetadataEvent(title, artist, album); });
+}
+
+void ServoControl::OnServoMediaSessionPlaybackStateChange(int state) {
+  RunOnUIThread([=] { mOnMediaSessionPlaybackStateChangeEvent(*this, state); });
+}
+
 template <typename Callable> void ServoControl::RunOnUIThread(Callable cb) {
   Dispatcher().RunAsync(CoreDispatcherPriority::High, cb);
 }

support/hololens/ServoApp/ServoControl/ServoControl.h

@@ -15,6 +15,7 @@ struct ServoControl : ServoControlT<ServoControl>, public servo::ServoDelegate {
   void Stop();
   void Shutdown();
   hstring LoadURIOrSearch(hstring);
+  void SendMediaSessionAction(int32_t);
 
   void OnLoaded(IInspectable const &,
                 Windows::UI::Xaml::RoutedEventArgs const &);
@@ -70,6 +71,23 @@ struct ServoControl : ServoControlT<ServoControl>, public servo::ServoDelegate {
     mOnCaptureGesturesEndedEvent.remove(token);
   }
 
+  winrt::event_token
+  OnMediaSessionMetadata(MediaSessionMetadataDelegate const &handler) {
+    return mOnMediaSessionMetadataEvent.add(handler);
+  };
+  void OnMediaSessionMetadata(winrt::event_token const &token) noexcept {
+    mOnMediaSessionMetadataEvent.remove(token);
+  }
+
+  winrt::event_token OnMediaSessionPlaybackStateChange(
+      Windows::Foundation::EventHandler<int> const &handler) {
+    return mOnMediaSessionPlaybackStateChangeEvent.add(handler);
+  };
+  void
+  OnMediaSessionPlaybackStateChange(winrt::event_token const &token) noexcept {
+    mOnMediaSessionPlaybackStateChangeEvent.remove(token);
+  }
+
   void SetTransientMode(bool transient) { mTransient = transient; }
 
   void SetArgs(hstring args) { mArgs = args; }
@@ -87,6 +105,9 @@ struct ServoControl : ServoControlT<ServoControl>, public servo::ServoDelegate {
   virtual bool OnServoAllowNavigation(winrt::hstring);
   virtual void OnServoAnimatingChanged(bool);
   virtual void OnServoIMEStateChanged(bool);
+  virtual void OnServoMediaSessionMetadata(winrt::hstring, winrt::hstring,
+                                           winrt::hstring);
+  virtual void OnServoMediaSessionPlaybackStateChange(int);
 
 private:
   winrt::event<Windows::Foundation::EventHandler<hstring>> mOnURLChangedEvent;
@@ -96,6 +117,9 @@ private:
   winrt::event<EventDelegate> mOnLoadEndedEvent;
   winrt::event<EventDelegate> mOnCaptureGesturesStartedEvent;
   winrt::event<EventDelegate> mOnCaptureGesturesEndedEvent;
+  winrt::event<MediaSessionMetadataDelegate> mOnMediaSessionMetadataEvent;
+  winrt::event<Windows::Foundation::EventHandler<int>>
+      mOnMediaSessionPlaybackStateChangeEvent;
 
   float mDPI = 1;
   hstring mInitialURL = DEFAULT_URL;

support/hololens/ServoApp/ServoControl/ServoControl.idl

@@ -2,6 +2,7 @@ namespace ServoApp {
 
   delegate void EventDelegate();
   delegate void HistoryChangedDelegate(Boolean back, Boolean forward);
+  delegate void MediaSessionMetadataDelegate(String title, String artist, String album);
 
   runtimeclass ServoControl : Windows.UI.Xaml.Controls.Control {
       ServoControl();
@@ -13,6 +14,7 @@ namespace ServoApp {
       void SetTransientMode(Boolean transient);
       void SetArgs(String args);
       void Shutdown();
+      void SendMediaSessionAction(UInt32 action);
       event EventDelegate OnLoadStarted;
       event EventDelegate OnLoadEnded;
       event EventDelegate OnCaptureGesturesStarted;
@@ -20,5 +22,7 @@ namespace ServoApp {
       event HistoryChangedDelegate OnHistoryChanged;
       event Windows.Foundation.EventHandler<String> OnTitleChanged;
       event Windows.Foundation.EventHandler<String> OnURLChanged;
+      event MediaSessionMetadataDelegate OnMediaSessionMetadata;
+      event Windows.Foundation.EventHandler<int> OnMediaSessionPlaybackStateChange;
 }
 } // namespace ServoApp