From 4b5b4d19bf897dc45c855fcdfc2294b12241d12f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 1 Oct 2019 13:25:16 +0200 Subject: [PATCH 01/37] MediaSession API DOM bindings --- components/script/dom/mediametadata.rs | 94 +++++++++++++++++++ components/script/dom/mediasession.rs | 70 ++++++++++++++ components/script/dom/mod.rs | 2 + components/script/dom/navigator.rs | 9 ++ .../script/dom/webidls/MediaMetadata.webidl | 30 ++++++ .../script/dom/webidls/MediaSession.webidl | 57 +++++++++++ 6 files changed, 262 insertions(+) create mode 100644 components/script/dom/mediametadata.rs create mode 100644 components/script/dom/mediasession.rs create mode 100644 components/script/dom/webidls/MediaMetadata.webidl create mode 100644 components/script/dom/webidls/MediaSession.webidl diff --git a/components/script/dom/mediametadata.rs b/components/script/dom/mediametadata.rs new file mode 100644 index 00000000000..f4d801506bc --- /dev/null +++ b/components/script/dom/mediametadata.rs @@ -0,0 +1,94 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::dom::bindings::cell::DomRefCell; +use crate::dom::bindings::codegen::Bindings::MediaMetadataBinding; +use crate::dom::bindings::codegen::Bindings::MediaMetadataBinding::MediaMetadataInit; +use crate::dom::bindings::codegen::Bindings::MediaMetadataBinding::MediaMetadataMethods; +use crate::dom::bindings::error::Fallible; +use crate::dom::bindings::reflector::{reflect_dom_object, Reflector}; +use crate::dom::bindings::root::{DomRoot, MutNullableDom}; +use crate::dom::bindings::str::DOMString; +use crate::dom::mediasession::MediaSession; +use crate::dom::window::Window; +use dom_struct::dom_struct; + +#[dom_struct] +pub struct MediaMetadata { + reflector_: Reflector, + session: MutNullableDom, + title: DomRefCell, + artist: DomRefCell, + album: DomRefCell, +} + +impl MediaMetadata { + fn new_inherited(init: &MediaMetadataInit) -> MediaMetadata { + MediaMetadata { + reflector_: Reflector::new(), + // TODO(ferjm): Set active media session? + session: Default::default(), + title: DomRefCell::new(init.title.clone()), + artist: DomRefCell::new(init.artist.clone()), + album: DomRefCell::new(init.album.clone()), + } + } + + pub fn new(global: &Window, init: &MediaMetadataInit) -> DomRoot { + reflect_dom_object( + Box::new(MediaMetadata::new_inherited(init)), + global, + MediaMetadataBinding::Wrap, + ) + } + + /// https://w3c.github.io/mediasession/#dom-mediametadata-mediametadata + pub fn Constructor( + window: &Window, + init: &MediaMetadataInit, + ) -> Fallible> { + Ok(MediaMetadata::new(window, init)) + } + + fn queue_update_metadata_algorithm(&self) { + if self.session.get().is_none() { + return; + } + } +} + +impl MediaMetadataMethods for MediaMetadata { + /// https://w3c.github.io/mediasession/#dom-mediametadata-title + fn Title(&self) -> DOMString { + self.title.borrow().clone() + } + + /// https://w3c.github.io/mediasession/#dom-mediametadata-title + fn SetTitle(&self, value: DOMString) -> () { + *self.title.borrow_mut() = value; + self.queue_update_metadata_algorithm(); + } + + /// https://w3c.github.io/mediasession/#dom-mediametadata-artist + fn Artist(&self) -> DOMString { + self.artist.borrow().clone() + } + + /// https://w3c.github.io/mediasession/#dom-mediametadata-artist + fn SetArtist(&self, value: DOMString) -> () { + *self.artist.borrow_mut() = value; + self.queue_update_metadata_algorithm(); + } + + /// https://w3c.github.io/mediasession/#dom-mediametadata-album + fn Album(&self) -> DOMString { + self.album.borrow().clone() + } + + /// https://w3c.github.io/mediasession/#dom-mediametadata-album + fn SetAlbum(&self, value: DOMString) -> () { + *self.album.borrow_mut() = value; + self.queue_update_metadata_algorithm(); + } +} diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs new file mode 100644 index 00000000000..010d51afa0c --- /dev/null +++ b/components/script/dom/mediasession.rs @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::dom::bindings::cell::DomRefCell; +use crate::dom::bindings::codegen::Bindings::MediaSessionBinding; +use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionAction; +use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionActionHandler; +use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionMethods; +use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionPlaybackState; +use crate::dom::bindings::reflector::{reflect_dom_object, Reflector}; +use crate::dom::bindings::root::{DomRoot, MutNullableDom}; +use crate::dom::mediametadata::MediaMetadata; +use crate::dom::window::Window; +use dom_struct::dom_struct; +use std::rc::Rc; + +#[dom_struct] +pub struct MediaSession { + reflector_: Reflector, + /// https://w3c.github.io/mediasession/#dom-mediasession-metadata + metadata: MutNullableDom, + /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate + playback_state: DomRefCell, +} + +impl MediaSession { + fn new_inherited() -> MediaSession { + MediaSession { + reflector_: Reflector::new(), + metadata: Default::default(), + playback_state: DomRefCell::new(MediaSessionPlaybackState::None), + } + } + + pub fn new(global: &Window) -> DomRoot { + reflect_dom_object( + Box::new(MediaSession::new_inherited()), + global, + MediaSessionBinding::Wrap, + ) + } +} + +impl MediaSessionMethods for MediaSession { + fn GetMetadata(&self) -> Option> { + self.metadata.get() + } + + fn SetMetadata(&self, value: Option<&MediaMetadata>) { + self.metadata.set(value); + } + + /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate + fn PlaybackState(&self) -> MediaSessionPlaybackState { + *self.playback_state.borrow() + } + + /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate + fn SetPlaybackState(&self, value: MediaSessionPlaybackState) { + *self.playback_state.borrow_mut() = value; + } + + fn SetActionHandler( + &self, + _action: MediaSessionAction, + _handler: Option>, + ) { + } +} diff --git a/components/script/dom/mod.rs b/components/script/dom/mod.rs index fbc4d457482..ef562931bee 100644 --- a/components/script/dom/mod.rs +++ b/components/script/dom/mod.rs @@ -398,8 +398,10 @@ pub mod mediaelementaudiosourcenode; pub mod mediaerror; pub mod mediafragmentparser; pub mod medialist; +pub mod mediametadata; pub mod mediaquerylist; pub mod mediaquerylistevent; +pub mod mediasession; pub mod mediastream; pub mod mediastreamtrack; pub mod messagechannel; diff --git a/components/script/dom/navigator.rs b/components/script/dom/navigator.rs index 8a0f6a21d98..2b0d0ab128b 100644 --- a/components/script/dom/navigator.rs +++ b/components/script/dom/navigator.rs @@ -12,6 +12,7 @@ use crate::dom::bindings::str::DOMString; use crate::dom::bluetooth::Bluetooth; use crate::dom::gamepadlist::GamepadList; use crate::dom::mediadevices::MediaDevices; +use crate::dom::mediasession::MediaSession; use crate::dom::mimetypearray::MimeTypeArray; use crate::dom::navigatorinfo; use crate::dom::permissions::Permissions; @@ -34,6 +35,7 @@ pub struct Navigator { mediadevices: MutNullableDom, gamepads: MutNullableDom, permissions: MutNullableDom, + mediasession: MutNullableDom, } impl Navigator { @@ -48,6 +50,7 @@ impl Navigator { mediadevices: Default::default(), gamepads: Default::default(), permissions: Default::default(), + mediasession: Default::default(), } } @@ -186,4 +189,10 @@ impl NavigatorMethods for Navigator { self.mediadevices .or_init(|| MediaDevices::new(&self.global())) } + + /// https://w3c.github.io/mediasession/#dom-navigator-mediasession + fn MediaSession(&self) -> DomRoot { + self.mediasession + .or_init(|| MediaSession::new(self.global().as_window())) + } } diff --git a/components/script/dom/webidls/MediaMetadata.webidl b/components/script/dom/webidls/MediaMetadata.webidl new file mode 100644 index 00000000000..b4142e431ae --- /dev/null +++ b/components/script/dom/webidls/MediaMetadata.webidl @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +/* + * The origin of this IDL file is + * https://w3c.github.io/mediasession/#mediametadata + */ + +dictionary MediaImage { + required USVString src; + DOMString sizes = ""; + DOMString type = ""; +}; + +[Exposed=Window, + Constructor(optional MediaMetadataInit init = {})] +interface MediaMetadata { + attribute DOMString title; + attribute DOMString artist; + attribute DOMString album; + // TODO: https://github.com/servo/servo/issues/10072 + // attribute FrozenArray artwork; +}; + +dictionary MediaMetadataInit { + DOMString title = ""; + DOMString artist = ""; + DOMString album = ""; + sequence artwork = []; +}; diff --git a/components/script/dom/webidls/MediaSession.webidl b/components/script/dom/webidls/MediaSession.webidl new file mode 100644 index 00000000000..0096a0ffa50 --- /dev/null +++ b/components/script/dom/webidls/MediaSession.webidl @@ -0,0 +1,57 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +/* + * The origin of this IDL file is + * https://w3c.github.io/mediasession/#mediasession + */ + +[Exposed=Window] +partial interface Navigator { + [SameObject] readonly attribute MediaSession mediaSession; +}; + +enum MediaSessionPlaybackState { + "none", + "paused", + "playing" +}; + +enum MediaSessionAction { + "play", + "pause", + "seekbackward", + "seekforward", + "previoustrack", + "nexttrack", + "skipad", + "stop", + "seekto" +}; + +dictionary MediaSessionActionDetails { + required MediaSessionAction action; +}; + +dictionary MediaSessionSeekActionDetails : MediaSessionActionDetails { + double? seekOffset; +}; + +dictionary MediaSessionSeekToActionDetails : MediaSessionActionDetails { + required double seekTime; + boolean? fastSeek; +}; + +callback MediaSessionActionHandler = void(MediaSessionActionDetails details); + +[Exposed=Window] +interface MediaSession { + attribute MediaMetadata? metadata; + + attribute MediaSessionPlaybackState playbackState; + + void setActionHandler(MediaSessionAction action, MediaSessionActionHandler? handler); + + //void setPositionState(optional MediaPositionState? state); +}; + From 1ab65005ae4d1925de357f42212bc2aaa972e77e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 2 Oct 2019 17:27:26 +0200 Subject: [PATCH 02/37] Set MediaMetadata MediaSession owner --- components/script/dom/mediametadata.rs | 5 ++++- components/script/dom/mediasession.rs | 11 +++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/components/script/dom/mediametadata.rs b/components/script/dom/mediametadata.rs index f4d801506bc..db84df878c5 100644 --- a/components/script/dom/mediametadata.rs +++ b/components/script/dom/mediametadata.rs @@ -27,7 +27,6 @@ impl MediaMetadata { fn new_inherited(init: &MediaMetadataInit) -> MediaMetadata { MediaMetadata { reflector_: Reflector::new(), - // TODO(ferjm): Set active media session? session: Default::default(), title: DomRefCell::new(init.title.clone()), artist: DomRefCell::new(init.artist.clone()), @@ -56,6 +55,10 @@ impl MediaMetadata { return; } } + + pub fn set_session(&self, session: &MediaSession) { + self.session.set(Some(&session)); + } } impl MediaMetadataMethods for MediaMetadata { diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 010d51afa0c..bb33c523374 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -47,8 +47,11 @@ impl MediaSessionMethods for MediaSession { self.metadata.get() } - fn SetMetadata(&self, value: Option<&MediaMetadata>) { - self.metadata.set(value); + fn SetMetadata(&self, metadata: Option<&MediaMetadata>) { + if let Some(ref metadata) = metadata { + metadata.set_session(self); + } + self.metadata.set(metadata); } /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate @@ -57,8 +60,8 @@ impl MediaSessionMethods for MediaSession { } /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate - fn SetPlaybackState(&self, value: MediaSessionPlaybackState) { - *self.playback_state.borrow_mut() = value; + fn SetPlaybackState(&self, state: MediaSessionPlaybackState) { + *self.playback_state.borrow_mut() = state; } fn SetActionHandler( From f8246801ba47f66d205dae142d4d3498a304220e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 4 Oct 2019 12:27:16 +0200 Subject: [PATCH 03/37] MediaSession registration --- components/script/dom/mediasession.rs | 33 +++++++++++++++++++++------ components/script/script_thread.rs | 27 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index bb33c523374..6d9afa592e0 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -8,11 +8,13 @@ use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionAc use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionActionHandler; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionMethods; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionPlaybackState; -use crate::dom::bindings::reflector::{reflect_dom_object, Reflector}; +use crate::dom::bindings::reflector::{reflect_dom_object, DomObject, Reflector}; use crate::dom::bindings::root::{DomRoot, MutNullableDom}; use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; +use crate::script_thread::ScriptThread; use dom_struct::dom_struct; +use msg::constellation_msg::PipelineId; use std::rc::Rc; #[dom_struct] @@ -25,18 +27,24 @@ pub struct MediaSession { } impl MediaSession { - fn new_inherited() -> MediaSession { - MediaSession { + #[allow(unrooted_must_root)] + fn new_inherited(pipeline_id: PipelineId) -> MediaSession { + let media_session = MediaSession { reflector_: Reflector::new(), metadata: Default::default(), playback_state: DomRefCell::new(MediaSessionPlaybackState::None), - } + }; + ScriptThread::register_media_session(&media_session, pipeline_id); + media_session } - pub fn new(global: &Window) -> DomRoot { + pub fn new(window: &Window) -> DomRoot { + let pipeline_id = window + .pipeline_id() + .expect("Cannot create MediaSession without a PipelineId"); reflect_dom_object( - Box::new(MediaSession::new_inherited()), - global, + Box::new(MediaSession::new_inherited(pipeline_id)), + window, MediaSessionBinding::Wrap, ) } @@ -71,3 +79,14 @@ impl MediaSessionMethods for MediaSession { ) { } } + +impl Drop for MediaSession { + fn drop(&mut self) { + let global = self.global(); + let pipeline_id = global + .as_window() + .pipeline_id() + .expect("No PipelineId while dropping MediaSession"); + ScriptThread::remove_media_session(pipeline_id); + } +} diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index dbc4f0fb444..5aec6a8a57d 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -51,6 +51,7 @@ use crate::dom::event::{Event, EventBubbles, EventCancelable}; use crate::dom::globalscope::GlobalScope; use crate::dom::htmlanchorelement::HTMLAnchorElement; use crate::dom::htmliframeelement::{HTMLIFrameElement, NavigationType}; +use crate::dom::mediasession::MediaSession; use crate::dom::mutationobserver::MutationObserver; use crate::dom::node::{ from_untrusted_node_address, window_from_node, Node, NodeDamage, ShadowIncluding, @@ -696,6 +697,11 @@ pub struct ScriptThread { /// Code is running as a consequence of a user interaction is_user_interacting: Cell, + + /// The MediaSessions registered for this Pipeline, if any. + /// There can only be one active MediaSession. The constellation + /// has the PipelineId of the active MediaSession, if any. + media_sessions: DomRefCell>>, } /// In the event of thread panic, all data on the stack runs its destructor. However, there @@ -1365,6 +1371,7 @@ impl ScriptThread { node_ids: Default::default(), is_user_interacting: Cell::new(false), + media_sessions: DomRefCell::new(HashMap::new()), } } @@ -3948,6 +3955,26 @@ impl ScriptThread { globals, ) } + + pub fn register_media_session(media_session: &MediaSession, pipeline_id: PipelineId) { + SCRIPT_THREAD_ROOT.with(|root| { + let script_thread = unsafe { &*root.get().unwrap() }; + script_thread + .media_sessions + .borrow_mut() + .insert(pipeline_id, Dom::from_ref(media_session)); + }) + } + + pub fn remove_media_session(pipeline_id: PipelineId) { + SCRIPT_THREAD_ROOT.with(|root| { + let script_thread = unsafe { &*root.get().unwrap() }; + script_thread + .media_sessions + .borrow_mut() + .remove(&pipeline_id); + }) + } } impl Drop for ScriptThread { From 7101a9d070dd3d7bb89475e125de65f85f93221e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 4 Oct 2019 12:54:16 +0200 Subject: [PATCH 04/37] Use BrowsingContextId for MediaSession registration --- components/script/dom/mediasession.rs | 20 +++++++++----------- components/script/script_thread.rs | 19 +++++++++++-------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 6d9afa592e0..70e63f3fd58 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -14,7 +14,7 @@ use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; use crate::script_thread::ScriptThread; use dom_struct::dom_struct; -use msg::constellation_msg::PipelineId; +use msg::constellation_msg::TopLevelBrowsingContextId; use std::rc::Rc; #[dom_struct] @@ -28,22 +28,20 @@ pub struct MediaSession { impl MediaSession { #[allow(unrooted_must_root)] - fn new_inherited(pipeline_id: PipelineId) -> MediaSession { + fn new_inherited(browsing_context_id: TopLevelBrowsingContextId) -> MediaSession { let media_session = MediaSession { reflector_: Reflector::new(), metadata: Default::default(), playback_state: DomRefCell::new(MediaSessionPlaybackState::None), }; - ScriptThread::register_media_session(&media_session, pipeline_id); + ScriptThread::register_media_session(&media_session, browsing_context_id); media_session } pub fn new(window: &Window) -> DomRoot { - let pipeline_id = window - .pipeline_id() - .expect("Cannot create MediaSession without a PipelineId"); + let browsing_context_id = window.window_proxy().top_level_browsing_context_id(); reflect_dom_object( - Box::new(MediaSession::new_inherited(pipeline_id)), + Box::new(MediaSession::new_inherited(browsing_context_id)), window, MediaSessionBinding::Wrap, ) @@ -83,10 +81,10 @@ impl MediaSessionMethods for MediaSession { impl Drop for MediaSession { fn drop(&mut self) { let global = self.global(); - let pipeline_id = global + let browsing_context_id = global .as_window() - .pipeline_id() - .expect("No PipelineId while dropping MediaSession"); - ScriptThread::remove_media_session(pipeline_id); + .window_proxy() + .top_level_browsing_context_id(); + ScriptThread::remove_media_session(browsing_context_id); } } diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 5aec6a8a57d..2bd7cc4907b 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -698,10 +698,10 @@ pub struct ScriptThread { /// Code is running as a consequence of a user interaction is_user_interacting: Cell, - /// The MediaSessions registered for this Pipeline, if any. - /// There can only be one active MediaSession. The constellation - /// has the PipelineId of the active MediaSession, if any. - media_sessions: DomRefCell>>, + /// The MediaSessions known by this thread, if any. + /// There can only be one active MediaSession. + /// The constellation has the BrowsingContextId of the active MediaSession, if any. + media_sessions: DomRefCell>>, } /// In the event of thread panic, all data on the stack runs its destructor. However, there @@ -3956,23 +3956,26 @@ impl ScriptThread { ) } - pub fn register_media_session(media_session: &MediaSession, pipeline_id: PipelineId) { + pub fn register_media_session( + media_session: &MediaSession, + browsing_context_id: TopLevelBrowsingContextId, + ) { SCRIPT_THREAD_ROOT.with(|root| { let script_thread = unsafe { &*root.get().unwrap() }; script_thread .media_sessions .borrow_mut() - .insert(pipeline_id, Dom::from_ref(media_session)); + .insert(browsing_context_id, Dom::from_ref(media_session)); }) } - pub fn remove_media_session(pipeline_id: PipelineId) { + pub fn remove_media_session(browsing_context_id: TopLevelBrowsingContextId) { SCRIPT_THREAD_ROOT.with(|root| { let script_thread = unsafe { &*root.get().unwrap() }; script_thread .media_sessions .borrow_mut() - .remove(&pipeline_id); + .remove(&browsing_context_id); }) } } From 6233f78de48a92786b34426e96ffcfe05c6bc370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 4 Oct 2019 17:05:56 +0200 Subject: [PATCH 05/37] MediaSessionAction message from embedder to script thread --- components/compositing/windowing.rs | 6 +++- components/constellation/constellation.rs | 36 ++++++++++++++++++++++- components/script/script_thread.rs | 14 ++++++++- components/script_traits/lib.rs | 34 +++++++++++++++++++++ components/servo/lib.rs | 10 +++++++ 5 files changed, 97 insertions(+), 3 deletions(-) diff --git a/components/compositing/windowing.rs b/components/compositing/windowing.rs index decece701f4..d2fab70c234 100644 --- a/components/compositing/windowing.rs +++ b/components/compositing/windowing.rs @@ -10,7 +10,7 @@ use euclid::Scale; use gleam::gl; use keyboard_types::KeyboardEvent; use msg::constellation_msg::{PipelineId, TopLevelBrowsingContextId, TraversalDirection}; -use script_traits::{MouseButton, TouchEventType, TouchId, WheelDelta}; +use script_traits::{MediaSessionActionType, MouseButton, TouchEventType, TouchId, WheelDelta}; use servo_geometry::DeviceIndependentPixel; use servo_media::player::context::{GlApi, GlContext, NativeDisplay}; use servo_url::ServoUrl; @@ -102,6 +102,9 @@ pub enum WindowEvent { CaptureWebRender, /// Toggle sampling profiler with the given sampling rate and max duration. ToggleSamplingProfiler(Duration, Duration), + /// Sent when the user triggers a media action through the UA exposed media UI + /// (play, pause, seek, etc.). + MediaSessionAction(TopLevelBrowsingContextId, MediaSessionActionType), } impl Debug for WindowEvent { @@ -132,6 +135,7 @@ impl Debug for WindowEvent { WindowEvent::CaptureWebRender => write!(f, "CaptureWebRender"), WindowEvent::ToggleSamplingProfiler(..) => write!(f, "ToggleSamplingProfiler"), WindowEvent::ExitFullScreen(..) => write!(f, "ExitFullScreen"), + WindowEvent::MediaSessionAction(..) => write!(f, "MediaSessionAction"), } } } diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index c24883f99e5..8f549ea685e 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -139,7 +139,6 @@ use net_traits::{self, FetchResponseMsg, IpcSend, ResourceThreads}; use profile_traits::mem; use profile_traits::time; use script_traits::CompositorEvent::{MouseButtonEvent, MouseMoveEvent}; -use script_traits::MouseEventType; use script_traits::{webdriver_msg, LogEntry, ScriptToConstellationChan, ServiceWorkerMsg}; use script_traits::{ AnimationState, AnimationTickType, AuxiliaryBrowsingContextLoadInfo, CompositorEvent, @@ -154,6 +153,7 @@ use script_traits::{ }; use script_traits::{LayoutMsg as FromLayoutMsg, ScriptMsg as FromScriptMsg, ScriptThreadFactory}; use script_traits::{MessagePortMsg, PortMessageTask, StructuredSerializedData}; +use script_traits::{MediaSessionActionType, MouseEventType}; use script_traits::{SWManagerMsg, ScopeThings, UpdatePipelineIdReason, WebDriverCommandMsg}; use serde::{Deserialize, Serialize}; use servo_config::{opts, pref}; @@ -1541,6 +1541,9 @@ where FromCompositorMsg::ExitFullScreen(top_level_browsing_context_id) => { self.handle_exit_fullscreen_msg(top_level_browsing_context_id); }, + FromCompositorMsg::MediaSessionAction(top_level_browsing_context_id, action) => { + self.handle_media_session_action_msg(top_level_browsing_context_id, action); + }, } } @@ -5019,4 +5022,35 @@ where .send(ToCompositorMsg::SetFrameTree(frame_tree)); } } + + fn handle_media_session_action_msg( + &mut self, + top_level_browsing_context_id: TopLevelBrowsingContextId, + action: MediaSessionActionType, + ) { + let browsing_context_id = BrowsingContextId::from(top_level_browsing_context_id); + let pipeline_id = match self.browsing_contexts.get(&browsing_context_id) { + Some(browsing_context) => browsing_context.pipeline_id, + None => { + return warn!( + "Browsing context {} got media session action request after closure.", + browsing_context_id + ); + }, + }; + let msg = + ConstellationControlMsg::MediaSessionAction(top_level_browsing_context_id, action); + let result = match self.pipelines.get(&pipeline_id) { + None => { + return warn!( + "Pipeline {} got media session action request after closure.", + pipeline_id + ) + }, + Some(pipeline) => pipeline.event_loop.send(msg), + }; + if let Err(e) = result { + self.handle_send_error(pipeline_id, e); + } + } } diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 2bd7cc4907b..1c1a7bd9538 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -139,7 +139,7 @@ use script_traits::{ DiscardBrowsingContext, DocumentActivity, EventResult, HistoryEntryReplacement, }; use script_traits::{InitialScriptState, JsEvalResult, LayoutMsg, LoadData, LoadOrigin}; -use script_traits::{MouseButton, MouseEventType, NewLayoutInfo}; +use script_traits::{MediaSessionActionType, MouseButton, MouseEventType, NewLayoutInfo}; use script_traits::{Painter, ProgressiveWebMetricType, ScriptMsg, ScriptThreadFactory}; use script_traits::{ScriptToConstellationChan, TimerEvent, TimerSchedulerMsg}; use script_traits::{TimerSource, TouchEventType, TouchId, UntrustedNodeAddress, WheelDelta}; @@ -1720,6 +1720,7 @@ impl ScriptThread { WebVREvents(id, ..) => Some(id), PaintMetric(..) => None, ExitFullScreen(id, ..) => Some(id), + MediaSessionAction(..) => None, } }, MixedMessage::FromDevtools(_) => None, @@ -1949,6 +1950,9 @@ impl ScriptThread { ConstellationControlMsg::PaintMetric(pipeline_id, metric_type, metric_value) => { self.handle_paint_metric(pipeline_id, metric_type, metric_value) }, + ConstellationControlMsg::MediaSessionAction(browsing_context_id, action) => { + self.handle_media_session_action(browsing_context_id, action) + }, msg @ ConstellationControlMsg::AttachLayout(..) | msg @ ConstellationControlMsg::Viewport(..) | msg @ ConstellationControlMsg::SetScrollState(..) | @@ -3932,6 +3936,14 @@ impl ScriptThread { } } + fn handle_media_session_action( + &self, + browsing_context_id: TopLevelBrowsingContextId, + action: MediaSessionActionType, + ) { + // TODO + } + pub fn enqueue_microtask(job: Microtask) { SCRIPT_THREAD_ROOT.with(|root| { let script_thread = unsafe { &*root.get().unwrap() }; diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index 0212ae021b6..4bfa594688a 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -388,6 +388,8 @@ pub enum ConstellationControlMsg { WebVREvents(PipelineId, Vec), /// Notifies the script thread about a new recorded paint metric. PaintMetric(PipelineId, ProgressiveWebMetricType, u64), + /// Notifies the media session about a user requested media session action. + MediaSessionAction(TopLevelBrowsingContextId, MediaSessionActionType), } impl fmt::Debug for ConstellationControlMsg { @@ -426,6 +428,7 @@ impl fmt::Debug for ConstellationControlMsg { WebVREvents(..) => "WebVREvents", PaintMetric(..) => "PaintMetric", ExitFullScreen(..) => "ExitFullScreen", + MediaSessionAction(..) => "MediaSessionAction", }; write!(formatter, "ConstellationControlMsg::{}", variant) } @@ -877,6 +880,8 @@ pub enum ConstellationMsg { DisableProfiler, /// Request to exit from fullscreen mode ExitFullScreen(TopLevelBrowsingContextId), + /// Media session action. + MediaSessionAction(TopLevelBrowsingContextId, MediaSessionActionType), } impl fmt::Debug for ConstellationMsg { @@ -907,6 +912,7 @@ impl fmt::Debug for ConstellationMsg { EnableProfiler(..) => "EnableProfiler", DisableProfiler => "DisableProfiler", ExitFullScreen(..) => "ExitFullScreen", + MediaSessionAction(..) => "MediaSessionAction", }; write!(formatter, "ConstellationMsg::{}", variant) } @@ -1052,4 +1058,32 @@ pub enum MessagePortMsg { RemoveMessagePort(MessagePortId), /// Handle a new port-message-task. NewTask(MessagePortId, PortMessageTask), + +/// The type of MediaSession action. +/// https://w3c.github.io/mediasession/#enumdef-mediasessionaction +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum MediaSessionActionType { + /// The action intent is to resume playback. + Play, + /// The action intent is to pause the currently active playback. + Pause, + /// The action intent is to move the playback time backward by a short period (i.e. a few + /// seconds). + SeekBackward, + /// The action intent is to move the playback time forward by a short period (i.e. a few + /// seconds). + SeekForward, + /// The action intent is to either start the current playback from the beginning if the + /// playback has a notion, of beginning, or move to the previous item in the playlist if the + /// playback has a notion of playlist. + PreviousTrack, + /// The action is to move to the playback to the next item in the playlist if the playback has + /// a notion of playlist. + NextTrack, + /// The action intent is to skip the advertisement that is currently playing. + SkipAd, + /// The action intent is to stop the playback and clear the state if appropriate. + Stop, + /// The action intent is to move the playback time to a specific time. + SeekTo, } diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 3d4b7bf14d3..67751234111 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -713,6 +713,16 @@ where ); } }, + + WindowEvent::MediaSessionAction(ctx, a) => { + let msg = ConstellationMsg::MediaSessionAction(ctx, a); + if let Err(e) = self.constellation_chan.send(msg) { + warn!( + "Sending MediaSessionAction message to constellation failed ({:?}).", + e + ); + } + }, } } From fa61191405a6e165ede14910eeb98b1398794e22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 4 Oct 2019 18:05:36 +0200 Subject: [PATCH 06/37] Make MediaSession handle embedder requested action --- components/script/dom/mediasession.rs | 5 +++++ components/script/script_thread.rs | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 70e63f3fd58..6fdfd9b27c4 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -15,6 +15,7 @@ use crate::dom::window::Window; use crate::script_thread::ScriptThread; use dom_struct::dom_struct; use msg::constellation_msg::TopLevelBrowsingContextId; +use script_traits::MediaSessionActionType; use std::rc::Rc; #[dom_struct] @@ -46,6 +47,10 @@ impl MediaSession { MediaSessionBinding::Wrap, ) } + + pub fn handle_action(&self, action: MediaSessionActionType) { + // TODO + } } impl MediaSessionMethods for MediaSession { diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 1c1a7bd9538..ec1f5852060 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -3941,7 +3941,10 @@ impl ScriptThread { browsing_context_id: TopLevelBrowsingContextId, action: MediaSessionActionType, ) { - // TODO + match self.media_sessions.borrow().get(&browsing_context_id) { + Some(session) => session.handle_action(action), + None => warn!("No MediaSession for this browsing context"), + }; } pub fn enqueue_microtask(job: Microtask) { From ec7a4bf32d355f2f115da016ac6aebb034493f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 7 Oct 2019 10:31:52 +0200 Subject: [PATCH 07/37] MediaSession: update action handler algorithm --- components/script/dom/bindings/trace.rs | 5 ++-- components/script/dom/mediasession.rs | 35 +++++++++++++++++++++++-- components/script_traits/lib.rs | 2 +- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 74e6996b67a..1d426c2a0e6 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -94,8 +94,8 @@ use profile_traits::time::ProfilerChan as TimeProfilerChan; use script_layout_interface::rpc::LayoutRPC; use script_layout_interface::OpaqueStyleAndLayoutData; use script_traits::transferable::MessagePortImpl; -use script_traits::DrawAPaintImageResult; -use script_traits::{DocumentActivity, ScriptToConstellationChan, TimerEventId, TimerSource}; +use script_traits::{DocumentActivity, DrawAPaintImageResult; +use script_traits::{MediaSessionActionType, ScriptToConstellationChan, TimerEventId, TimerSource}; use script_traits::{UntrustedNodeAddress, WindowSizeData, WindowSizeType}; use selectors::matching::ElementSelectorFlags; use serde::{Deserialize, Serialize}; @@ -536,6 +536,7 @@ unsafe_no_jsmanaged_fields!(WindowGLContext); unsafe_no_jsmanaged_fields!(VideoFrame); unsafe_no_jsmanaged_fields!(WebGLContextId); unsafe_no_jsmanaged_fields!(Arc>); +unsafe_no_jsmanaged_fields!(MediaSessionActionType); unsafe impl<'a> JSTraceable for &'a str { #[inline] diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 6fdfd9b27c4..4105ddb3b5f 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -16,6 +16,7 @@ use crate::script_thread::ScriptThread; use dom_struct::dom_struct; use msg::constellation_msg::TopLevelBrowsingContextId; use script_traits::MediaSessionActionType; +use std::collections::HashMap; use std::rc::Rc; #[dom_struct] @@ -25,6 +26,9 @@ pub struct MediaSession { metadata: MutNullableDom, /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate playback_state: DomRefCell, + /// https://w3c.github.io/mediasession/#supported-media-session-actions + #[ignore_malloc_size_of = "Rc"] + action_handlers: DomRefCell>>, } impl MediaSession { @@ -34,6 +38,7 @@ impl MediaSession { reflector_: Reflector::new(), metadata: Default::default(), playback_state: DomRefCell::new(MediaSessionPlaybackState::None), + action_handlers: DomRefCell::new(HashMap::new()), }; ScriptThread::register_media_session(&media_session, browsing_context_id); media_session @@ -54,10 +59,12 @@ impl MediaSession { } impl MediaSessionMethods for MediaSession { + /// https://w3c.github.io/mediasession/#dom-mediasession-metadata fn GetMetadata(&self) -> Option> { self.metadata.get() } + /// https://w3c.github.io/mediasession/#dom-mediasession-metadata fn SetMetadata(&self, metadata: Option<&MediaMetadata>) { if let Some(ref metadata) = metadata { metadata.set_session(self); @@ -75,11 +82,19 @@ impl MediaSessionMethods for MediaSession { *self.playback_state.borrow_mut() = state; } + /// https://w3c.github.io/mediasession/#update-action-handler-algorithm fn SetActionHandler( &self, - _action: MediaSessionAction, - _handler: Option>, + action: MediaSessionAction, + handler: Option>, ) { + match handler { + Some(handler) => self + .action_handlers + .borrow_mut() + .insert(action.into(), handler.clone()), + None => self.action_handlers.borrow_mut().remove(&action.into()), + }; } } @@ -93,3 +108,19 @@ impl Drop for MediaSession { ScriptThread::remove_media_session(browsing_context_id); } } + +impl From for MediaSessionActionType { + fn from(action: MediaSessionAction) -> MediaSessionActionType { + match action { + MediaSessionAction::Play => MediaSessionActionType::Play, + MediaSessionAction::Pause => MediaSessionActionType::Pause, + MediaSessionAction::Seekbackward => MediaSessionActionType::SeekBackward, + MediaSessionAction::Seekforward => MediaSessionActionType::SeekForward, + MediaSessionAction::Previoustrack => MediaSessionActionType::PreviousTrack, + MediaSessionAction::Nexttrack => MediaSessionActionType::NextTrack, + MediaSessionAction::Skipad => MediaSessionActionType::SkipAd, + MediaSessionAction::Stop => MediaSessionActionType::Stop, + MediaSessionAction::Seekto => MediaSessionActionType::SeekTo, + } + } +} diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index 4bfa594688a..b4b85889ea0 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -1061,7 +1061,7 @@ pub enum MessagePortMsg { /// The type of MediaSession action. /// https://w3c.github.io/mediasession/#enumdef-mediasessionaction -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, Hash, MallocSizeOf, PartialEq, Serialize)] pub enum MediaSessionActionType { /// The action intent is to resume playback. Play, From 31ce7a2b5cd96eaf6eca78f811d49379e6082774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 7 Oct 2019 10:51:56 +0200 Subject: [PATCH 08/37] Handle media session action, no default handling yet --- components/script/dom/mediasession.rs | 9 ++++++++- components/script/dom/webidls/MediaSession.webidl | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 4105ddb3b5f..139264953ca 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +use crate::dom::bindings::callback::ExceptionHandling; use crate::dom::bindings::cell::DomRefCell; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionAction; @@ -54,7 +55,13 @@ impl MediaSession { } pub fn handle_action(&self, action: MediaSessionActionType) { - // TODO + if let Some(handler) = self.action_handlers.borrow().get(&action) { + if handler.Call__(ExceptionHandling::Report).is_err() { + warn!("Error calling MediaSessionActionHandler callback"); + } + return; + } + // TODO default action. } } diff --git a/components/script/dom/webidls/MediaSession.webidl b/components/script/dom/webidls/MediaSession.webidl index 0096a0ffa50..12b3fe062ba 100644 --- a/components/script/dom/webidls/MediaSession.webidl +++ b/components/script/dom/webidls/MediaSession.webidl @@ -42,7 +42,7 @@ dictionary MediaSessionSeekToActionDetails : MediaSessionActionDetails { boolean? fastSeek; }; -callback MediaSessionActionHandler = void(MediaSessionActionDetails details); +callback MediaSessionActionHandler = void(/*MediaSessionActionDetails details*/); [Exposed=Window] interface MediaSession { From d7d775ac606ee8f1d71c58fa321bc41b378e465d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 7 Oct 2019 11:12:25 +0200 Subject: [PATCH 09/37] Extend libsimpleservo API with a method to request a media session action --- ports/libsimpleservo/api/src/lib.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 0b415ac669c..952d85f9d31 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -19,7 +19,7 @@ use servo::embedder_traits::EmbedderMsg; use servo::euclid::{Point2D, Rect, Scale, Size2D, Vector2D}; use servo::keyboard_types::{Key, KeyState, KeyboardEvent}; use servo::msg::constellation_msg::TraversalDirection; -use servo::script_traits::{TouchEventType, TouchId}; +use servo::script_traits::{MediaSessionActionType, TouchEventType, TouchId}; use servo::servo_config::opts; use servo::servo_config::{pref, set_pref}; use servo::servo_url::ServoUrl; @@ -466,6 +466,15 @@ impl ServoGlue { self.process_event(WindowEvent::Keyboard(key_event)) } + pub fn media_session_action( + &mut self, + action: MediaSessionActionType, + ) -> Result<(), &'static str> { + info!("Media session action {:?}", action); + let browser_id = self.get_browser_id()?; + self.process_event(WindowEvent::MediaSessionAction(browser_id, action)) + } + fn process_event(&mut self, event: WindowEvent) -> Result<(), &'static str> { self.events.push(event); if !self.batch_mode { From fd040b0a5525e62c0483db0986aa63a4f611d6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 7 Oct 2019 11:21:55 +0200 Subject: [PATCH 10/37] Extend libsimpleservo API with a callback for media session activation --- ports/libsimpleservo/api/src/lib.rs | 2 ++ ports/libsimpleservo/capi/src/lib.rs | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 952d85f9d31..0984b7c49f3 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -130,6 +130,8 @@ pub trait HostTrait { fn get_clipboard_contents(&self) -> Option; /// Sets system clipboard contents fn set_clipboard_contents(&self, contents: String); + /// Called when a media session is activated or deactived. + fn on_media_session(&self, active: bool); } pub struct ServoGlue { diff --git a/ports/libsimpleservo/capi/src/lib.rs b/ports/libsimpleservo/capi/src/lib.rs index c6b27ae6d48..4fbd239000e 100644 --- a/ports/libsimpleservo/capi/src/lib.rs +++ b/ports/libsimpleservo/capi/src/lib.rs @@ -216,6 +216,7 @@ pub struct CHostCallbacks { pub on_ime_state_changed: extern "C" fn(show: bool), pub get_clipboard_contents: extern "C" fn() -> *const c_char, pub set_clipboard_contents: extern "C" fn(contents: *const c_char), + pub on_media_session: extern "C" fn(active: bool), } /// Servo options @@ -708,4 +709,9 @@ impl HostTrait for HostCallbacks { let contents = CString::new(contents).expect("Can't create string"); (self.0.set_clipboard_contents)(contents.as_ptr()); } + + fn on_media_session(&self, active: bool) { + debug!("on_media_session (active: {:?})", active); + (self.0.on_media_session)(active); + } } From 9c329a79354cb8b6c1000aa7c78364a5541de421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 7 Oct 2019 12:09:53 +0200 Subject: [PATCH 11/37] Add embedder message to (de)activate media session --- components/embedder_traits/lib.rs | 4 ++++ ports/libsimpleservo/api/src/lib.rs | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/components/embedder_traits/lib.rs b/components/embedder_traits/lib.rs index 949b3ba5acb..da526d5f478 100644 --- a/components/embedder_traits/lib.rs +++ b/components/embedder_traits/lib.rs @@ -162,6 +162,9 @@ pub enum EmbedderMsg { Shutdown, /// Report a complete sampled profile ReportProfile(Vec), + /// Sent when a media session is activated or deactivated. + /// There can only be a single active media session. + MediaSession(bool), } impl Debug for EmbedderMsg { @@ -194,6 +197,7 @@ impl Debug for EmbedderMsg { EmbedderMsg::AllowOpeningBrowser(..) => write!(f, "AllowOpeningBrowser"), EmbedderMsg::BrowserCreated(..) => write!(f, "BrowserCreated"), EmbedderMsg::ReportProfile(..) => write!(f, "ReportProfile"), + EmbedderMsg::MediaSession(..) => write!(f, "MediaSession"), } } } diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 0984b7c49f3..4f1d0aa6a22 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -130,7 +130,7 @@ pub trait HostTrait { fn get_clipboard_contents(&self) -> Option; /// Sets system clipboard contents fn set_clipboard_contents(&self, contents: String); - /// Called when a media session is activated or deactived. + /// Called when a media session is activated or deactivated. fn on_media_session(&self, active: bool); } @@ -583,6 +583,9 @@ impl ServoGlue { EmbedderMsg::HideIME => { self.callbacks.host_callbacks.on_ime_state_changed(false); }, + EmbedderMsg::MediaSession(active) => { + self.callbacks.host_callbacks.on_media_session(active); + }, EmbedderMsg::Status(..) | EmbedderMsg::SelectFiles(..) | EmbedderMsg::MoveTo(..) | From 4d147d2c56465405e7c3281073ef57fe1bd1c062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 8 Oct 2019 10:18:13 +0200 Subject: [PATCH 12/37] Register media instance with session and prepare communication with embedder --- components/constellation/constellation.rs | 7 +++++ components/script/dom/htmlmediaelement.rs | 10 +++++++- components/script/dom/mediasession.rs | 19 +++++++++++++- components/script/dom/navigator.rs | 22 ++++++++++++++-- components/script/script_thread.rs | 13 ++++++++++ components/script_traits/lib.rs | 31 +++++++++++++++++++++++ components/script_traits/script_msg.rs | 5 ++++ ports/glutin/browser.rs | 4 +++ 8 files changed, 107 insertions(+), 4 deletions(-) diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 8f549ea685e..8b0a7b073a3 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -474,6 +474,9 @@ pub struct Constellation { /// Mechanism to force the compositor to process events. event_loop_waker: Option>, + + /// Browser ID of the active media session, if any. + active_media_session: Option, } /// State needed to construct a constellation. @@ -843,6 +846,7 @@ where glplayer_threads: state.glplayer_threads, player_context: state.player_context, event_loop_waker: state.event_loop_waker, + active_media_session: None, }; constellation.run(); @@ -1774,6 +1778,9 @@ where new_value, ); }, + FromScriptMsg::MediaSessionEventMsg(browser_id, event) => { + // TODO + }, } } diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index d10801c5779..c816a69ca1a 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -15,8 +15,10 @@ use crate::dom::bindings::codegen::Bindings::HTMLMediaElementBinding::HTMLMediaE use crate::dom::bindings::codegen::Bindings::HTMLSourceElementBinding::HTMLSourceElementMethods; use crate::dom::bindings::codegen::Bindings::MediaErrorBinding::MediaErrorConstants::*; use crate::dom::bindings::codegen::Bindings::MediaErrorBinding::MediaErrorMethods; +use crate::dom::bindings::codegen::Bindings::NavigatorBinding::NavigatorBinding::NavigatorMethods; use crate::dom::bindings::codegen::Bindings::NodeBinding::NodeBinding::NodeMethods; use crate::dom::bindings::codegen::Bindings::TextTrackBinding::{TextTrackKind, TextTrackMode}; +use crate::dom::bindings::codegen::Bindings::WindowBinding::WindowBinding::WindowMethods; use crate::dom::bindings::codegen::InheritTypes::{ElementTypeId, HTMLElementTypeId}; use crate::dom::bindings::codegen::InheritTypes::{HTMLMediaElementTypeId, NodeTypeId}; use crate::dom::bindings::codegen::UnionTypes::{ @@ -78,6 +80,7 @@ use net_traits::request::{Destination, Referrer}; use net_traits::{CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseListener, Metadata}; use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType}; use script_layout_interface::HTMLMediaData; +use script_traits::MediaSessionEvent; use servo_config::pref; use servo_media::player::audio::AudioRenderer; use servo_media::player::video::{VideoFrame, VideoFrameRenderer}; @@ -592,7 +595,6 @@ impl HTMLMediaElement { match (old_ready_state, ready_state) { (ReadyState::HaveNothing, ReadyState::HaveMetadata) => { task_source.queue_simple_event(self.upcast(), atom!("loadedmetadata"), &window); - // No other steps are applicable in this case. return; }, @@ -1883,6 +1885,12 @@ impl HTMLMediaElement { self.media_element_load_algorithm(); } } + + fn send_media_session_event(&self, event: MediaSessionEvent) { + let global = self.global(); + let media_session = global.as_window().Navigator().MediaSession(); + media_session.send_event(event); + } } // XXX Placeholder for [https://github.com/servo/servo/issues/22293] diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 139264953ca..f87b0dbee63 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -11,12 +11,13 @@ use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionMe use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionPlaybackState; use crate::dom::bindings::reflector::{reflect_dom_object, DomObject, Reflector}; use crate::dom::bindings::root::{DomRoot, MutNullableDom}; +use crate::dom::htmlmediaelement::HTMLMediaElement; use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; use crate::script_thread::ScriptThread; use dom_struct::dom_struct; use msg::constellation_msg::TopLevelBrowsingContextId; -use script_traits::MediaSessionActionType; +use script_traits::{MediaSessionActionType, MediaSessionEvent, ScriptMsg}; use std::collections::HashMap; use std::rc::Rc; @@ -30,6 +31,9 @@ pub struct MediaSession { /// https://w3c.github.io/mediasession/#supported-media-session-actions #[ignore_malloc_size_of = "Rc"] action_handlers: DomRefCell>>, + /// The media instance controlled by this media session. + /// For now only HTMLMediaElements are controlled by media sessions. + media_instance: MutNullableDom, } impl MediaSession { @@ -40,6 +44,7 @@ impl MediaSession { metadata: Default::default(), playback_state: DomRefCell::new(MediaSessionPlaybackState::None), action_handlers: DomRefCell::new(HashMap::new()), + media_instance: Default::default(), }; ScriptThread::register_media_session(&media_session, browsing_context_id); media_session @@ -63,6 +68,18 @@ impl MediaSession { } // TODO default action. } + + pub fn send_event(&self, event: MediaSessionEvent) { + let global = self.global(); + let browser_id = global + .as_window() + .window_proxy() + .top_level_browsing_context_id(); + let _ = global + .script_to_constellation_chan() + .send(ScriptMsg::MediaSessionEventMsg(browser_id, event)) + .unwrap(); + } } impl MediaSessionMethods for MediaSession { diff --git a/components/script/dom/navigator.rs b/components/script/dom/navigator.rs index 2b0d0ab128b..a759d3dbe54 100644 --- a/components/script/dom/navigator.rs +++ b/components/script/dom/navigator.rs @@ -21,6 +21,7 @@ use crate::dom::promise::Promise; use crate::dom::serviceworkercontainer::ServiceWorkerContainer; use crate::dom::window::Window; use crate::dom::xr::XR; +use crate::script_thread::ScriptThread; use dom_struct::dom_struct; use std::rc::Rc; @@ -192,7 +193,24 @@ impl NavigatorMethods for Navigator { /// https://w3c.github.io/mediasession/#dom-navigator-mediasession fn MediaSession(&self) -> DomRoot { - self.mediasession - .or_init(|| MediaSession::new(self.global().as_window())) + self.mediasession.or_init(|| { + // There is a single MediaSession instance per top level browsing context + // and only one active MediaSession globally. + // + // MediaSession creation can happen in two cases: + // + // - If content gets `navigator.mediaSession` + // - If a media instance (HTMLMediaElement so far) starts playing media. + // + // The MediaSession constructor is in charge of registering itself with + // the script thread. + let global = self.global(); + let window = global.as_window(); + let browsing_context_id = window.window_proxy().top_level_browsing_context_id(); + match ScriptThread::get_media_session(browsing_context_id) { + Some(session) => session, + None => MediaSession::new(window), + } + }) } } diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index ec1f5852060..e7b39d26420 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -3993,6 +3993,19 @@ impl ScriptThread { .remove(&browsing_context_id); }) } + + pub fn get_media_session( + browsing_context_id: TopLevelBrowsingContextId, + ) -> Option> { + SCRIPT_THREAD_ROOT.with(|root| { + let script_thread = unsafe { &*root.get().unwrap() }; + script_thread + .media_sessions + .borrow() + .get(&browsing_context_id) + .map(|s| DomRoot::from_ref(&**s)) + }) + } } impl Drop for ScriptThread { diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index b4b85889ea0..f5006f1df6f 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -1087,3 +1087,34 @@ pub enum MediaSessionActionType { /// The action intent is to move the playback time to a specific time. SeekTo, } + +/// https://w3c.github.io/mediasession/#mediametadata +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MediaMetadata { + /// Title + pub title: String, + /// Artist + pub artist: Option, + /// Album + pub album: Option, +} + +/// https://w3c.github.io/mediasession/#enumdef-mediasessionplaybackstate +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum MediaSessionPlaybackState { + /// The browsing context does not specify whether it’s playing or paused. + None_, + /// The browsing context has paused media and it can be resumed. + Playing, + /// The browsing context is currently playing media and it can be paused. + Paused, +} + +/// Type of events sent from script to the constellation about the media session. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum MediaSessionEvent { + /// Indicates that the media metadata is available. + SetMetadata(MediaMetadata), + /// Indicates that the playback state has changed. + PlaybackStateChange(MediaSessionPlaybackState), +} diff --git a/components/script_traits/script_msg.rs b/components/script_traits/script_msg.rs index a12101380c9..df1889ab6b2 100644 --- a/components/script_traits/script_msg.rs +++ b/components/script_traits/script_msg.rs @@ -8,6 +8,7 @@ use crate::DocumentState; use crate::IFrameLoadInfoWithData; use crate::LayoutControlMsg; use crate::LoadData; +use crate::MediaSessionEvent; use crate::MessagePortMsg; use crate::PortMessageTask; use crate::StructuredSerializedData; @@ -254,6 +255,9 @@ pub enum ScriptMsg { GetScreenSize(IpcSender), /// Get the available screen size (pixel) GetScreenAvailSize(IpcSender), + /// Notifies the constellation about media session events + /// (i.e. when there is metadata for the active media session, playback state changes...). + MediaSessionEventMsg(TopLevelBrowsingContextId, MediaSessionEvent), } impl fmt::Debug for ScriptMsg { @@ -305,6 +309,7 @@ impl fmt::Debug for ScriptMsg { GetClientWindow(..) => "GetClientWindow", GetScreenSize(..) => "GetScreenSize", GetScreenAvailSize(..) => "GetScreenAvailSize", + MediaSessionEventMsg(..) => "MediaSessionEventMsg", }; write!(formatter, "ScriptMsg::{}", variant) } diff --git a/ports/glutin/browser.rs b/ports/glutin/browser.rs index e3da4d13d61..95bea1e72bb 100644 --- a/ports/glutin/browser.rs +++ b/ports/glutin/browser.rs @@ -449,6 +449,10 @@ where error!("Failed to store profile: {}", e); } }, + EmbedderMsg::MediaSession(_) => { + debug!("MediaSession received"); + // TODO(ferjm) + }, } } } From 89d9e3ad78ce4698dfab9b2fc310a20f4964e380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 9 Oct 2019 12:16:35 +0200 Subject: [PATCH 13/37] Introduce embedder MediaSessionEvent and move active session to Servo --- components/constellation/constellation.rs | 3 -- components/embedder_traits/lib.rs | 39 ++++++++++++++++++++--- components/script/dom/htmlmediaelement.rs | 10 +++++- components/script/dom/mediasession.rs | 13 +++----- components/script_traits/lib.rs | 31 ------------------ components/script_traits/script_msg.rs | 4 --- components/servo/lib.rs | 31 ++++++++++++++++++ ports/libsimpleservo/api/src/lib.rs | 9 +++--- ports/libsimpleservo/capi/src/lib.rs | 11 ++++--- 9 files changed, 90 insertions(+), 61 deletions(-) diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 8b0a7b073a3..db9d94e0c88 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -1778,9 +1778,6 @@ where new_value, ); }, - FromScriptMsg::MediaSessionEventMsg(browser_id, event) => { - // TODO - }, } } diff --git a/components/embedder_traits/lib.rs b/components/embedder_traits/lib.rs index da526d5f478..7d967e46277 100644 --- a/components/embedder_traits/lib.rs +++ b/components/embedder_traits/lib.rs @@ -162,9 +162,9 @@ pub enum EmbedderMsg { Shutdown, /// Report a complete sampled profile ReportProfile(Vec), - /// Sent when a media session is activated or deactivated. - /// There can only be a single active media session. - MediaSession(bool), + /// Notifies the embedder about media session events + /// (i.e. when there is metadata for the active media session, playback state changes...). + MediaSessionEvent(MediaSessionEvent), } impl Debug for EmbedderMsg { @@ -197,7 +197,7 @@ impl Debug for EmbedderMsg { EmbedderMsg::AllowOpeningBrowser(..) => write!(f, "AllowOpeningBrowser"), EmbedderMsg::BrowserCreated(..) => write!(f, "BrowserCreated"), EmbedderMsg::ReportProfile(..) => write!(f, "ReportProfile"), - EmbedderMsg::MediaSession(..) => write!(f, "MediaSession"), + EmbedderMsg::MediaSessionEvent(..) => write!(f, "MediaSessionEvent"), } } } @@ -206,3 +206,34 @@ impl Debug for EmbedderMsg { /// the `String` content is expected to be extension (e.g, "doc", without the prefixing ".") #[derive(Clone, Debug, Deserialize, Serialize)] pub struct FilterPattern(pub String); + +/// https://w3c.github.io/mediasession/#mediametadata +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MediaMetadata { + /// Title + pub title: String, + /// Artist + pub artist: Option, + /// Album + pub album: Option, +} + +/// https://w3c.github.io/mediasession/#enumdef-mediasessionplaybackstate +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum MediaSessionPlaybackState { + /// The browsing context does not specify whether it’s playing or paused. + None_, + /// The browsing context has paused media and it can be resumed. + Playing, + /// The browsing context is currently playing media and it can be paused. + Paused, +} + +/// Type of events sent from script to the embedder about the media session. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum MediaSessionEvent { + /// Indicates that the media metadata is available. + SetMetadata(MediaMetadata), + /// Indicates that the playback state has changed. + PlaybackStateChange(MediaSessionPlaybackState), +} diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index c816a69ca1a..5b6c0d35e93 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -67,6 +67,7 @@ use crate::script_thread::ScriptThread; use crate::task_source::TaskSource; use dom_struct::dom_struct; use embedder_traits::resources::{self, Resource as EmbedderResource}; +use embedder_traits::{MediaMetadata, MediaSessionEvent}; use euclid::default::Size2D; use headers::{ContentLength, ContentRange, HeaderMapExt}; use html5ever::{LocalName, Prefix}; @@ -80,7 +81,6 @@ use net_traits::request::{Destination, Referrer}; use net_traits::{CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseListener, Metadata}; use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType}; use script_layout_interface::HTMLMediaData; -use script_traits::MediaSessionEvent; use servo_config::pref; use servo_media::player::audio::AudioRenderer; use servo_media::player::video::{VideoFrame, VideoFrameRenderer}; @@ -1727,6 +1727,14 @@ impl HTMLMediaElement { if self.Controls() { self.render_controls(); } + + // Send a media session event with the obtained metadata. + self.send_media_session_event(MediaSessionEvent::SetMetadata(MediaMetadata { + // TODO(ferjm) set url if no title. + title: metadata.title.clone().unwrap_or("".to_string()), + artist: None, + album: None, + })); }, PlayerEvent::NeedData => { // The player needs more data. diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index f87b0dbee63..ef64170f312 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -16,8 +16,9 @@ use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; use crate::script_thread::ScriptThread; use dom_struct::dom_struct; +use embedder_traits::{EmbedderMsg, MediaSessionEvent}; use msg::constellation_msg::TopLevelBrowsingContextId; -use script_traits::{MediaSessionActionType, MediaSessionEvent, ScriptMsg}; +use script_traits::MediaSessionActionType; use std::collections::HashMap; use std::rc::Rc; @@ -71,14 +72,8 @@ impl MediaSession { pub fn send_event(&self, event: MediaSessionEvent) { let global = self.global(); - let browser_id = global - .as_window() - .window_proxy() - .top_level_browsing_context_id(); - let _ = global - .script_to_constellation_chan() - .send(ScriptMsg::MediaSessionEventMsg(browser_id, event)) - .unwrap(); + let window = global.as_window(); + window.send_to_embedder(EmbedderMsg::MediaSessionEvent(event)); } } diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index f5006f1df6f..b4b85889ea0 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -1087,34 +1087,3 @@ pub enum MediaSessionActionType { /// The action intent is to move the playback time to a specific time. SeekTo, } - -/// https://w3c.github.io/mediasession/#mediametadata -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct MediaMetadata { - /// Title - pub title: String, - /// Artist - pub artist: Option, - /// Album - pub album: Option, -} - -/// https://w3c.github.io/mediasession/#enumdef-mediasessionplaybackstate -#[derive(Clone, Debug, Deserialize, Serialize)] -pub enum MediaSessionPlaybackState { - /// The browsing context does not specify whether it’s playing or paused. - None_, - /// The browsing context has paused media and it can be resumed. - Playing, - /// The browsing context is currently playing media and it can be paused. - Paused, -} - -/// Type of events sent from script to the constellation about the media session. -#[derive(Clone, Debug, Deserialize, Serialize)] -pub enum MediaSessionEvent { - /// Indicates that the media metadata is available. - SetMetadata(MediaMetadata), - /// Indicates that the playback state has changed. - PlaybackStateChange(MediaSessionPlaybackState), -} diff --git a/components/script_traits/script_msg.rs b/components/script_traits/script_msg.rs index df1889ab6b2..64d6b1f5389 100644 --- a/components/script_traits/script_msg.rs +++ b/components/script_traits/script_msg.rs @@ -255,9 +255,6 @@ pub enum ScriptMsg { GetScreenSize(IpcSender), /// Get the available screen size (pixel) GetScreenAvailSize(IpcSender), - /// Notifies the constellation about media session events - /// (i.e. when there is metadata for the active media session, playback state changes...). - MediaSessionEventMsg(TopLevelBrowsingContextId, MediaSessionEvent), } impl fmt::Debug for ScriptMsg { @@ -309,7 +306,6 @@ impl fmt::Debug for ScriptMsg { GetClientWindow(..) => "GetClientWindow", GetScreenSize(..) => "GetScreenSize", GetScreenAvailSize(..) => "GetScreenAvailSize", - MediaSessionEventMsg(..) => "MediaSessionEventMsg", }; write!(formatter, "ScriptMsg::{}", variant) } diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 67751234111..3da82d299fd 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -83,6 +83,7 @@ use constellation::{Constellation, InitialConstellationState, UnprivilegedPipeli use constellation::{FromCompositorLogger, FromScriptLogger}; use crossbeam_channel::{unbounded, Sender}; use embedder_traits::{EmbedderMsg, EmbedderProxy, EmbedderReceiver, EventLoopWaker}; +use embedder_traits::{MediaSessionEvent, MediaSessionPlaybackState}; use env_logger::Builder as EnvLoggerBuilder; use euclid::{Scale, Size2D}; #[cfg(all( @@ -270,6 +271,9 @@ pub struct Servo { embedder_receiver: EmbedderReceiver, embedder_events: Vec<(Option, EmbedderMsg)>, profiler_enabled: bool, + webgl_thread_data: Option>, + /// Browser ID of the active media session, if any. + active_media_session: Option, } #[derive(Clone)] @@ -555,6 +559,8 @@ where embedder_receiver: embedder_receiver, embedder_events: Vec::new(), profiler_enabled: false, + webgl_thread_data, + active_media_session: None, } } @@ -744,6 +750,31 @@ where self.embedder_events.push(event); }, + (EmbedderMsg::MediaSessionEvent(event), ShutdownState::NotShuttingDown) => { + // Unlikely at this point, but we may receive events coming from + // different media sessions, so we set the active media session based + // on Playing events. + // The last media session claiming to be in playing state is set to + // the active media session. + // Events coming from inactive media sessions are discarded. + if self.active_media_session.is_some() { + match event { + MediaSessionEvent::PlaybackStateChange(ref state) => { + match state { + MediaSessionPlaybackState::Playing => (), + _ => return, + }; + }, + _ => (), + }; + } + self.active_media_session = top_level_browsing_context; + self.embedder_events.push(( + top_level_browsing_context, + EmbedderMsg::MediaSessionEvent(event), + )); + }, + (msg, ShutdownState::NotShuttingDown) => { self.embedder_events.push((top_level_browsing_context, msg)); }, diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 4f1d0aa6a22..b7f0d3587b4 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -42,6 +42,7 @@ thread_local! { /// It will be called to notify embedder that some events are available, /// and that perform_updates need to be called pub use servo::embedder_traits::EventLoopWaker; +pub use servo::embedder_traits::MediaSessionEvent; pub struct InitOptions { pub args: Vec, @@ -130,8 +131,8 @@ pub trait HostTrait { fn get_clipboard_contents(&self) -> Option; /// Sets system clipboard contents fn set_clipboard_contents(&self, contents: String); - /// Called when a media session is activated or deactivated. - fn on_media_session(&self, active: bool); + /// Called when there is a new media session event. + fn on_media_session_event(&self, event: MediaSessionEvent); } pub struct ServoGlue { @@ -583,8 +584,8 @@ impl ServoGlue { EmbedderMsg::HideIME => { self.callbacks.host_callbacks.on_ime_state_changed(false); }, - EmbedderMsg::MediaSession(active) => { - self.callbacks.host_callbacks.on_media_session(active); + EmbedderMsg::MediaSessionEvent(event) => { + self.callbacks.host_callbacks.on_media_session_event(event); }, EmbedderMsg::Status(..) | EmbedderMsg::SelectFiles(..) | diff --git a/ports/libsimpleservo/capi/src/lib.rs b/ports/libsimpleservo/capi/src/lib.rs index 4fbd239000e..57043407879 100644 --- a/ports/libsimpleservo/capi/src/lib.rs +++ b/ports/libsimpleservo/capi/src/lib.rs @@ -17,7 +17,7 @@ use env_logger; use log::LevelFilter; use simpleservo::{self, gl_glue, ServoGlue, SERVO}; use simpleservo::{ - Coordinates, EventLoopWaker, HostTrait, InitOptions, MouseButton, VRInitOptions, + Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionEvent, MouseButton, VRInitOptions, }; use std::ffi::{CStr, CString}; #[cfg(target_os = "windows")] @@ -216,7 +216,8 @@ pub struct CHostCallbacks { pub on_ime_state_changed: extern "C" fn(show: bool), pub get_clipboard_contents: extern "C" fn() -> *const c_char, pub set_clipboard_contents: extern "C" fn(contents: *const c_char), - pub on_media_session: extern "C" fn(active: bool), + // TODO(ferjm) pass C representation of media event. + pub on_media_session_event: extern "C" fn(), } /// Servo options @@ -710,8 +711,8 @@ impl HostTrait for HostCallbacks { (self.0.set_clipboard_contents)(contents.as_ptr()); } - fn on_media_session(&self, active: bool) { - debug!("on_media_session (active: {:?})", active); - (self.0.on_media_session)(active); + fn on_media_session_event(&self, event: MediaSessionEvent) { + debug!("on_media_session_event (event: {:?})", event); + (self.0.on_media_session_event)(); } } From 7ca74d598cedd5a826e3b6c9e37cee0f219f77ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 9 Oct 2019 18:06:27 +0200 Subject: [PATCH 14/37] MediaSession: refactor embedder API. Fix Android build --- components/embedder_traits/lib.rs | 3 ++- ports/glutin/browser.rs | 6 ++--- ports/libsimpleservo/api/src/lib.rs | 32 ++++++++++++++++++++------ ports/libsimpleservo/capi/src/lib.rs | 27 ++++++++++++++++++---- ports/libsimpleservo/jniapi/src/lib.rs | 14 ++++++++++- 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/components/embedder_traits/lib.rs b/components/embedder_traits/lib.rs index 7d967e46277..94ae2493b68 100644 --- a/components/embedder_traits/lib.rs +++ b/components/embedder_traits/lib.rs @@ -219,10 +219,11 @@ pub struct MediaMetadata { } /// https://w3c.github.io/mediasession/#enumdef-mediasessionplaybackstate +#[repr(i32)] #[derive(Clone, Debug, Deserialize, Serialize)] pub enum MediaSessionPlaybackState { /// The browsing context does not specify whether it’s playing or paused. - None_, + None_ = 1, /// The browsing context has paused media and it can be resumed. Playing, /// The browsing context is currently playing media and it can be paused. diff --git a/ports/glutin/browser.rs b/ports/glutin/browser.rs index 95bea1e72bb..976026a483b 100644 --- a/ports/glutin/browser.rs +++ b/ports/glutin/browser.rs @@ -449,9 +449,9 @@ where error!("Failed to store profile: {}", e); } }, - EmbedderMsg::MediaSession(_) => { - debug!("MediaSession received"); - // TODO(ferjm) + EmbedderMsg::MediaSessionEvent(_) => { + debug!("MediaSessionEvent received"); + // TODO(ferjm): MediaSession support for Glutin based browsers. }, } } diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index b7f0d3587b4..894928bc2b8 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -15,7 +15,7 @@ use servo::compositing::windowing::{ WindowMethods, }; use servo::embedder_traits::resources::{self, Resource, ResourceReaderMethods}; -use servo::embedder_traits::EmbedderMsg; +use servo::embedder_traits::{EmbedderMsg, MediaSessionEvent}; use servo::euclid::{Point2D, Rect, Scale, Size2D, Vector2D}; use servo::keyboard_types::{Key, KeyState, KeyboardEvent}; use servo::msg::constellation_msg::TraversalDirection; @@ -42,7 +42,6 @@ thread_local! { /// It will be called to notify embedder that some events are available, /// and that perform_updates need to be called pub use servo::embedder_traits::EventLoopWaker; -pub use servo::embedder_traits::MediaSessionEvent; pub struct InitOptions { pub args: Vec, @@ -127,12 +126,19 @@ pub trait HostTrait { fn on_shutdown_complete(&self); /// A text input is focused. fn on_ime_state_changed(&self, show: bool); - /// Gets sytem clipboard contents + /// Gets sytem clipboard contents. fn get_clipboard_contents(&self) -> Option; - /// Sets system clipboard contents + /// Sets system clipboard contents. fn set_clipboard_contents(&self, contents: String); - /// Called when there is a new media session event. - fn on_media_session_event(&self, event: MediaSessionEvent); + /// Called when we get the media session metadata/ + fn on_media_session_metadata( + &self, + title: String, + artist: Option, + album: Option, + ); + /// Called when the media sessoin playback state changes. + fn on_media_session_playback_state_change(&self, state: i32); } pub struct ServoGlue { @@ -585,7 +591,19 @@ impl ServoGlue { self.callbacks.host_callbacks.on_ime_state_changed(false); }, EmbedderMsg::MediaSessionEvent(event) => { - self.callbacks.host_callbacks.on_media_session_event(event); + match event { + MediaSessionEvent::SetMetadata(metadata) => { + self.callbacks.host_callbacks.on_media_session_metadata( + metadata.title, + metadata.artist, + metadata.album, + ) + }, + MediaSessionEvent::PlaybackStateChange(state) => self + .callbacks + .host_callbacks + .on_media_session_playback_state_change(state as i32), + }; }, EmbedderMsg::Status(..) | EmbedderMsg::SelectFiles(..) | diff --git a/ports/libsimpleservo/capi/src/lib.rs b/ports/libsimpleservo/capi/src/lib.rs index 57043407879..31a066d9416 100644 --- a/ports/libsimpleservo/capi/src/lib.rs +++ b/ports/libsimpleservo/capi/src/lib.rs @@ -216,8 +216,9 @@ pub struct CHostCallbacks { pub on_ime_state_changed: extern "C" fn(show: bool), pub get_clipboard_contents: extern "C" fn() -> *const c_char, pub set_clipboard_contents: extern "C" fn(contents: *const c_char), - // TODO(ferjm) pass C representation of media event. - pub on_media_session_event: extern "C" fn(), + pub on_media_session_metadata: + extern "C" fn(title: *const c_char, album: *const c_char, artist: *const c_char), + pub on_media_session_playback_state_change: extern "C" fn(state: i32), } /// Servo options @@ -711,8 +712,24 @@ impl HostTrait for HostCallbacks { (self.0.set_clipboard_contents)(contents.as_ptr()); } - fn on_media_session_event(&self, event: MediaSessionEvent) { - debug!("on_media_session_event (event: {:?})", event); - (self.0.on_media_session_event)(); + fn on_media_session_metadata( + &self, + title: String, + artist: Option, + album: Option, + ) { + debug!( + "on_media_session_metadata ({:?} {:?} {:?})", + title, artist, album + ); + let title = CString::new(title).expect("Can't create string"); + let artist = CString::new(artist.unwrap_or(String::new())).expect("Can't create string"); + let album = CString::new(album.unwrap_or(String::new())).expect("Can't create string"); + (self.0.on_media_session_metadata)(title.as_ptr(), artist.as_ptr(), album.as_ptr()); + } + + fn on_media_session_playback_state_change(&self, state: i32) { + debug!("on_media_session_playback_state_change {:?}", state); + (self.0.on_media_session_playback_state_change)(state); } } diff --git a/ports/libsimpleservo/jniapi/src/lib.rs b/ports/libsimpleservo/jniapi/src/lib.rs index 963e01dec6d..fe6538f9c2e 100644 --- a/ports/libsimpleservo/jniapi/src/lib.rs +++ b/ports/libsimpleservo/jniapi/src/lib.rs @@ -15,7 +15,9 @@ use jni::{errors, JNIEnv, JavaVM}; use libc::{dup2, pipe, read}; use log::Level; use simpleservo::{self, gl_glue, ServoGlue, SERVO}; -use simpleservo::{Coordinates, EventLoopWaker, HostTrait, InitOptions, VRInitOptions}; +use simpleservo::{ + Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionEvent, VRInitOptions, +}; use std::os::raw::{c_char, c_int, c_void}; use std::ptr::{null, null_mut}; use std::sync::Arc; @@ -508,6 +510,16 @@ impl HostTrait for HostCallbacks { } fn set_clipboard_contents(&self, _contents: String) {} + + fn on_media_session_metadata( + &self, + _title: String, + _artist: Option, + _album: Option, + ) { + } + + fn on_media_session_playback_state_change(&self, state: i32) {} } fn initialize_android_glue(env: &JNIEnv, activity: JObject) { From 3f6569447d552ae861f61c73333cfb20bbba7d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 10 Oct 2019 17:09:55 +0200 Subject: [PATCH 15/37] MediaSession Android bits --- ports/libsimpleservo/jniapi/src/lib.rs | 4 +--- .../java/org/mozilla/servo/MainActivity.java | 17 +++++++++++++++++ .../java/org/mozilla/servoview/JNIServo.java | 4 ++++ .../main/java/org/mozilla/servoview/Servo.java | 12 ++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/ports/libsimpleservo/jniapi/src/lib.rs b/ports/libsimpleservo/jniapi/src/lib.rs index fe6538f9c2e..ff91408244a 100644 --- a/ports/libsimpleservo/jniapi/src/lib.rs +++ b/ports/libsimpleservo/jniapi/src/lib.rs @@ -15,9 +15,7 @@ use jni::{errors, JNIEnv, JavaVM}; use libc::{dup2, pipe, read}; use log::Level; use simpleservo::{self, gl_glue, ServoGlue, SERVO}; -use simpleservo::{ - Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionEvent, VRInitOptions, -}; +use simpleservo::{Coordinates, EventLoopWaker, HostTrait, InitOptions, VRInitOptions}; use std::os::raw::{c_char, c_int, c_void}; use std::ptr::{null, null_mut}; use std::sync::Arc; diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java index e4ef5a51b52..6a2117962dd 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java @@ -23,6 +23,8 @@ import android.widget.ProgressBar; import android.widget.TextView; import android.util.Log; +//import org.mozilla.servo.MediaSessionCallback; + import org.mozilla.servoview.ServoView; import org.mozilla.servoview.Servo; @@ -41,6 +43,7 @@ public class MainActivity extends Activity implements Servo.Client { ProgressBar mProgressBar; TextView mIdleText; boolean mCanGoBack; + // MediaSession mSession; @Override protected void onCreate(Bundle savedInstanceState) { @@ -217,4 +220,18 @@ public class MainActivity extends Activity implements Servo.Client { super.onBackPressed(); } } + + @Override + public void onMediaSessionMetadata(String title, String artist, String album) { + Log.d("SERVOMEDIA", "METADATA"); + } + + @Override + public void onMediaSessionPlaybackStateChange(int state) { + Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED"); + } + +/* private void createMediaSession() { + mSession = new MediaSession(this, "ServoMediaSession"); + }*/ } diff --git a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java index 7bfcf233d8d..b1ce302e0aa 100644 --- a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java +++ b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java @@ -109,6 +109,10 @@ public class JNIServo { void onHistoryChanged(boolean canGoBack, boolean canGoForward); void onShutdownComplete(); + + void onMediaSessionMetadata(String title, String artist, String album); + + void onMediaSessionPlaybackStateChange(int state); } } diff --git a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java index dac970d1acf..7daea963bb5 100644 --- a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java +++ b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java @@ -184,6 +184,10 @@ public class Servo { void onHistoryChanged(boolean canGoBack, boolean canGoForward); void onRedrawing(boolean redrawing); + + void onMediaSessionMetadata(String title, String artist, String album); + + void onMediaSessionPlaybackStateChange(int state); } public interface RunCallback { @@ -269,5 +273,13 @@ public class Servo { public void onRedrawing(boolean redrawing) { mRunCallback.inUIThread(() -> mClient.onRedrawing(redrawing)); } + + public void onMediaSessionMetadata(String title, String artist, String album) { + mRunCallback.inUIThread(() -> mClient.onMediaSessionMetadata(title, artist, album)); + } + + public void onMediaSessionPlaybackStateChange(int state) { + mRunCallback.inUIThread(() -> mClient.onMediaSessionPlaybackStateChange(state)); + } } } From 761f21fc8b136aa76bbbc3743a762779bf8284d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 16 Oct 2019 11:33:17 +0200 Subject: [PATCH 16/37] Send MediaSessionEvent::PlaybackStateChange when needed --- components/script/dom/htmlmediaelement.rs | 30 +++++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 5b6c0d35e93..57a7332713f 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -67,7 +67,7 @@ use crate::script_thread::ScriptThread; use crate::task_source::TaskSource; use dom_struct::dom_struct; use embedder_traits::resources::{self, Resource as EmbedderResource}; -use embedder_traits::{MediaMetadata, MediaSessionEvent}; +use embedder_traits::{MediaMetadata, MediaSessionEvent, MediaSessionPlaybackState}; use euclid::default::Size2D; use headers::{ContentLength, ContentRange, HeaderMapExt}; use html5ever::{LocalName, Prefix}; @@ -1792,13 +1792,27 @@ impl HTMLMediaElement { }; ScriptThread::await_stable_state(Microtask::MediaElement(task)); }, - PlayerEvent::StateChanged(ref state) => match *state { - PlaybackState::Paused => { - if self.ready_state.get() == ReadyState::HaveMetadata { - self.change_ready_state(ReadyState::HaveEnoughData); - } - }, - _ => {}, + PlayerEvent::StateChanged(ref state) => { + let mut media_session_playback_state = MediaSessionPlaybackState::None_; + match *state { + PlaybackState::Paused => { + media_session_playback_state = MediaSessionPlaybackState::Paused; + if self.ready_state.get() == ReadyState::HaveMetadata { + self.change_ready_state(ReadyState::HaveEnoughData); + } + }, + PlaybackState::Playing => { + media_session_playback_state = MediaSessionPlaybackState::Playing; + }, + _ => {}, + }; + println!( + "Sending media session event playback state changed to {:?}", + media_session_playback_state + ); + self.send_media_session_event(MediaSessionEvent::PlaybackStateChange( + media_session_playback_state, + )); }, } } From dd63ba425f6e910275b1bfa597a91827df10bde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 18 Oct 2019 12:02:47 +0200 Subject: [PATCH 17/37] MediaSession show media controls on Android --- components/embedder_traits/lib.rs | 4 +- components/script/dom/htmlmediaelement.rs | 8 +- ports/libsimpleservo/api/src/lib.rs | 11 +- ports/libsimpleservo/jniapi/src/lib.rs | 46 +++++- .../java/org/mozilla/servo/MainActivity.java | 136 +++++++++++++++++- .../main/res/drawable/media_session_icon.png | Bin 0 -> 213 bytes .../main/res/drawable/media_session_next.png | Bin 0 -> 203 bytes .../main/res/drawable/media_session_pause.png | Bin 0 -> 114 bytes .../main/res/drawable/media_session_prev.png | Bin 0 -> 205 bytes .../main/res/drawable/media_session_stop.png | Bin 0 -> 106 bytes .../servoapp/src/main/res/values/strings.xml | 4 +- 11 files changed, 183 insertions(+), 26 deletions(-) create mode 100755 support/android/apk/servoapp/src/main/res/drawable/media_session_icon.png create mode 100755 support/android/apk/servoapp/src/main/res/drawable/media_session_next.png create mode 100755 support/android/apk/servoapp/src/main/res/drawable/media_session_pause.png create mode 100755 support/android/apk/servoapp/src/main/res/drawable/media_session_prev.png create mode 100755 support/android/apk/servoapp/src/main/res/drawable/media_session_stop.png diff --git a/components/embedder_traits/lib.rs b/components/embedder_traits/lib.rs index 94ae2493b68..f65ed514f4d 100644 --- a/components/embedder_traits/lib.rs +++ b/components/embedder_traits/lib.rs @@ -224,9 +224,9 @@ pub struct MediaMetadata { pub enum MediaSessionPlaybackState { /// The browsing context does not specify whether it’s playing or paused. None_ = 1, - /// The browsing context has paused media and it can be resumed. - Playing, /// The browsing context is currently playing media and it can be paused. + Playing, + /// The browsing context has paused media and it can be resumed. Paused, } diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 57a7332713f..3204871d459 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -1804,9 +1804,15 @@ impl HTMLMediaElement { PlaybackState::Playing => { media_session_playback_state = MediaSessionPlaybackState::Playing; }, + PlaybackState::Buffering => { + // Do not send the media session playback state change event + // in this case as a None_ state is expected to clean up the + // session. + return; + }, _ => {}, }; - println!( + debug!( "Sending media session event playback state changed to {:?}", media_session_playback_state ); diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 894928bc2b8..3eb28992427 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -131,12 +131,7 @@ pub trait HostTrait { /// Sets system clipboard contents. fn set_clipboard_contents(&self, contents: String); /// Called when we get the media session metadata/ - fn on_media_session_metadata( - &self, - title: String, - artist: Option, - album: Option, - ); + fn on_media_session_metadata(&self, title: String, artist: String, album: String); /// Called when the media sessoin playback state changes. fn on_media_session_playback_state_change(&self, state: i32); } @@ -595,8 +590,8 @@ impl ServoGlue { MediaSessionEvent::SetMetadata(metadata) => { self.callbacks.host_callbacks.on_media_session_metadata( metadata.title, - metadata.artist, - metadata.album, + metadata.artist.unwrap_or("".to_owned()), + metadata.album.unwrap_or("".to_owned()), ) }, MediaSessionEvent::PlaybackStateChange(state) => self diff --git a/ports/libsimpleservo/jniapi/src/lib.rs b/ports/libsimpleservo/jniapi/src/lib.rs index ff91408244a..44b6207d3e6 100644 --- a/ports/libsimpleservo/jniapi/src/lib.rs +++ b/ports/libsimpleservo/jniapi/src/lib.rs @@ -509,15 +509,47 @@ impl HostTrait for HostCallbacks { fn set_clipboard_contents(&self, _contents: String) {} - fn on_media_session_metadata( - &self, - _title: String, - _artist: Option, - _album: Option, - ) { + fn on_media_session_metadata(&self, title: String, artist: String, album: String) { + info!("on_media_session_metadata"); + let env = self.jvm.get_env().unwrap(); + let title = match new_string(&env, &title) { + Ok(s) => s, + Err(_) => return, + }; + let title = JValue::Object(JObject::from(title)); + + let artist = match new_string(&env, &artist) { + Ok(s) => s, + Err(_) => return, + }; + let artist = JValue::Object(JObject::from(artist)); + + let album = match new_string(&env, &album) { + Ok(s) => s, + Err(_) => return, + }; + let album = JValue::Object(JObject::from(album)); + env.call_method( + self.callbacks.as_obj(), + "onMediaSessionMetadata", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V", + &[title, artist, album], + ) + .unwrap(); } - fn on_media_session_playback_state_change(&self, state: i32) {} + fn on_media_session_playback_state_change(&self, state: i32) { + info!("on_media_session_playback_state_change {:?}", state); + let env = self.jvm.get_env().unwrap(); + let state = JValue::Int(state as jint); + env.call_method( + self.callbacks.as_obj(), + "onMediaSessionPlaybackStateChange", + "(I)V", + &[state], + ) + .unwrap(); + } } fn initialize_android_glue(env: &JNIEnv, activity: JObject) { diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java index 6a2117962dd..ddff2e57b1a 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java @@ -7,9 +7,16 @@ package org.mozilla.servo; import android.app.Activity; import android.app.AlertDialog; +import android.app.PendingIntent; +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; +import android.content.IntentFilter; import android.net.Uri; +import android.os.Build; import android.os.Bundle; import android.system.ErrnoException; import android.system.Os; @@ -23,8 +30,6 @@ import android.widget.ProgressBar; import android.widget.TextView; import android.util.Log; -//import org.mozilla.servo.MediaSessionCallback; - import org.mozilla.servoview.ServoView; import org.mozilla.servoview.Servo; @@ -32,7 +37,24 @@ import java.io.File; public class MainActivity extends Activity implements Servo.Client { + private class NotificationID { + private int lastID = 0; + public int getNext() { + lastID++; + return lastID; + } + + public int get() { + return lastID; + } + } + private static final String LOGTAG = "MainActivity"; + private static final String MEDIA_CHANNEL_ID = "MediaNotificationChannel"; + private static final String KEY_MEDIA_PAUSE = "org.mozilla.servoview.MainActivity.pause"; + private static final String KEY_MEDIA_PREV = "org.mozilla.servoview.MainActivity.prev"; + private static final String KEY_MEDIA_NEXT = "org.mozilla.servoview.MainActivity.next"; + private static final String KEY_MEDIA_STOP = "org.mozilla.servoview.MainActivity.stop"; ServoView mServoView; Button mBackButton; @@ -43,7 +65,8 @@ public class MainActivity extends Activity implements Servo.Client { ProgressBar mProgressBar; TextView mIdleText; boolean mCanGoBack; - // MediaSession mSession; + NotificationID mNotificationID; + BroadcastReceiver mMediaSessionActionReceiver; @Override protected void onCreate(Bundle savedInstanceState) { @@ -86,6 +109,16 @@ public class MainActivity extends Activity implements Servo.Client { mServoView.loadUri(intent.getData()); } setupUrlField(); + + mNotificationID = new NotificationID(); + createMediaNotificationChannel(); + } + + @Override + protected void onDestroy() { + Log.d("SERVOMEDIA", "onDestroy"); + super.onDestroy(); + hideMediaSessionControls(); } private void setupUrlField() { @@ -206,6 +239,7 @@ public class MainActivity extends Activity implements Servo.Client { mServoView.onPause(); super.onPause(); } + @Override public void onResume() { mServoView.onResume(); @@ -228,10 +262,98 @@ public class MainActivity extends Activity implements Servo.Client { @Override public void onMediaSessionPlaybackStateChange(int state) { - Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED"); + Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED " + state); + if (state == 1 /* none */) { + hideMediaSessionControls(); + return; + } + if (state == 2 /* playing */) { + showMediaSessionControls(); + return; + } } -/* private void createMediaSession() { - mSession = new MediaSession(this, "ServoMediaSession"); - }*/ + private void createMediaNotificationChannel() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + CharSequence name = getString(R.string.media_channel_name); + String description = getString(R.string.media_channel_description); + int importance = NotificationManager.IMPORTANCE_DEFAULT; + NotificationChannel channel = new NotificationChannel(MEDIA_CHANNEL_ID, name, importance); + channel.setDescription(description); + NotificationManager notificationManager = getSystemService(NotificationManager.class); + notificationManager.createNotificationChannel(channel); + } + } + + private void showMediaSessionControls() { + Context context = getApplicationContext(); + + IntentFilter filter = new IntentFilter(); + filter.addAction(KEY_MEDIA_PAUSE); + filter.addAction(KEY_MEDIA_STOP); + filter.addAction(KEY_MEDIA_PREV); + filter.addAction(KEY_MEDIA_NEXT); + + mMediaSessionActionReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + if (intent.getAction().equals(KEY_MEDIA_PAUSE)) { + Log.d("SERVOMEDIA", "PAUSE"); + } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { + Log.d("SERVOMEDIA", "STOP"); + } else if (intent.getAction().equals(KEY_MEDIA_NEXT)) { + Log.d("SERVOMEDIA", "NEXT"); + } else if (intent.getAction().equals(KEY_MEDIA_PREV)) { + Log.d("SERVOMEDIA", "PREV"); + } + } + }; + + registerReceiver(mMediaSessionActionReceiver, filter); + + Intent pauseIntent = new Intent(KEY_MEDIA_PAUSE); + Notification.Action pauseAction = + new Notification.Action(R.drawable.media_session_pause, "Pause", + PendingIntent.getBroadcast(context, 0, pauseIntent, 0)); + + Intent nextIntent = new Intent(KEY_MEDIA_NEXT); + Notification.Action nextAction = + new Notification.Action(R.drawable.media_session_next, "Next", + PendingIntent.getBroadcast(context, 0, nextIntent, 0)); + + Intent prevIntent = new Intent(KEY_MEDIA_PREV); + Notification.Action prevAction = + new Notification.Action(R.drawable.media_session_prev, "Previous", + PendingIntent.getBroadcast(context, 0, prevIntent, 0)); + + Intent stopIntent = new Intent(KEY_MEDIA_STOP); + Notification.Action stopAction = + new Notification.Action(R.drawable.media_session_stop, "Stop", + PendingIntent.getBroadcast(context, 0, stopIntent, 0)); + + Notification.Builder builder = new Notification.Builder(context, this.MEDIA_CHANNEL_ID); + builder + .setSmallIcon(R.drawable.media_session_icon) + .setContentTitle("This is the notification title") + .setVisibility(Notification.VISIBILITY_PUBLIC) + .addAction(prevAction) + .addAction(pauseAction) + .addAction(nextAction) + .addAction(stopAction) + .setStyle(new Notification.MediaStyle() + .setShowActionsInCompactView(1 /* pause button */ ) + .setShowActionsInCompactView(3 /* stop button */)); + + NotificationManager notificationManager = getSystemService(NotificationManager.class); + notificationManager.notify(mNotificationID.getNext(), builder.build()); + } + + private void hideMediaSessionControls() { + Log.d("SERVOMEDIA", "hideMediaSessionControls"); + NotificationManager notificationManager = getSystemService(NotificationManager.class); + notificationManager.cancel(mNotificationID.get()); + unregisterReceiver(mMediaSessionActionReceiver); + } + + } diff --git a/support/android/apk/servoapp/src/main/res/drawable/media_session_icon.png b/support/android/apk/servoapp/src/main/res/drawable/media_session_icon.png new file mode 100755 index 0000000000000000000000000000000000000000..aafd2bbad02f165f375baf7a432d6a97657ee0b8 GIT binary patch literal 213 zcmeAS@N?(olHy`uVBq!ia0vp^LLkh+0wn(&ce?|m=6Jd|hEyUE|?iN0X0I5Xp8`Yfw`MlM=8A+-q# z0ft4z0l$~9EO?m0%23+rq9w!n>(vXcT!w2-9G87mN&^GS%$29-sNVA0DI)Ynr{24| zJL!(i^mUPU4*%HubP0l+XkKmW)d$ literal 0 HcmV?d00001 diff --git a/support/android/apk/servoapp/src/main/res/drawable/media_session_next.png b/support/android/apk/servoapp/src/main/res/drawable/media_session_next.png new file mode 100755 index 0000000000000000000000000000000000000000..b4208472c5d620c1053a3aafcacaa80d1af10e86 GIT binary patch literal 203 zcmeAS@N?(olHy`uVBq!ia0vp^Dj>|k0wldT1B8LpBu^K|kP60Ri8l!%Jx!uKH5DoU zn-!TJg#Kzd*rmnB_%ObWk4>zLx9LzZ&ngiqmiGG+Mw;d?xToAZr{FWw(TivLf;Hhh zav~BeOClwAN&IuHyewgH!jyyQjYOQ|&+QT=n#M0I?|k1|%Oc%$NbBEInNuLn;{GUR%h^V8FwCuhDF<*WYJmWMtF@Vi54P{W<9in5 literal 0 HcmV?d00001 diff --git a/support/android/apk/servoapp/src/main/res/drawable/media_session_prev.png b/support/android/apk/servoapp/src/main/res/drawable/media_session_prev.png new file mode 100755 index 0000000000000000000000000000000000000000..8bd9feef1951a59ae5acbb414d723704fa392242 GIT binary patch literal 205 zcmeAS@N?(olHy`uVBq!ia0vp^Dj>|k0wldT1B8Lp6i*k&kP60Ri8l!%Jx%`;I;1}s zu36CL7{=q@ut`Mf$f*m*K8S0$FLYeR^W4E`iqw>oA7nMW7CIi{xxH{vyr$ua=7*vo zZ-knKcCr;3`80p}Z&<>}JiF1RxkNJ0vGTm+8waK*QYjso7aFQ~1#jeQaObcHIJ?-# z3adQSDdJRe@AbLOcXCUKo*RdFPPQbQxdiKC28M&jX6C)o;W`O)AA_f>pUXO@geCw{ Cokg1f literal 0 HcmV?d00001 diff --git a/support/android/apk/servoapp/src/main/res/drawable/media_session_stop.png b/support/android/apk/servoapp/src/main/res/drawable/media_session_stop.png new file mode 100755 index 0000000000000000000000000000000000000000..b9321def65dea20b033523b37296922e221b78f0 GIT binary patch literal 106 zcmeAS@N?(olHy`uVBq!ia0vp^Dj>|k1|%Oc%$NbB3_V>OLn;{GUS$LdaQ!d7vX562kt!#DdXrzo2NH8Xg+`njxgN@xNA Djp-h~ literal 0 HcmV?d00001 diff --git a/support/android/apk/servoapp/src/main/res/values/strings.xml b/support/android/apk/servoapp/src/main/res/values/strings.xml index ed1fb279213..4f3d30a0a84 100644 --- a/support/android/apk/servoapp/src/main/res/values/strings.xml +++ b/support/android/apk/servoapp/src/main/res/values/strings.xml @@ -1,3 +1,5 @@ - Servo + Servo + ServoMedia + Notication channel for multimedia activity From 0bbacdda0c2d3526ba47e30f21e9b8a7846be817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 18 Oct 2019 16:46:28 +0200 Subject: [PATCH 18/37] Move media session related code out of MainActivity --- .../java/org/mozilla/servo/MainActivity.java | 128 ++-------------- .../java/org/mozilla/servo/MediaSession.java | 140 ++++++++++++++++++ 2 files changed, 151 insertions(+), 117 deletions(-) create mode 100644 support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java index ddff2e57b1a..7cc2ab99eca 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java @@ -7,16 +7,9 @@ package org.mozilla.servo; import android.app.Activity; import android.app.AlertDialog; -import android.app.PendingIntent; -import android.app.Notification; -import android.app.NotificationChannel; -import android.app.NotificationManager; -import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; -import android.content.IntentFilter; import android.net.Uri; -import android.os.Build; import android.os.Bundle; import android.system.ErrnoException; import android.system.Os; @@ -30,6 +23,7 @@ import android.widget.ProgressBar; import android.widget.TextView; import android.util.Log; +import org.mozilla.servo.MediaSession; import org.mozilla.servoview.ServoView; import org.mozilla.servoview.Servo; @@ -37,24 +31,7 @@ import java.io.File; public class MainActivity extends Activity implements Servo.Client { - private class NotificationID { - private int lastID = 0; - public int getNext() { - lastID++; - return lastID; - } - - public int get() { - return lastID; - } - } - private static final String LOGTAG = "MainActivity"; - private static final String MEDIA_CHANNEL_ID = "MediaNotificationChannel"; - private static final String KEY_MEDIA_PAUSE = "org.mozilla.servoview.MainActivity.pause"; - private static final String KEY_MEDIA_PREV = "org.mozilla.servoview.MainActivity.prev"; - private static final String KEY_MEDIA_NEXT = "org.mozilla.servoview.MainActivity.next"; - private static final String KEY_MEDIA_STOP = "org.mozilla.servoview.MainActivity.stop"; ServoView mServoView; Button mBackButton; @@ -65,8 +42,7 @@ public class MainActivity extends Activity implements Servo.Client { ProgressBar mProgressBar; TextView mIdleText; boolean mCanGoBack; - NotificationID mNotificationID; - BroadcastReceiver mMediaSessionActionReceiver; + MediaSession mMediaSession; @Override protected void onCreate(Bundle savedInstanceState) { @@ -109,16 +85,12 @@ public class MainActivity extends Activity implements Servo.Client { mServoView.loadUri(intent.getData()); } setupUrlField(); - - mNotificationID = new NotificationID(); - createMediaNotificationChannel(); } @Override protected void onDestroy() { - Log.d("SERVOMEDIA", "onDestroy"); super.onDestroy(); - hideMediaSessionControls(); + mMediaSession.hideMediaSessionControls(); } private void setupUrlField() { @@ -257,103 +229,25 @@ public class MainActivity extends Activity implements Servo.Client { @Override public void onMediaSessionMetadata(String title, String artist, String album) { + if (mMediaSession == null) { + mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); + } Log.d("SERVOMEDIA", "METADATA"); } @Override public void onMediaSessionPlaybackStateChange(int state) { + if (mMediaSession == null) { + mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); + } Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED " + state); if (state == 1 /* none */) { - hideMediaSessionControls(); + mMediaSession.hideMediaSessionControls(); return; } if (state == 2 /* playing */) { - showMediaSessionControls(); + mMediaSession.showMediaSessionControls(); return; } } - - private void createMediaNotificationChannel() { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { - CharSequence name = getString(R.string.media_channel_name); - String description = getString(R.string.media_channel_description); - int importance = NotificationManager.IMPORTANCE_DEFAULT; - NotificationChannel channel = new NotificationChannel(MEDIA_CHANNEL_ID, name, importance); - channel.setDescription(description); - NotificationManager notificationManager = getSystemService(NotificationManager.class); - notificationManager.createNotificationChannel(channel); - } - } - - private void showMediaSessionControls() { - Context context = getApplicationContext(); - - IntentFilter filter = new IntentFilter(); - filter.addAction(KEY_MEDIA_PAUSE); - filter.addAction(KEY_MEDIA_STOP); - filter.addAction(KEY_MEDIA_PREV); - filter.addAction(KEY_MEDIA_NEXT); - - mMediaSessionActionReceiver = new BroadcastReceiver() { - @Override - public void onReceive(Context context, Intent intent) { - if (intent.getAction().equals(KEY_MEDIA_PAUSE)) { - Log.d("SERVOMEDIA", "PAUSE"); - } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { - Log.d("SERVOMEDIA", "STOP"); - } else if (intent.getAction().equals(KEY_MEDIA_NEXT)) { - Log.d("SERVOMEDIA", "NEXT"); - } else if (intent.getAction().equals(KEY_MEDIA_PREV)) { - Log.d("SERVOMEDIA", "PREV"); - } - } - }; - - registerReceiver(mMediaSessionActionReceiver, filter); - - Intent pauseIntent = new Intent(KEY_MEDIA_PAUSE); - Notification.Action pauseAction = - new Notification.Action(R.drawable.media_session_pause, "Pause", - PendingIntent.getBroadcast(context, 0, pauseIntent, 0)); - - Intent nextIntent = new Intent(KEY_MEDIA_NEXT); - Notification.Action nextAction = - new Notification.Action(R.drawable.media_session_next, "Next", - PendingIntent.getBroadcast(context, 0, nextIntent, 0)); - - Intent prevIntent = new Intent(KEY_MEDIA_PREV); - Notification.Action prevAction = - new Notification.Action(R.drawable.media_session_prev, "Previous", - PendingIntent.getBroadcast(context, 0, prevIntent, 0)); - - Intent stopIntent = new Intent(KEY_MEDIA_STOP); - Notification.Action stopAction = - new Notification.Action(R.drawable.media_session_stop, "Stop", - PendingIntent.getBroadcast(context, 0, stopIntent, 0)); - - Notification.Builder builder = new Notification.Builder(context, this.MEDIA_CHANNEL_ID); - builder - .setSmallIcon(R.drawable.media_session_icon) - .setContentTitle("This is the notification title") - .setVisibility(Notification.VISIBILITY_PUBLIC) - .addAction(prevAction) - .addAction(pauseAction) - .addAction(nextAction) - .addAction(stopAction) - .setStyle(new Notification.MediaStyle() - .setShowActionsInCompactView(1 /* pause button */ ) - .setShowActionsInCompactView(3 /* stop button */)); - - NotificationManager notificationManager = getSystemService(NotificationManager.class); - notificationManager.notify(mNotificationID.getNext(), builder.build()); - } - - private void hideMediaSessionControls() { - Log.d("SERVOMEDIA", "hideMediaSessionControls"); - NotificationManager notificationManager = getSystemService(NotificationManager.class); - notificationManager.cancel(mNotificationID.get()); - unregisterReceiver(mMediaSessionActionReceiver); - } - - } diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java new file mode 100644 index 00000000000..b786bee2669 --- /dev/null +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -0,0 +1,140 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*- + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +package org.mozilla.servo; + +import android.app.Activity; +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.app.PendingIntent; +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.os.Build; +import android.util.Log; + +import org.mozilla.servoview.ServoView; + +public class MediaSession { + private class NotificationID { + private int lastID = 0; + public int getNext() { + lastID++; + return lastID; + } + + public int get() { + return lastID; + } + } + + private static final String MEDIA_CHANNEL_ID = "MediaNotificationChannel"; + private static final String KEY_MEDIA_PAUSE = "org.mozilla.servoview.MainActivity.pause"; + private static final String KEY_MEDIA_PREV = "org.mozilla.servoview.MainActivity.prev"; + private static final String KEY_MEDIA_NEXT = "org.mozilla.servoview.MainActivity.next"; + private static final String KEY_MEDIA_STOP = "org.mozilla.servoview.MainActivity.stop"; + + ServoView mView; + MainActivity mActivity; + Context mContext; + NotificationID mNotificationID; + BroadcastReceiver mMediaSessionActionReceiver; + + public MediaSession(ServoView view, MainActivity activity, Context context) { + mView = view; + mActivity = activity; + mContext = context; + mNotificationID = new NotificationID(); + createMediaNotificationChannel(); + } + + private void createMediaNotificationChannel() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + CharSequence name = + mContext.getResources().getString(R.string.media_channel_name); + String description = + mContext.getResources().getString(R.string.media_channel_description); + int importance = NotificationManager.IMPORTANCE_DEFAULT; + NotificationChannel channel = + new NotificationChannel(MEDIA_CHANNEL_ID, name, importance); + channel.setDescription(description); + NotificationManager notificationManager = + mContext.getSystemService(NotificationManager.class); + notificationManager.createNotificationChannel(channel); + } + } + + public void showMediaSessionControls() { + IntentFilter filter = new IntentFilter(); + filter.addAction(KEY_MEDIA_PAUSE); + filter.addAction(KEY_MEDIA_STOP); + filter.addAction(KEY_MEDIA_PREV); + filter.addAction(KEY_MEDIA_NEXT); + + mMediaSessionActionReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + if (intent.getAction().equals(KEY_MEDIA_PAUSE)) { + Log.d("SERVOMEDIA", "PAUSE"); + } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { + Log.d("SERVOMEDIA", "STOP"); + } else if (intent.getAction().equals(KEY_MEDIA_NEXT)) { + Log.d("SERVOMEDIA", "NEXT"); + } else if (intent.getAction().equals(KEY_MEDIA_PREV)) { + Log.d("SERVOMEDIA", "PREV"); + } + } + }; + + mContext.registerReceiver(mMediaSessionActionReceiver, filter); + + Intent pauseIntent = new Intent(KEY_MEDIA_PAUSE); + Notification.Action pauseAction = + new Notification.Action(R.drawable.media_session_pause, "Pause", + PendingIntent.getBroadcast(mContext, 0, pauseIntent, 0)); + + Intent nextIntent = new Intent(KEY_MEDIA_NEXT); + Notification.Action nextAction = + new Notification.Action(R.drawable.media_session_next, "Next", + PendingIntent.getBroadcast(mContext, 0, nextIntent, 0)); + + Intent prevIntent = new Intent(KEY_MEDIA_PREV); + Notification.Action prevAction = + new Notification.Action(R.drawable.media_session_prev, "Previous", + PendingIntent.getBroadcast(mContext, 0, prevIntent, 0)); + + Intent stopIntent = new Intent(KEY_MEDIA_STOP); + Notification.Action stopAction = + new Notification.Action(R.drawable.media_session_stop, "Stop", + PendingIntent.getBroadcast(mContext, 0, stopIntent, 0)); + + Notification.Builder builder = new Notification.Builder(mContext, this.MEDIA_CHANNEL_ID); + builder + .setSmallIcon(R.drawable.media_session_icon) + .setContentTitle("This is the notification title") + .setVisibility(Notification.VISIBILITY_PUBLIC) + .addAction(prevAction) + .addAction(pauseAction) + .addAction(nextAction) + .addAction(stopAction) + .setStyle(new Notification.MediaStyle() + .setShowActionsInCompactView(1 /* pause button */ ) + .setShowActionsInCompactView(3 /* stop button */)); + + NotificationManager notificationManager = + mContext.getSystemService(NotificationManager.class); + notificationManager.notify(mNotificationID.getNext(), builder.build()); + } + + public void hideMediaSessionControls() { + Log.d("SERVOMEDIA", "hideMediaSessionControls"); + NotificationManager notificationManager = + mContext.getSystemService(NotificationManager.class); + notificationManager.cancel(mNotificationID.get()); + mContext.unregisterReceiver(mMediaSessionActionReceiver); + } +} From b494acbf19b52feb122d7d52a84c735e6522a5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 18 Oct 2019 17:29:23 +0200 Subject: [PATCH 19/37] Adapt MediaMetadata interface to new way of declaring constructors --- components/script/dom/webidls/MediaMetadata.webidl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/script/dom/webidls/MediaMetadata.webidl b/components/script/dom/webidls/MediaMetadata.webidl index b4142e431ae..495aeef8e35 100644 --- a/components/script/dom/webidls/MediaMetadata.webidl +++ b/components/script/dom/webidls/MediaMetadata.webidl @@ -12,9 +12,9 @@ dictionary MediaImage { DOMString type = ""; }; -[Exposed=Window, - Constructor(optional MediaMetadataInit init = {})] +[Exposed=Window] interface MediaMetadata { + [Throws] constructor(optional MediaMetadataInit init = {}); attribute DOMString title; attribute DOMString artist; attribute DOMString album; From 08f9f17ed36c1102db51eaa1a88e6914757f2026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 18 Oct 2019 22:29:11 +0200 Subject: [PATCH 20/37] Send MediaSessionAction from Android --- components/script/dom/htmlmediaelement.rs | 6 ------ components/script/dom/mediasession.rs | 1 + components/script_traits/lib.rs | 17 ++++++++++++++++ ports/libsimpleservo/api/src/lib.rs | 9 +++------ ports/libsimpleservo/jniapi/src/lib.rs | 10 ++++++++++ .../java/org/mozilla/servo/MainActivity.java | 4 ++-- .../java/org/mozilla/servo/MediaSession.java | 20 +++++++++++++++++++ .../java/org/mozilla/servoview/JNIServo.java | 2 ++ .../java/org/mozilla/servoview/Servo.java | 4 ++++ .../java/org/mozilla/servoview/ServoView.java | 6 +++++- 10 files changed, 64 insertions(+), 15 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 3204871d459..14a667b8918 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -1913,12 +1913,6 @@ impl HTMLMediaElement { self.media_element_load_algorithm(); } } - - fn send_media_session_event(&self, event: MediaSessionEvent) { - let global = self.global(); - let media_session = global.as_window().Navigator().MediaSession(); - media_session.send_event(event); - } } // XXX Placeholder for [https://github.com/servo/servo/issues/22293] diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index ef64170f312..29d7108e3e3 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -61,6 +61,7 @@ impl MediaSession { } pub fn handle_action(&self, action: MediaSessionActionType) { + println!("HANDLE ACTION {:?}", action); if let Some(handler) = self.action_handlers.borrow().get(&action) { if handler.Call__(ExceptionHandling::Report).is_err() { warn!("Error calling MediaSessionActionHandler callback"); diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index b4b85889ea0..7b921db70e4 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -1087,3 +1087,20 @@ pub enum MediaSessionActionType { /// The action intent is to move the playback time to a specific time. SeekTo, } + +impl From for MediaSessionActionType { + fn from(value: i32) -> MediaSessionActionType { + match value { + 1 => MediaSessionActionType::Play, + 2 => MediaSessionActionType::Pause, + 3 => MediaSessionActionType::SeekBackward, + 4 => MediaSessionActionType::SeekForward, + 5 => MediaSessionActionType::PreviousTrack, + 6 => MediaSessionActionType::NextTrack, + 7 => MediaSessionActionType::SkipAd, + 8 => MediaSessionActionType::Stop, + 9 => MediaSessionActionType::SeekTo, + _ => panic!("Unknown MediaSessionActionType"), + } + } +} diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 3eb28992427..85f5baf56a9 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -19,7 +19,7 @@ use servo::embedder_traits::{EmbedderMsg, MediaSessionEvent}; use servo::euclid::{Point2D, Rect, Scale, Size2D, Vector2D}; use servo::keyboard_types::{Key, KeyState, KeyboardEvent}; use servo::msg::constellation_msg::TraversalDirection; -use servo::script_traits::{MediaSessionActionType, TouchEventType, TouchId}; +use servo::script_traits::{TouchEventType, TouchId}; use servo::servo_config::opts; use servo::servo_config::{pref, set_pref}; use servo::servo_url::ServoUrl; @@ -470,13 +470,10 @@ impl ServoGlue { self.process_event(WindowEvent::Keyboard(key_event)) } - pub fn media_session_action( - &mut self, - action: MediaSessionActionType, - ) -> Result<(), &'static str> { + pub fn media_session_action(&mut self, action: i32) -> Result<(), &'static str> { info!("Media session action {:?}", action); let browser_id = self.get_browser_id()?; - self.process_event(WindowEvent::MediaSessionAction(browser_id, action)) + self.process_event(WindowEvent::MediaSessionAction(browser_id, action.into())) } fn process_event(&mut self, event: WindowEvent) -> Result<(), &'static str> { diff --git a/ports/libsimpleservo/jniapi/src/lib.rs b/ports/libsimpleservo/jniapi/src/lib.rs index 44b6207d3e6..1125f3a5680 100644 --- a/ports/libsimpleservo/jniapi/src/lib.rs +++ b/ports/libsimpleservo/jniapi/src/lib.rs @@ -333,6 +333,16 @@ pub fn Java_org_mozilla_servoview_JNIServo_click(env: JNIEnv, _: JClass, x: jint call(&env, |s| s.click(x as f32, y as f32)); } +#[no_mangle] +pub fn Java_org_mozilla_servoview_JNIServo_mediaSessionAction( + env: JNIEnv, + _: JClass, + action: jint, +) { + debug!("mediaSessionAction"); + call(&env, |s| s.media_session_action(action as i32)); +} + pub struct WakeupCallback { callback: GlobalRef, jvm: Arc, diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java index 7cc2ab99eca..f95cfe4a64e 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java @@ -241,11 +241,11 @@ public class MainActivity extends Activity implements Servo.Client { mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); } Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED " + state); - if (state == 1 /* none */) { + if (state == MediaSession.PLAYBACK_STATE_NONE) { mMediaSession.hideMediaSessionControls(); return; } - if (state == 2 /* playing */) { + if (state == MediaSession.PLAYBACK_STATE_PLAYING) { mMediaSession.showMediaSessionControls(); return; } diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java index b786bee2669..ce8d1e6315a 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -32,6 +32,22 @@ public class MediaSession { } } + // https://w3c.github.io/mediasession/#enumdef-mediasessionplaybackstate + public static final int PLAYBACK_STATE_NONE = 1; + public static final int PLAYBACK_STATE_PLAYING = 2; + public static final int PLAYBACK_STATE_PAUSED = 3; + + // https://w3c.github.io/mediasession/#enumdef-mediasessionaction + private static final int ACTION_PLAY = 1; + private static final int ACTION_PAUSE = 2; + private static final int ACTON_SEEK_BACKWARD = 3; + private static final int ACTION_SEEK_FORWARD = 4; + private static final int ACTION_PREVIOUS_TRACK = 5; + private static final int ACTION_NEXT_TRACK = 6; + private static final int ACTION_SKIP_AD = 7; + private static final int ACTION_STOP = 8; + private static final int ACTION_SEEK_TO = 9; + private static final String MEDIA_CHANNEL_ID = "MediaNotificationChannel"; private static final String KEY_MEDIA_PAUSE = "org.mozilla.servoview.MainActivity.pause"; private static final String KEY_MEDIA_PREV = "org.mozilla.servoview.MainActivity.prev"; @@ -79,12 +95,16 @@ public class MediaSession { @Override public void onReceive(Context context, Intent intent) { if (intent.getAction().equals(KEY_MEDIA_PAUSE)) { + mView.mediaSessionAction(ACTION_PAUSE); Log.d("SERVOMEDIA", "PAUSE"); } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { + mView.mediaSessionAction(ACTION_STOP); Log.d("SERVOMEDIA", "STOP"); } else if (intent.getAction().equals(KEY_MEDIA_NEXT)) { + mView.mediaSessionAction(ACTION_NEXT_TRACK); Log.d("SERVOMEDIA", "NEXT"); } else if (intent.getAction().equals(KEY_MEDIA_PREV)) { + mView.mediaSessionAction(ACTION_PREVIOUS_TRACK); Log.d("SERVOMEDIA", "PREV"); } } diff --git a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java index b1ce302e0aa..6181a298f0a 100644 --- a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java +++ b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/JNIServo.java @@ -66,6 +66,8 @@ public class JNIServo { public native void click(float x, float y); + public native void mediaSessionAction(int action); + public static class ServoOptions { public String args; public String url; diff --git a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java index 7daea963bb5..f45f55e79f5 100644 --- a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java +++ b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/Servo.java @@ -168,6 +168,10 @@ public class Servo { mSuspended = suspended; } + public void mediaSessionAction(int action) { + mRunCallback.inGLThread(() -> mJNI.mediaSessionAction(action)); + } + public interface Client { void onAlert(String message); diff --git a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/ServoView.java b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/ServoView.java index 33431afa19a..3784bbf58cb 100644 --- a/support/android/apk/servoview/src/main/java/org/mozilla/servoview/ServoView.java +++ b/support/android/apk/servoview/src/main/java/org/mozilla/servoview/ServoView.java @@ -134,8 +134,12 @@ public class ServoView extends GLSurfaceView } } + public void mediaSessionAction(int action) { + mServo.mediaSessionAction(action); + } + public void flushGLBuffers() { - requestRender(); + requestRender(); } // Scroll and click From 85ec66b43edea5c91ba9a77390867cb9b2db6baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 28 Oct 2019 16:08:05 +0100 Subject: [PATCH 21/37] Move active media session logic to constellation --- components/constellation/constellation.rs | 32 ++++++++++++++++++++--- components/script/dom/mediasession.rs | 17 ++++++------ components/script/dom/navigator.rs | 4 +-- components/script/script_thread.rs | 10 +++---- components/script_traits/lib.rs | 2 +- components/script_traits/script_msg.rs | 3 ++- components/servo/lib.rs | 29 -------------------- 7 files changed, 46 insertions(+), 51 deletions(-) diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index db9d94e0c88..2f1d0a06ac7 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -112,6 +112,7 @@ use compositing::SendableFrameTree; use crossbeam_channel::{after, never, unbounded, Receiver, Sender}; use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg}; use embedder_traits::{Cursor, EmbedderMsg, EmbedderProxy, EventLoopWaker}; +use embedder_traits::{MediaSessionEvent, MediaSessionPlaybackState}; use euclid::{default::Size2D as UntypedSize2D, Size2D}; use gfx::font_cache_thread::FontCacheThread; use gfx_traits::Epoch; @@ -475,8 +476,8 @@ pub struct Constellation { /// Mechanism to force the compositor to process events. event_loop_waker: Option>, - /// Browser ID of the active media session, if any. - active_media_session: Option, + /// Browing context ID of the active media session. + active_media_session: Option, } /// State needed to construct a constellation. @@ -1778,6 +1779,30 @@ where new_value, ); }, + FromScriptMsg::MediaSessionEvent(browsing_context_id, event) => { + // Unlikely at this point, but we may receive events coming from + // different media sessions, so we set the active media session based + // on Playing events. + // The last media session claiming to be in playing state is set to + // the active media session. + // Events coming from inactive media sessions are discarded. + if self.active_media_session.is_some() { + match event { + MediaSessionEvent::PlaybackStateChange(ref state) => { + match state { + MediaSessionPlaybackState::Playing => (), + _ => return, + }; + }, + _ => (), + }; + } + self.active_media_session = Some(browsing_context_id); + self.embedder_proxy.send(( + Some(source_top_ctx_id), + EmbedderMsg::MediaSessionEvent(event), + )); + }, } } @@ -5042,8 +5067,7 @@ where ); }, }; - let msg = - ConstellationControlMsg::MediaSessionAction(top_level_browsing_context_id, action); + let msg = ConstellationControlMsg::MediaSessionAction(browsing_context_id, action); let result = match self.pipelines.get(&pipeline_id) { None => { return warn!( diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 29d7108e3e3..2ad4586e6b2 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -16,9 +16,10 @@ use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; use crate::script_thread::ScriptThread; use dom_struct::dom_struct; -use embedder_traits::{EmbedderMsg, MediaSessionEvent}; -use msg::constellation_msg::TopLevelBrowsingContextId; +use embedder_traits::MediaSessionEvent; +use msg::constellation_msg::BrowsingContextId; use script_traits::MediaSessionActionType; +use script_traits::ScriptMsg; use std::collections::HashMap; use std::rc::Rc; @@ -39,7 +40,7 @@ pub struct MediaSession { impl MediaSession { #[allow(unrooted_must_root)] - fn new_inherited(browsing_context_id: TopLevelBrowsingContextId) -> MediaSession { + fn new_inherited(browsing_context_id: BrowsingContextId) -> MediaSession { let media_session = MediaSession { reflector_: Reflector::new(), metadata: Default::default(), @@ -52,7 +53,7 @@ impl MediaSession { } pub fn new(window: &Window) -> DomRoot { - let browsing_context_id = window.window_proxy().top_level_browsing_context_id(); + let browsing_context_id = window.window_proxy().browsing_context_id(); reflect_dom_object( Box::new(MediaSession::new_inherited(browsing_context_id)), window, @@ -74,7 +75,8 @@ impl MediaSession { pub fn send_event(&self, event: MediaSessionEvent) { let global = self.global(); let window = global.as_window(); - window.send_to_embedder(EmbedderMsg::MediaSessionEvent(event)); + let browsing_context_id = window.window_proxy().browsing_context_id(); + window.send_to_constellation(ScriptMsg::MediaSessionEvent(browsing_context_id, event)); } } @@ -121,10 +123,7 @@ impl MediaSessionMethods for MediaSession { impl Drop for MediaSession { fn drop(&mut self) { let global = self.global(); - let browsing_context_id = global - .as_window() - .window_proxy() - .top_level_browsing_context_id(); + let browsing_context_id = global.as_window().window_proxy().browsing_context_id(); ScriptThread::remove_media_session(browsing_context_id); } } diff --git a/components/script/dom/navigator.rs b/components/script/dom/navigator.rs index a759d3dbe54..d4cde572a47 100644 --- a/components/script/dom/navigator.rs +++ b/components/script/dom/navigator.rs @@ -194,7 +194,7 @@ impl NavigatorMethods for Navigator { /// https://w3c.github.io/mediasession/#dom-navigator-mediasession fn MediaSession(&self) -> DomRoot { self.mediasession.or_init(|| { - // There is a single MediaSession instance per top level browsing context + // There is a single MediaSession instance per browsing context // and only one active MediaSession globally. // // MediaSession creation can happen in two cases: @@ -206,7 +206,7 @@ impl NavigatorMethods for Navigator { // the script thread. let global = self.global(); let window = global.as_window(); - let browsing_context_id = window.window_proxy().top_level_browsing_context_id(); + let browsing_context_id = window.window_proxy().browsing_context_id(); match ScriptThread::get_media_session(browsing_context_id) { Some(session) => session, None => MediaSession::new(window), diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index e7b39d26420..886977e1945 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -701,7 +701,7 @@ pub struct ScriptThread { /// The MediaSessions known by this thread, if any. /// There can only be one active MediaSession. /// The constellation has the BrowsingContextId of the active MediaSession, if any. - media_sessions: DomRefCell>>, + media_sessions: DomRefCell>>, } /// In the event of thread panic, all data on the stack runs its destructor. However, there @@ -3938,7 +3938,7 @@ impl ScriptThread { fn handle_media_session_action( &self, - browsing_context_id: TopLevelBrowsingContextId, + browsing_context_id: BrowsingContextId, action: MediaSessionActionType, ) { match self.media_sessions.borrow().get(&browsing_context_id) { @@ -3973,7 +3973,7 @@ impl ScriptThread { pub fn register_media_session( media_session: &MediaSession, - browsing_context_id: TopLevelBrowsingContextId, + browsing_context_id: BrowsingContextId, ) { SCRIPT_THREAD_ROOT.with(|root| { let script_thread = unsafe { &*root.get().unwrap() }; @@ -3984,7 +3984,7 @@ impl ScriptThread { }) } - pub fn remove_media_session(browsing_context_id: TopLevelBrowsingContextId) { + pub fn remove_media_session(browsing_context_id: BrowsingContextId) { SCRIPT_THREAD_ROOT.with(|root| { let script_thread = unsafe { &*root.get().unwrap() }; script_thread @@ -3995,7 +3995,7 @@ impl ScriptThread { } pub fn get_media_session( - browsing_context_id: TopLevelBrowsingContextId, + browsing_context_id: BrowsingContextId, ) -> Option> { SCRIPT_THREAD_ROOT.with(|root| { let script_thread = unsafe { &*root.get().unwrap() }; diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index 7b921db70e4..a732141a9ff 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -389,7 +389,7 @@ pub enum ConstellationControlMsg { /// Notifies the script thread about a new recorded paint metric. PaintMetric(PipelineId, ProgressiveWebMetricType, u64), /// Notifies the media session about a user requested media session action. - MediaSessionAction(TopLevelBrowsingContextId, MediaSessionActionType), + MediaSessionAction(BrowsingContextId, MediaSessionActionType), } impl fmt::Debug for ConstellationControlMsg { diff --git a/components/script_traits/script_msg.rs b/components/script_traits/script_msg.rs index 64d6b1f5389..9db4e7ac7e2 100644 --- a/components/script_traits/script_msg.rs +++ b/components/script_traits/script_msg.rs @@ -17,7 +17,7 @@ use crate::WorkerGlobalScopeInit; use crate::WorkerScriptLoadOrigin; use canvas_traits::canvas::{CanvasId, CanvasMsg}; use devtools_traits::{ScriptToDevtoolsControlMsg, WorkerId}; -use embedder_traits::EmbedderMsg; +use embedder_traits::{EmbedderMsg, MediaSessionEvent}; use euclid::default::Size2D as UntypedSize2D; use euclid::Size2D; use gfx_traits::Epoch; @@ -306,6 +306,7 @@ impl fmt::Debug for ScriptMsg { GetClientWindow(..) => "GetClientWindow", GetScreenSize(..) => "GetScreenSize", GetScreenAvailSize(..) => "GetScreenAvailSize", + MediaSessionEvent(..) => "MediaSessionEvent", }; write!(formatter, "ScriptMsg::{}", variant) } diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 3da82d299fd..fa188bc5b3d 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -83,7 +83,6 @@ use constellation::{Constellation, InitialConstellationState, UnprivilegedPipeli use constellation::{FromCompositorLogger, FromScriptLogger}; use crossbeam_channel::{unbounded, Sender}; use embedder_traits::{EmbedderMsg, EmbedderProxy, EmbedderReceiver, EventLoopWaker}; -use embedder_traits::{MediaSessionEvent, MediaSessionPlaybackState}; use env_logger::Builder as EnvLoggerBuilder; use euclid::{Scale, Size2D}; #[cfg(all( @@ -272,8 +271,6 @@ pub struct Servo { embedder_events: Vec<(Option, EmbedderMsg)>, profiler_enabled: bool, webgl_thread_data: Option>, - /// Browser ID of the active media session, if any. - active_media_session: Option, } #[derive(Clone)] @@ -560,7 +557,6 @@ where embedder_events: Vec::new(), profiler_enabled: false, webgl_thread_data, - active_media_session: None, } } @@ -750,31 +746,6 @@ where self.embedder_events.push(event); }, - (EmbedderMsg::MediaSessionEvent(event), ShutdownState::NotShuttingDown) => { - // Unlikely at this point, but we may receive events coming from - // different media sessions, so we set the active media session based - // on Playing events. - // The last media session claiming to be in playing state is set to - // the active media session. - // Events coming from inactive media sessions are discarded. - if self.active_media_session.is_some() { - match event { - MediaSessionEvent::PlaybackStateChange(ref state) => { - match state { - MediaSessionPlaybackState::Playing => (), - _ => return, - }; - }, - _ => (), - }; - } - self.active_media_session = top_level_browsing_context; - self.embedder_events.push(( - top_level_browsing_context, - EmbedderMsg::MediaSessionEvent(event), - )); - }, - (msg, ShutdownState::NotShuttingDown) => { self.embedder_events.push((top_level_browsing_context, msg)); }, From 68baabba63d84385b94a31c013dd1e55452e4310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 6 Nov 2019 18:37:04 +0100 Subject: [PATCH 22/37] Format code and fix rebase errors --- components/constellation/constellation.rs | 4 ++-- components/script/dom/bindings/trace.rs | 2 +- components/script_traits/lib.rs | 1 + components/script_traits/script_msg.rs | 4 +++- components/servo/lib.rs | 2 -- ports/libsimpleservo/capi/src/lib.rs | 3 ++- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 2f1d0a06ac7..1802952e31e 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -153,8 +153,8 @@ use script_traits::{ IFrameLoadInfo, IFrameLoadInfoWithData, IFrameSandboxState, TimerSchedulerMsg, }; use script_traits::{LayoutMsg as FromLayoutMsg, ScriptMsg as FromScriptMsg, ScriptThreadFactory}; -use script_traits::{MessagePortMsg, PortMessageTask, StructuredSerializedData}; use script_traits::{MediaSessionActionType, MouseEventType}; +use script_traits::{MessagePortMsg, PortMessageTask, StructuredSerializedData}; use script_traits::{SWManagerMsg, ScopeThings, UpdatePipelineIdReason, WebDriverCommandMsg}; use serde::{Deserialize, Serialize}; use servo_config::{opts, pref}; @@ -475,7 +475,7 @@ pub struct Constellation { /// Mechanism to force the compositor to process events. event_loop_waker: Option>, - + /// Browing context ID of the active media session. active_media_session: Option, } diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 1d426c2a0e6..87f41eec5ef 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -94,7 +94,7 @@ use profile_traits::time::ProfilerChan as TimeProfilerChan; use script_layout_interface::rpc::LayoutRPC; use script_layout_interface::OpaqueStyleAndLayoutData; use script_traits::transferable::MessagePortImpl; -use script_traits::{DocumentActivity, DrawAPaintImageResult; +use script_traits::{DocumentActivity, DrawAPaintImageResult}; use script_traits::{MediaSessionActionType, ScriptToConstellationChan, TimerEventId, TimerSource}; use script_traits::{UntrustedNodeAddress, WindowSizeData, WindowSizeType}; use selectors::matching::ElementSelectorFlags; diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index a732141a9ff..7216a369693 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -1058,6 +1058,7 @@ pub enum MessagePortMsg { RemoveMessagePort(MessagePortId), /// Handle a new port-message-task. NewTask(MessagePortId, PortMessageTask), +} /// The type of MediaSession action. /// https://w3c.github.io/mediasession/#enumdef-mediasessionaction diff --git a/components/script_traits/script_msg.rs b/components/script_traits/script_msg.rs index 9db4e7ac7e2..4cce2501c88 100644 --- a/components/script_traits/script_msg.rs +++ b/components/script_traits/script_msg.rs @@ -8,7 +8,6 @@ use crate::DocumentState; use crate::IFrameLoadInfoWithData; use crate::LayoutControlMsg; use crate::LoadData; -use crate::MediaSessionEvent; use crate::MessagePortMsg; use crate::PortMessageTask; use crate::StructuredSerializedData; @@ -255,6 +254,9 @@ pub enum ScriptMsg { GetScreenSize(IpcSender), /// Get the available screen size (pixel) GetScreenAvailSize(IpcSender), + /// Notifies the constellation about media session events + /// (i.e. when there is metadata for the active media session, playback state changes...). + MediaSessionEvent(BrowsingContextId, MediaSessionEvent) } impl fmt::Debug for ScriptMsg { diff --git a/components/servo/lib.rs b/components/servo/lib.rs index fa188bc5b3d..67751234111 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -270,7 +270,6 @@ pub struct Servo { embedder_receiver: EmbedderReceiver, embedder_events: Vec<(Option, EmbedderMsg)>, profiler_enabled: bool, - webgl_thread_data: Option>, } #[derive(Clone)] @@ -556,7 +555,6 @@ where embedder_receiver: embedder_receiver, embedder_events: Vec::new(), profiler_enabled: false, - webgl_thread_data, } } diff --git a/ports/libsimpleservo/capi/src/lib.rs b/ports/libsimpleservo/capi/src/lib.rs index 31a066d9416..2d67f088b9a 100644 --- a/ports/libsimpleservo/capi/src/lib.rs +++ b/ports/libsimpleservo/capi/src/lib.rs @@ -17,7 +17,8 @@ use env_logger; use log::LevelFilter; use simpleservo::{self, gl_glue, ServoGlue, SERVO}; use simpleservo::{ - Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionEvent, MouseButton, VRInitOptions, + Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionEvent, MouseButton, + VRInitOptions, }; use std::ffi::{CStr, CString}; #[cfg(target_os = "windows")] From 9da1dd359290bf59a0990a8664b58b5cfbb1f091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 12 Nov 2019 17:12:54 +0100 Subject: [PATCH 23/37] Default media session actions --- components/script/dom/htmlmediaelement.rs | 9 +++++++ components/script/dom/mediasession.rs | 29 +++++++++++++++++++++-- components/script_traits/script_msg.rs | 2 +- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 14a667b8918..da8ffd76aed 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -1913,6 +1913,15 @@ impl HTMLMediaElement { self.media_element_load_algorithm(); } } + + fn send_media_session_event(&self, event: MediaSessionEvent) { + let global = self.global(); + let media_session = global.as_window().Navigator().MediaSession(); + + media_session.register_media_instance(&self); + + media_session.send_event(event); + } } // XXX Placeholder for [https://github.com/servo/servo/issues/22293] diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 2ad4586e6b2..e29f47597f9 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -2,8 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +use crate::compartments::{AlreadyInCompartment, InCompartment}; use crate::dom::bindings::callback::ExceptionHandling; use crate::dom::bindings::cell::DomRefCell; +use crate::dom::bindings::codegen::Bindings::HTMLMediaElementBinding::HTMLMediaElementMethods; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionAction; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionActionHandler; @@ -61,15 +63,38 @@ impl MediaSession { ) } + pub fn register_media_instance(&self, media_instance: &HTMLMediaElement) { + self.media_instance.set(Some(media_instance)); + } + pub fn handle_action(&self, action: MediaSessionActionType) { - println!("HANDLE ACTION {:?}", action); + debug!("Handle media session action {:?} {:?}", action); if let Some(handler) = self.action_handlers.borrow().get(&action) { if handler.Call__(ExceptionHandling::Report).is_err() { warn!("Error calling MediaSessionActionHandler callback"); } return; } - // TODO default action. + + // Default action. + if let Some(media) = self.media_instance.get() { + match action { + MediaSessionActionType::Play => { + let in_compartment_proof = AlreadyInCompartment::assert(&self.global()); + media.Play(InCompartment::Already(&in_compartment_proof)); + }, + MediaSessionActionType::Pause => { + media.Pause(); + }, + MediaSessionActionType::SeekBackward => {}, + MediaSessionActionType::SeekForward => {}, + MediaSessionActionType::PreviousTrack => {}, + MediaSessionActionType::NextTrack => {}, + MediaSessionActionType::SkipAd => {}, + MediaSessionActionType::Stop => {}, + MediaSessionActionType::SeekTo => {}, + } + } } pub fn send_event(&self, event: MediaSessionEvent) { diff --git a/components/script_traits/script_msg.rs b/components/script_traits/script_msg.rs index 4cce2501c88..7216b9281c9 100644 --- a/components/script_traits/script_msg.rs +++ b/components/script_traits/script_msg.rs @@ -256,7 +256,7 @@ pub enum ScriptMsg { GetScreenAvailSize(IpcSender), /// Notifies the constellation about media session events /// (i.e. when there is metadata for the active media session, playback state changes...). - MediaSessionEvent(BrowsingContextId, MediaSessionEvent) + MediaSessionEvent(BrowsingContextId, MediaSessionEvent), } impl fmt::Debug for ScriptMsg { From b048d7faf717bb5ac90f02301b567b25d37681f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 12 Nov 2019 22:09:39 +0100 Subject: [PATCH 24/37] Fix media session action handling --- components/compositing/windowing.rs | 2 +- components/constellation/constellation.rs | 61 +++++++++++------------ components/script/dom/mediasession.rs | 26 ++++------ components/script/dom/navigator.rs | 12 +---- components/script/script_thread.rs | 61 +++-------------------- components/script_traits/lib.rs | 4 +- components/script_traits/script_msg.rs | 2 +- components/servo/lib.rs | 4 +- ports/libsimpleservo/api/src/lib.rs | 3 +- 9 files changed, 54 insertions(+), 121 deletions(-) diff --git a/components/compositing/windowing.rs b/components/compositing/windowing.rs index d2fab70c234..cdc1b7ff883 100644 --- a/components/compositing/windowing.rs +++ b/components/compositing/windowing.rs @@ -104,7 +104,7 @@ pub enum WindowEvent { ToggleSamplingProfiler(Duration, Duration), /// Sent when the user triggers a media action through the UA exposed media UI /// (play, pause, seek, etc.). - MediaSessionAction(TopLevelBrowsingContextId, MediaSessionActionType), + MediaSessionAction(MediaSessionActionType), } impl Debug for WindowEvent { diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 1802952e31e..8f6cdce0808 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -476,8 +476,8 @@ pub struct Constellation { /// Mechanism to force the compositor to process events. event_loop_waker: Option>, - /// Browing context ID of the active media session. - active_media_session: Option, + /// Pipeline ID of the active media session. + active_media_session: Option, } /// State needed to construct a constellation. @@ -1546,8 +1546,8 @@ where FromCompositorMsg::ExitFullScreen(top_level_browsing_context_id) => { self.handle_exit_fullscreen_msg(top_level_browsing_context_id); }, - FromCompositorMsg::MediaSessionAction(top_level_browsing_context_id, action) => { - self.handle_media_session_action_msg(top_level_browsing_context_id, action); + FromCompositorMsg::MediaSessionAction(action) => { + self.handle_media_session_action_msg(action); }, } } @@ -1779,7 +1779,7 @@ where new_value, ); }, - FromScriptMsg::MediaSessionEvent(browsing_context_id, event) => { + FromScriptMsg::MediaSessionEvent(pipeline_id, event) => { // Unlikely at this point, but we may receive events coming from // different media sessions, so we set the active media session based // on Playing events. @@ -1797,7 +1797,7 @@ where _ => (), }; } - self.active_media_session = Some(browsing_context_id); + self.active_media_session = Some(pipeline_id); self.embedder_proxy.send(( Some(source_top_ctx_id), EmbedderMsg::MediaSessionEvent(event), @@ -5052,33 +5052,28 @@ where } } - fn handle_media_session_action_msg( - &mut self, - top_level_browsing_context_id: TopLevelBrowsingContextId, - action: MediaSessionActionType, - ) { - let browsing_context_id = BrowsingContextId::from(top_level_browsing_context_id); - let pipeline_id = match self.browsing_contexts.get(&browsing_context_id) { - Some(browsing_context) => browsing_context.pipeline_id, - None => { - return warn!( - "Browsing context {} got media session action request after closure.", - browsing_context_id - ); - }, - }; - let msg = ConstellationControlMsg::MediaSessionAction(browsing_context_id, action); - let result = match self.pipelines.get(&pipeline_id) { - None => { - return warn!( - "Pipeline {} got media session action request after closure.", - pipeline_id - ) - }, - Some(pipeline) => pipeline.event_loop.send(msg), - }; - if let Err(e) = result { - self.handle_send_error(pipeline_id, e); + fn handle_media_session_action_msg(&mut self, action: MediaSessionActionType) { + if let Some(media_session_pipeline_id) = self.active_media_session { + let result = match self.pipelines.get(&media_session_pipeline_id) { + None => { + return warn!( + "Pipeline {} got media session action request after closure.", + media_session_pipeline_id, + ) + }, + Some(pipeline) => { + let msg = ConstellationControlMsg::MediaSessionAction( + media_session_pipeline_id, + action, + ); + pipeline.event_loop.send(msg) + }, + }; + if let Err(e) = result { + self.handle_send_error(media_session_pipeline_id, e); + } + } else { + error!("Got a media session action but no active media session is registered"); } } } diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index e29f47597f9..f0d0bcbca43 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -16,10 +16,9 @@ use crate::dom::bindings::root::{DomRoot, MutNullableDom}; use crate::dom::htmlmediaelement::HTMLMediaElement; use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; -use crate::script_thread::ScriptThread; use dom_struct::dom_struct; use embedder_traits::MediaSessionEvent; -use msg::constellation_msg::BrowsingContextId; +use msg::constellation_msg::PipelineId; use script_traits::MediaSessionActionType; use script_traits::ScriptMsg; use std::collections::HashMap; @@ -42,7 +41,7 @@ pub struct MediaSession { impl MediaSession { #[allow(unrooted_must_root)] - fn new_inherited(browsing_context_id: BrowsingContextId) -> MediaSession { + fn new_inherited() -> MediaSession { let media_session = MediaSession { reflector_: Reflector::new(), metadata: Default::default(), @@ -50,14 +49,12 @@ impl MediaSession { action_handlers: DomRefCell::new(HashMap::new()), media_instance: Default::default(), }; - ScriptThread::register_media_session(&media_session, browsing_context_id); media_session } pub fn new(window: &Window) -> DomRoot { - let browsing_context_id = window.window_proxy().browsing_context_id(); reflect_dom_object( - Box::new(MediaSession::new_inherited(browsing_context_id)), + Box::new(MediaSession::new_inherited()), window, MediaSessionBinding::Wrap, ) @@ -68,7 +65,8 @@ impl MediaSession { } pub fn handle_action(&self, action: MediaSessionActionType) { - debug!("Handle media session action {:?} {:?}", action); + debug!("Handle media session action {:?}", action); + if let Some(handler) = self.action_handlers.borrow().get(&action) { if handler.Call__(ExceptionHandling::Report).is_err() { warn!("Error calling MediaSessionActionHandler callback"); @@ -100,8 +98,10 @@ impl MediaSession { pub fn send_event(&self, event: MediaSessionEvent) { let global = self.global(); let window = global.as_window(); - let browsing_context_id = window.window_proxy().browsing_context_id(); - window.send_to_constellation(ScriptMsg::MediaSessionEvent(browsing_context_id, event)); + let pipeline_id = window + .pipeline_id() + .expect("Cannot send media session event outside of a pipeline"); + window.send_to_constellation(ScriptMsg::MediaSessionEvent(pipeline_id, event)); } } @@ -145,14 +145,6 @@ impl MediaSessionMethods for MediaSession { } } -impl Drop for MediaSession { - fn drop(&mut self) { - let global = self.global(); - let browsing_context_id = global.as_window().window_proxy().browsing_context_id(); - ScriptThread::remove_media_session(browsing_context_id); - } -} - impl From for MediaSessionActionType { fn from(action: MediaSessionAction) -> MediaSessionActionType { match action { diff --git a/components/script/dom/navigator.rs b/components/script/dom/navigator.rs index d4cde572a47..127883dd956 100644 --- a/components/script/dom/navigator.rs +++ b/components/script/dom/navigator.rs @@ -21,7 +21,6 @@ use crate::dom::promise::Promise; use crate::dom::serviceworkercontainer::ServiceWorkerContainer; use crate::dom::window::Window; use crate::dom::xr::XR; -use crate::script_thread::ScriptThread; use dom_struct::dom_struct; use std::rc::Rc; @@ -194,23 +193,16 @@ impl NavigatorMethods for Navigator { /// https://w3c.github.io/mediasession/#dom-navigator-mediasession fn MediaSession(&self) -> DomRoot { self.mediasession.or_init(|| { - // There is a single MediaSession instance per browsing context + // There is a single MediaSession instance per Pipeline // and only one active MediaSession globally. // // MediaSession creation can happen in two cases: // // - If content gets `navigator.mediaSession` // - If a media instance (HTMLMediaElement so far) starts playing media. - // - // The MediaSession constructor is in charge of registering itself with - // the script thread. let global = self.global(); let window = global.as_window(); - let browsing_context_id = window.window_proxy().browsing_context_id(); - match ScriptThread::get_media_session(browsing_context_id) { - Some(session) => session, - None => MediaSession::new(window), - } + MediaSession::new(window) }) } } diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 886977e1945..c0155ea63a5 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -51,7 +51,6 @@ use crate::dom::event::{Event, EventBubbles, EventCancelable}; use crate::dom::globalscope::GlobalScope; use crate::dom::htmlanchorelement::HTMLAnchorElement; use crate::dom::htmliframeelement::{HTMLIFrameElement, NavigationType}; -use crate::dom::mediasession::MediaSession; use crate::dom::mutationobserver::MutationObserver; use crate::dom::node::{ from_untrusted_node_address, window_from_node, Node, NodeDamage, ShadowIncluding, @@ -697,11 +696,6 @@ pub struct ScriptThread { /// Code is running as a consequence of a user interaction is_user_interacting: Cell, - - /// The MediaSessions known by this thread, if any. - /// There can only be one active MediaSession. - /// The constellation has the BrowsingContextId of the active MediaSession, if any. - media_sessions: DomRefCell>>, } /// In the event of thread panic, all data on the stack runs its destructor. However, there @@ -1371,7 +1365,6 @@ impl ScriptThread { node_ids: Default::default(), is_user_interacting: Cell::new(false), - media_sessions: DomRefCell::new(HashMap::new()), } } @@ -1950,8 +1943,8 @@ impl ScriptThread { ConstellationControlMsg::PaintMetric(pipeline_id, metric_type, metric_value) => { self.handle_paint_metric(pipeline_id, metric_type, metric_value) }, - ConstellationControlMsg::MediaSessionAction(browsing_context_id, action) => { - self.handle_media_session_action(browsing_context_id, action) + ConstellationControlMsg::MediaSessionAction(pipeline_id, action) => { + self.handle_media_session_action(pipeline_id, action) }, msg @ ConstellationControlMsg::AttachLayout(..) | msg @ ConstellationControlMsg::Viewport(..) | @@ -3936,14 +3929,12 @@ impl ScriptThread { } } - fn handle_media_session_action( - &self, - browsing_context_id: BrowsingContextId, - action: MediaSessionActionType, - ) { - match self.media_sessions.borrow().get(&browsing_context_id) { - Some(session) => session.handle_action(action), - None => warn!("No MediaSession for this browsing context"), + fn handle_media_session_action(&self, pipeline_id: PipelineId, action: MediaSessionActionType) { + if let Some(window) = self.documents.borrow().find_window(pipeline_id) { + let media_session = window.Navigator().MediaSession(); + media_session.handle_action(action); + } else { + warn!("No MediaSession for this pipeline ID"); }; } @@ -3970,42 +3961,6 @@ impl ScriptThread { globals, ) } - - pub fn register_media_session( - media_session: &MediaSession, - browsing_context_id: BrowsingContextId, - ) { - SCRIPT_THREAD_ROOT.with(|root| { - let script_thread = unsafe { &*root.get().unwrap() }; - script_thread - .media_sessions - .borrow_mut() - .insert(browsing_context_id, Dom::from_ref(media_session)); - }) - } - - pub fn remove_media_session(browsing_context_id: BrowsingContextId) { - SCRIPT_THREAD_ROOT.with(|root| { - let script_thread = unsafe { &*root.get().unwrap() }; - script_thread - .media_sessions - .borrow_mut() - .remove(&browsing_context_id); - }) - } - - pub fn get_media_session( - browsing_context_id: BrowsingContextId, - ) -> Option> { - SCRIPT_THREAD_ROOT.with(|root| { - let script_thread = unsafe { &*root.get().unwrap() }; - script_thread - .media_sessions - .borrow() - .get(&browsing_context_id) - .map(|s| DomRoot::from_ref(&**s)) - }) - } } impl Drop for ScriptThread { diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index 7216a369693..a8a19b5a40a 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -389,7 +389,7 @@ pub enum ConstellationControlMsg { /// Notifies the script thread about a new recorded paint metric. PaintMetric(PipelineId, ProgressiveWebMetricType, u64), /// Notifies the media session about a user requested media session action. - MediaSessionAction(BrowsingContextId, MediaSessionActionType), + MediaSessionAction(PipelineId, MediaSessionActionType), } impl fmt::Debug for ConstellationControlMsg { @@ -881,7 +881,7 @@ pub enum ConstellationMsg { /// Request to exit from fullscreen mode ExitFullScreen(TopLevelBrowsingContextId), /// Media session action. - MediaSessionAction(TopLevelBrowsingContextId, MediaSessionActionType), + MediaSessionAction(MediaSessionActionType), } impl fmt::Debug for ConstellationMsg { diff --git a/components/script_traits/script_msg.rs b/components/script_traits/script_msg.rs index 7216b9281c9..1eccc794398 100644 --- a/components/script_traits/script_msg.rs +++ b/components/script_traits/script_msg.rs @@ -256,7 +256,7 @@ pub enum ScriptMsg { GetScreenAvailSize(IpcSender), /// Notifies the constellation about media session events /// (i.e. when there is metadata for the active media session, playback state changes...). - MediaSessionEvent(BrowsingContextId, MediaSessionEvent), + MediaSessionEvent(PipelineId, MediaSessionEvent), } impl fmt::Debug for ScriptMsg { diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 67751234111..e32c28ef740 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -714,8 +714,8 @@ where } }, - WindowEvent::MediaSessionAction(ctx, a) => { - let msg = ConstellationMsg::MediaSessionAction(ctx, a); + WindowEvent::MediaSessionAction(a) => { + let msg = ConstellationMsg::MediaSessionAction(a); if let Err(e) = self.constellation_chan.send(msg) { warn!( "Sending MediaSessionAction message to constellation failed ({:?}).", diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 85f5baf56a9..554cf7ee67c 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -472,8 +472,7 @@ impl ServoGlue { pub fn media_session_action(&mut self, action: i32) -> Result<(), &'static str> { info!("Media session action {:?}", action); - let browser_id = self.get_browser_id()?; - self.process_event(WindowEvent::MediaSessionAction(browser_id, action.into())) + self.process_event(WindowEvent::MediaSessionAction(action.into())) } fn process_event(&mut self, event: WindowEvent) -> Result<(), &'static str> { From b5b8c6c2a8b41c9829af52f22e685dfae249c5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 13 Nov 2019 11:05:35 +0100 Subject: [PATCH 25/37] Remove prev and next track action buttons for now --- components/script/dom/mediasession.rs | 1 - .../java/org/mozilla/servo/MediaSession.java | 24 ++----------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index f0d0bcbca43..868565f540c 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -18,7 +18,6 @@ use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; use dom_struct::dom_struct; use embedder_traits::MediaSessionEvent; -use msg::constellation_msg::PipelineId; use script_traits::MediaSessionActionType; use script_traits::ScriptMsg; use std::collections::HashMap; diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java index ce8d1e6315a..b1f8c3794de 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -88,8 +88,6 @@ public class MediaSession { IntentFilter filter = new IntentFilter(); filter.addAction(KEY_MEDIA_PAUSE); filter.addAction(KEY_MEDIA_STOP); - filter.addAction(KEY_MEDIA_PREV); - filter.addAction(KEY_MEDIA_NEXT); mMediaSessionActionReceiver = new BroadcastReceiver() { @Override @@ -100,12 +98,6 @@ public class MediaSession { } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { mView.mediaSessionAction(ACTION_STOP); Log.d("SERVOMEDIA", "STOP"); - } else if (intent.getAction().equals(KEY_MEDIA_NEXT)) { - mView.mediaSessionAction(ACTION_NEXT_TRACK); - Log.d("SERVOMEDIA", "NEXT"); - } else if (intent.getAction().equals(KEY_MEDIA_PREV)) { - mView.mediaSessionAction(ACTION_PREVIOUS_TRACK); - Log.d("SERVOMEDIA", "PREV"); } } }; @@ -117,16 +109,6 @@ public class MediaSession { new Notification.Action(R.drawable.media_session_pause, "Pause", PendingIntent.getBroadcast(mContext, 0, pauseIntent, 0)); - Intent nextIntent = new Intent(KEY_MEDIA_NEXT); - Notification.Action nextAction = - new Notification.Action(R.drawable.media_session_next, "Next", - PendingIntent.getBroadcast(mContext, 0, nextIntent, 0)); - - Intent prevIntent = new Intent(KEY_MEDIA_PREV); - Notification.Action prevAction = - new Notification.Action(R.drawable.media_session_prev, "Previous", - PendingIntent.getBroadcast(mContext, 0, prevIntent, 0)); - Intent stopIntent = new Intent(KEY_MEDIA_STOP); Notification.Action stopAction = new Notification.Action(R.drawable.media_session_stop, "Stop", @@ -137,13 +119,11 @@ public class MediaSession { .setSmallIcon(R.drawable.media_session_icon) .setContentTitle("This is the notification title") .setVisibility(Notification.VISIBILITY_PUBLIC) - .addAction(prevAction) .addAction(pauseAction) - .addAction(nextAction) .addAction(stopAction) .setStyle(new Notification.MediaStyle() - .setShowActionsInCompactView(1 /* pause button */ ) - .setShowActionsInCompactView(3 /* stop button */)); + .setShowActionsInCompactView(0 /* pause button */ ) + .setShowActionsInCompactView(1 /* stop button */)); NotificationManager notificationManager = mContext.getSystemService(NotificationManager.class); From 07483f1d44cc3ea474e0a47e53253955da3aa4d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 13 Nov 2019 12:09:33 +0100 Subject: [PATCH 26/37] Switch play and pause buttons according to playback state --- components/constellation/constellation.rs | 3 +- .../java/org/mozilla/servo/MainActivity.java | 7 ++-- .../java/org/mozilla/servo/MediaSession.java | 39 +++++++++++++----- .../main/res/drawable/media_session_play.png | Bin 0 -> 227 bytes 4 files changed, 34 insertions(+), 15 deletions(-) create mode 100755 support/android/apk/servoapp/src/main/res/drawable/media_session_play.png diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 8f6cdce0808..82b99938f3b 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -1790,7 +1790,8 @@ where match event { MediaSessionEvent::PlaybackStateChange(ref state) => { match state { - MediaSessionPlaybackState::Playing => (), + MediaSessionPlaybackState::Playing | + MediaSessionPlaybackState::Paused => (), _ => return, }; }, diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java index f95cfe4a64e..60abbc43747 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java @@ -237,16 +237,17 @@ public class MainActivity extends Activity implements Servo.Client { @Override public void onMediaSessionPlaybackStateChange(int state) { + Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED " + state); if (mMediaSession == null) { mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); } - Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED " + state); if (state == MediaSession.PLAYBACK_STATE_NONE) { mMediaSession.hideMediaSessionControls(); return; } - if (state == MediaSession.PLAYBACK_STATE_PLAYING) { - mMediaSession.showMediaSessionControls(); + if (state == MediaSession.PLAYBACK_STATE_PLAYING || + state == MediaSession.PLAYBACK_STATE_PAUSED) { + mMediaSession.showMediaSessionControls(state); return; } } diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java index b1f8c3794de..2b03206931a 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -49,6 +49,7 @@ public class MediaSession { private static final int ACTION_SEEK_TO = 9; private static final String MEDIA_CHANNEL_ID = "MediaNotificationChannel"; + private static final String KEY_MEDIA_PLAY = "org.mozilla.servoview.MainActivity.play"; private static final String KEY_MEDIA_PAUSE = "org.mozilla.servoview.MainActivity.pause"; private static final String KEY_MEDIA_PREV = "org.mozilla.servoview.MainActivity.prev"; private static final String KEY_MEDIA_NEXT = "org.mozilla.servoview.MainActivity.next"; @@ -84,9 +85,14 @@ public class MediaSession { } } - public void showMediaSessionControls() { + public void showMediaSessionControls(int playbackState) { IntentFilter filter = new IntentFilter(); - filter.addAction(KEY_MEDIA_PAUSE); + if (playbackState == PLAYBACK_STATE_PAUSED) { + filter.addAction(KEY_MEDIA_PLAY); + } + if (playbackState == PLAYBACK_STATE_PLAYING) { + filter.addAction(KEY_MEDIA_PAUSE); + } filter.addAction(KEY_MEDIA_STOP); mMediaSessionActionReceiver = new BroadcastReceiver() { @@ -95,6 +101,9 @@ public class MediaSession { if (intent.getAction().equals(KEY_MEDIA_PAUSE)) { mView.mediaSessionAction(ACTION_PAUSE); Log.d("SERVOMEDIA", "PAUSE"); + } else if (intent.getAction().equals(KEY_MEDIA_PLAY)) { + mView.mediaSessionAction(ACTION_PLAY); + Log.d("SERVOMEDIA", "PLAY"); } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { mView.mediaSessionAction(ACTION_STOP); Log.d("SERVOMEDIA", "STOP"); @@ -104,11 +113,6 @@ public class MediaSession { mContext.registerReceiver(mMediaSessionActionReceiver, filter); - Intent pauseIntent = new Intent(KEY_MEDIA_PAUSE); - Notification.Action pauseAction = - new Notification.Action(R.drawable.media_session_pause, "Pause", - PendingIntent.getBroadcast(mContext, 0, pauseIntent, 0)); - Intent stopIntent = new Intent(KEY_MEDIA_STOP); Notification.Action stopAction = new Notification.Action(R.drawable.media_session_stop, "Stop", @@ -119,11 +123,24 @@ public class MediaSession { .setSmallIcon(R.drawable.media_session_icon) .setContentTitle("This is the notification title") .setVisibility(Notification.VISIBILITY_PUBLIC) - .addAction(pauseAction) .addAction(stopAction) - .setStyle(new Notification.MediaStyle() - .setShowActionsInCompactView(0 /* pause button */ ) - .setShowActionsInCompactView(1 /* stop button */)); + .setStyle(new Notification.MediaStyle()); + + if (playbackState == PLAYBACK_STATE_PAUSED) { + Intent playIntent = new Intent(KEY_MEDIA_PLAY); + Notification.Action playAction = + new Notification.Action(R.drawable.media_session_play, "Play", + PendingIntent.getBroadcast(mContext, 0, playIntent, 0)); + builder.addAction(playAction); + } + + if (playbackState == PLAYBACK_STATE_PLAYING) { + Intent pauseIntent = new Intent(KEY_MEDIA_PAUSE); + Notification.Action pauseAction = + new Notification.Action(R.drawable.media_session_pause, "Pause", + PendingIntent.getBroadcast(mContext, 0, pauseIntent, 0)); + builder.addAction(pauseAction); + } NotificationManager notificationManager = mContext.getSystemService(NotificationManager.class); diff --git a/support/android/apk/servoapp/src/main/res/drawable/media_session_play.png b/support/android/apk/servoapp/src/main/res/drawable/media_session_play.png new file mode 100755 index 0000000000000000000000000000000000000000..65b4620171869f74289cb16db2f2a0bb50f0969a GIT binary patch literal 227 zcmeAS@N?(olHy`uVBq!ia0vp^Dj>|k0wldT1B8LpDo+>3kP61+18i;wH4GL_bF7!B zlCW5rc%fdQ=D-6F*5(BThRq7CYzH4^w?%mIIvuFx`7}jRQ@>6H96I2#&Cy|2 zf1=i5dyV%t&5S$Q92-~$cQ$8Mw`4oWpzX|i*k57(7Z=8>Ej%HTe2VH^pLm0u z3_V(6)`uGYS#T#?ZNq`KCk{o6>$%*%1kY1oH|sv|P`b_Hm58!Z^WubN7n54<|A7Wv Z3|WU-9j~o7i~+iu!PC{xWt~$(697iwPf-8> literal 0 HcmV?d00001 From 7b5b46f56071667a421a679a98812d755b63a4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 13 Nov 2019 16:50:02 +0100 Subject: [PATCH 27/37] Fix play action --- .../java/org/mozilla/servo/MainActivity.java | 4 +- .../java/org/mozilla/servo/MediaSession.java | 41 +++++++++++-------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java index 60abbc43747..366ec017a3d 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java @@ -232,12 +232,12 @@ public class MainActivity extends Activity implements Servo.Client { if (mMediaSession == null) { mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); } - Log.d("SERVOMEDIA", "METADATA"); + Log.d("onMediaSessionMetadata", title + " " + artist + " " + album); } @Override public void onMediaSessionPlaybackStateChange(int state) { - Log.d("SERVOMEDIA", "PLAYBACK STATE CHANGED " + state); + Log.d("onMediaSessionPlaybackStateChange", String.valueOf(state)); if (mMediaSession == null) { mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); } diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java index 2b03206931a..eacf529f0eb 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -86,6 +86,7 @@ public class MediaSession { } public void showMediaSessionControls(int playbackState) { + Log.d("MediaSession", "showMediaSessionControls " + playbackState); IntentFilter filter = new IntentFilter(); if (playbackState == PLAYBACK_STATE_PAUSED) { filter.addAction(KEY_MEDIA_PLAY); @@ -95,21 +96,28 @@ public class MediaSession { } filter.addAction(KEY_MEDIA_STOP); - mMediaSessionActionReceiver = new BroadcastReceiver() { - @Override - public void onReceive(Context context, Intent intent) { - if (intent.getAction().equals(KEY_MEDIA_PAUSE)) { - mView.mediaSessionAction(ACTION_PAUSE); - Log.d("SERVOMEDIA", "PAUSE"); - } else if (intent.getAction().equals(KEY_MEDIA_PLAY)) { - mView.mediaSessionAction(ACTION_PLAY); - Log.d("SERVOMEDIA", "PLAY"); - } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { - mView.mediaSessionAction(ACTION_STOP); - Log.d("SERVOMEDIA", "STOP"); + int id; + if (mMediaSessionActionReceiver == null) { + id = mNotificationID.getNext(); + + mMediaSessionActionReceiver = new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + if (intent.getAction().equals(KEY_MEDIA_PAUSE)) { + mView.mediaSessionAction(ACTION_PAUSE); + Log.d("MediaSession", "PAUSE action"); + } else if (intent.getAction().equals(KEY_MEDIA_PLAY)) { + mView.mediaSessionAction(ACTION_PLAY); + Log.d("MediaSession", "PLAY action"); + } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { + mView.mediaSessionAction(ACTION_STOP); + Log.d("MediaSession", "STOP action"); + } } - } - }; + }; + } else { + id = mNotificationID.get(); + } mContext.registerReceiver(mMediaSessionActionReceiver, filter); @@ -144,14 +152,15 @@ public class MediaSession { NotificationManager notificationManager = mContext.getSystemService(NotificationManager.class); - notificationManager.notify(mNotificationID.getNext(), builder.build()); + notificationManager.notify(id, builder.build()); } public void hideMediaSessionControls() { - Log.d("SERVOMEDIA", "hideMediaSessionControls"); + Log.d("MediaSession", "hideMediaSessionControls"); NotificationManager notificationManager = mContext.getSystemService(NotificationManager.class); notificationManager.cancel(mNotificationID.get()); mContext.unregisterReceiver(mMediaSessionActionReceiver); + mMediaSessionActionReceiver = null; } } From caedc28118d6c6b4e8affa77818dac80b0a244a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Wed, 13 Nov 2019 16:59:24 +0100 Subject: [PATCH 28/37] Do not play notification sound when creating media session on Android --- .../servoapp/src/main/java/org/mozilla/servo/MediaSession.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java index eacf529f0eb..5a3316fc68f 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -75,7 +75,7 @@ public class MediaSession { mContext.getResources().getString(R.string.media_channel_name); String description = mContext.getResources().getString(R.string.media_channel_description); - int importance = NotificationManager.IMPORTANCE_DEFAULT; + int importance = NotificationManager.IMPORTANCE_LOW; NotificationChannel channel = new NotificationChannel(MEDIA_CHANNEL_ID, name, importance); channel.setDescription(description); From 6ee21af0c428ceff04f885f040ee98827207349d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 14 Nov 2019 16:42:36 +0100 Subject: [PATCH 29/37] Update media session metadata according to media player --- components/script/dom/bindings/trace.rs | 3 ++- components/script/dom/mediasession.rs | 35 ++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 87f41eec5ef..2051a1a19d8 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -57,7 +57,7 @@ use content_security_policy::CspList; use crossbeam_channel::{Receiver, Sender}; use cssparser::RGBA; use devtools_traits::{CSSError, TimelineMarkerType, WorkerId}; -use embedder_traits::EventLoopWaker; +use embedder_traits::{EventLoopWaker, MediaMetadata}; use encoding_rs::{Decoder, Encoding}; use euclid::default::{Point2D, Rect, Rotation3D, Transform2D, Transform3D}; use euclid::Length as EuclidLength; @@ -537,6 +537,7 @@ unsafe_no_jsmanaged_fields!(VideoFrame); unsafe_no_jsmanaged_fields!(WebGLContextId); unsafe_no_jsmanaged_fields!(Arc>); unsafe_no_jsmanaged_fields!(MediaSessionActionType); +unsafe_no_jsmanaged_fields!(MediaMetadata); unsafe impl<'a> JSTraceable for &'a str { #[inline] diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 868565f540c..250f420df60 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -6,6 +6,8 @@ use crate::compartments::{AlreadyInCompartment, InCompartment}; use crate::dom::bindings::callback::ExceptionHandling; use crate::dom::bindings::cell::DomRefCell; use crate::dom::bindings::codegen::Bindings::HTMLMediaElementBinding::HTMLMediaElementMethods; +use crate::dom::bindings::codegen::Bindings::MediaMetadataBinding::MediaMetadataInit; +use crate::dom::bindings::codegen::Bindings::MediaMetadataBinding::MediaMetadataMethods; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionAction; use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionActionHandler; @@ -13,10 +15,12 @@ use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionMe use crate::dom::bindings::codegen::Bindings::MediaSessionBinding::MediaSessionPlaybackState; use crate::dom::bindings::reflector::{reflect_dom_object, DomObject, Reflector}; use crate::dom::bindings::root::{DomRoot, MutNullableDom}; +use crate::dom::bindings::str::DOMString; use crate::dom::htmlmediaelement::HTMLMediaElement; use crate::dom::mediametadata::MediaMetadata; use crate::dom::window::Window; use dom_struct::dom_struct; +use embedder_traits::MediaMetadata as EmbedderMediaMetadata; use embedder_traits::MediaSessionEvent; use script_traits::MediaSessionActionType; use script_traits::ScriptMsg; @@ -27,7 +31,8 @@ use std::rc::Rc; pub struct MediaSession { reflector_: Reflector, /// https://w3c.github.io/mediasession/#dom-mediasession-metadata - metadata: MutNullableDom, + #[ignore_malloc_size_of = "defined in embedder_traits"] + metadata: DomRefCell>, /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate playback_state: DomRefCell, /// https://w3c.github.io/mediasession/#supported-media-session-actions @@ -43,7 +48,7 @@ impl MediaSession { fn new_inherited() -> MediaSession { let media_session = MediaSession { reflector_: Reflector::new(), - metadata: Default::default(), + metadata: DomRefCell::new(None), playback_state: DomRefCell::new(MediaSessionPlaybackState::None), action_handlers: DomRefCell::new(HashMap::new()), media_instance: Default::default(), @@ -95,6 +100,13 @@ impl MediaSession { } pub fn send_event(&self, event: MediaSessionEvent) { + match event { + MediaSessionEvent::SetMetadata(ref metadata) => { + *self.metadata.borrow_mut() = Some(metadata.clone()); + }, + _ => (), + } + let global = self.global(); let window = global.as_window(); let pipeline_id = window @@ -107,7 +119,16 @@ impl MediaSession { impl MediaSessionMethods for MediaSession { /// https://w3c.github.io/mediasession/#dom-mediasession-metadata fn GetMetadata(&self) -> Option> { - self.metadata.get() + if let Some(ref metadata) = *self.metadata.borrow() { + let mut init = MediaMetadataInit::empty(); + init.title = DOMString::from_string(metadata.title.clone()); + init.artist = DOMString::from_string(metadata.artist.clone().unwrap_or("".to_owned())); + init.album = DOMString::from_string(metadata.album.clone().unwrap_or("".to_owned())); + let global = self.global(); + Some(MediaMetadata::new(&global.as_window(), &init)) + } else { + None + } } /// https://w3c.github.io/mediasession/#dom-mediasession-metadata @@ -115,7 +136,13 @@ impl MediaSessionMethods for MediaSession { if let Some(ref metadata) = metadata { metadata.set_session(self); } - self.metadata.set(metadata); + + let _metadata = metadata.map(|m| EmbedderMediaMetadata { + title: m.Title().into(), + artist: Some(m.Artist().into()), + album: Some(m.Album().into()), + }); + *self.metadata.borrow_mut() = _metadata; } /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate From d33c96b31e9370dd3254901a3164c78026daf4c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 15 Nov 2019 10:02:38 +0100 Subject: [PATCH 30/37] Fallback to current url as metadata title --- components/script/dom/htmlmediaelement.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index da8ffd76aed..333412ed3cd 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -1728,10 +1728,15 @@ impl HTMLMediaElement { self.render_controls(); } + let global = self.global(); + let window = global.as_window(); + // Send a media session event with the obtained metadata. self.send_media_session_event(MediaSessionEvent::SetMetadata(MediaMetadata { - // TODO(ferjm) set url if no title. - title: metadata.title.clone().unwrap_or("".to_string()), + title: metadata + .title + .clone() + .unwrap_or(window.get_url().into_string()), artist: None, album: None, })); From f65c4008287d6c71b6d3c94183139b16b170b5ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 15 Nov 2019 11:37:49 +0100 Subject: [PATCH 31/37] Update media session metadata and show content text with artist and album --- .../java/org/mozilla/servo/MainActivity.java | 6 ++- .../java/org/mozilla/servo/MediaSession.java | 51 ++++++++++++++++--- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java index 366ec017a3d..21a6e7f996f 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MainActivity.java @@ -233,6 +233,7 @@ public class MainActivity extends Activity implements Servo.Client { mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); } Log.d("onMediaSessionMetadata", title + " " + artist + " " + album); + mMediaSession.updateMetadata(title, artist, album); } @Override @@ -241,13 +242,16 @@ public class MainActivity extends Activity implements Servo.Client { if (mMediaSession == null) { mMediaSession = new MediaSession(mServoView, this, getApplicationContext()); } + + mMediaSession.setPlaybackState(state); + if (state == MediaSession.PLAYBACK_STATE_NONE) { mMediaSession.hideMediaSessionControls(); return; } if (state == MediaSession.PLAYBACK_STATE_PLAYING || state == MediaSession.PLAYBACK_STATE_PAUSED) { - mMediaSession.showMediaSessionControls(state); + mMediaSession.showMediaSessionControls(); return; } } diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java index 5a3316fc68f..6444dff4d99 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -58,9 +58,16 @@ public class MediaSession { ServoView mView; MainActivity mActivity; Context mContext; + NotificationID mNotificationID; BroadcastReceiver mMediaSessionActionReceiver; + int mPlaybackState = PLAYBACK_STATE_PAUSED; + + String mTitle; + String mArtist; + String mAlbum; + public MediaSession(ServoView view, MainActivity activity, Context context) { mView = view; mActivity = activity; @@ -85,13 +92,13 @@ public class MediaSession { } } - public void showMediaSessionControls(int playbackState) { - Log.d("MediaSession", "showMediaSessionControls " + playbackState); + public void showMediaSessionControls() { + Log.d("MediaSession", "showMediaSessionControls " + mPlaybackState); IntentFilter filter = new IntentFilter(); - if (playbackState == PLAYBACK_STATE_PAUSED) { + if (mPlaybackState == PLAYBACK_STATE_PAUSED) { filter.addAction(KEY_MEDIA_PLAY); } - if (playbackState == PLAYBACK_STATE_PLAYING) { + if (mPlaybackState == PLAYBACK_STATE_PLAYING) { filter.addAction(KEY_MEDIA_PAUSE); } filter.addAction(KEY_MEDIA_STOP); @@ -129,12 +136,28 @@ public class MediaSession { Notification.Builder builder = new Notification.Builder(mContext, this.MEDIA_CHANNEL_ID); builder .setSmallIcon(R.drawable.media_session_icon) - .setContentTitle("This is the notification title") + .setContentTitle(mTitle) .setVisibility(Notification.VISIBILITY_PUBLIC) .addAction(stopAction) .setStyle(new Notification.MediaStyle()); - if (playbackState == PLAYBACK_STATE_PAUSED) { + String contentText = new String(); + if (mArtist != null && !mArtist.isEmpty()) { + contentText = mArtist; + } + if (mAlbum != null && !mAlbum.isEmpty()) { + if (!contentText.isEmpty()) { + contentText += " - " + mAlbum; + } else { + contentText = mAlbum; + } + } + + if (!contentText.isEmpty()) { + builder.setContentText(contentText); + } + + if (mPlaybackState == PLAYBACK_STATE_PAUSED) { Intent playIntent = new Intent(KEY_MEDIA_PLAY); Notification.Action playAction = new Notification.Action(R.drawable.media_session_play, "Play", @@ -142,7 +165,7 @@ public class MediaSession { builder.addAction(playAction); } - if (playbackState == PLAYBACK_STATE_PLAYING) { + if (mPlaybackState == PLAYBACK_STATE_PLAYING) { Intent pauseIntent = new Intent(KEY_MEDIA_PAUSE); Notification.Action pauseAction = new Notification.Action(R.drawable.media_session_pause, "Pause", @@ -163,4 +186,18 @@ public class MediaSession { mContext.unregisterReceiver(mMediaSessionActionReceiver); mMediaSessionActionReceiver = null; } + + public void setPlaybackState(int state) { + mPlaybackState = state; + } + + public void updateMetadata(String title, String artist, String album) { + mTitle = title; + mArtist = artist; + mAlbum = album; + + if (mMediaSessionActionReceiver != null) { + showMediaSessionControls(); + } + } } From 35c4c354164b7526fb77fc315846a0338f66cd81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 18 Nov 2019 11:36:21 +0100 Subject: [PATCH 32/37] Fix metadata update --- components/embedder_traits/lib.rs | 14 +++++- components/script/dom/htmlmediaelement.rs | 12 +++-- components/script/dom/mediametadata.rs | 6 +-- components/script/dom/mediasession.rs | 55 ++++++++++++++++------- ports/libsimpleservo/api/src/lib.rs | 4 +- 5 files changed, 62 insertions(+), 29 deletions(-) diff --git a/components/embedder_traits/lib.rs b/components/embedder_traits/lib.rs index f65ed514f4d..9d4e56dd297 100644 --- a/components/embedder_traits/lib.rs +++ b/components/embedder_traits/lib.rs @@ -213,9 +213,19 @@ pub struct MediaMetadata { /// Title pub title: String, /// Artist - pub artist: Option, + pub artist: String, /// Album - pub album: Option, + pub album: String, +} + +impl MediaMetadata { + pub fn new(title: String) -> Self { + Self { + title, + artist: "".to_owned(), + album: "".to_owned(), + } + } } /// https://w3c.github.io/mediasession/#enumdef-mediasessionplaybackstate diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 333412ed3cd..f8f707a0dd9 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -67,7 +67,7 @@ use crate::script_thread::ScriptThread; use crate::task_source::TaskSource; use dom_struct::dom_struct; use embedder_traits::resources::{self, Resource as EmbedderResource}; -use embedder_traits::{MediaMetadata, MediaSessionEvent, MediaSessionPlaybackState}; +use embedder_traits::{MediaSessionEvent, MediaSessionPlaybackState}; use euclid::default::Size2D; use headers::{ContentLength, ContentRange, HeaderMapExt}; use html5ever::{LocalName, Prefix}; @@ -1731,15 +1731,13 @@ impl HTMLMediaElement { let global = self.global(); let window = global.as_window(); - // Send a media session event with the obtained metadata. - self.send_media_session_event(MediaSessionEvent::SetMetadata(MediaMetadata { - title: metadata + // Update the media session metadata title with the obtained metadata. + window.Navigator().MediaSession().update_title( + metadata .title .clone() .unwrap_or(window.get_url().into_string()), - artist: None, - album: None, - })); + ); }, PlayerEvent::NeedData => { // The player needs more data. diff --git a/components/script/dom/mediametadata.rs b/components/script/dom/mediametadata.rs index db84df878c5..f2e94abfaa1 100644 --- a/components/script/dom/mediametadata.rs +++ b/components/script/dom/mediametadata.rs @@ -68,7 +68,7 @@ impl MediaMetadataMethods for MediaMetadata { } /// https://w3c.github.io/mediasession/#dom-mediametadata-title - fn SetTitle(&self, value: DOMString) -> () { + fn SetTitle(&self, value: DOMString) { *self.title.borrow_mut() = value; self.queue_update_metadata_algorithm(); } @@ -79,7 +79,7 @@ impl MediaMetadataMethods for MediaMetadata { } /// https://w3c.github.io/mediasession/#dom-mediametadata-artist - fn SetArtist(&self, value: DOMString) -> () { + fn SetArtist(&self, value: DOMString) { *self.artist.borrow_mut() = value; self.queue_update_metadata_algorithm(); } @@ -90,7 +90,7 @@ impl MediaMetadataMethods for MediaMetadata { } /// https://w3c.github.io/mediasession/#dom-mediametadata-album - fn SetAlbum(&self, value: DOMString) -> () { + fn SetAlbum(&self, value: DOMString) { *self.album.borrow_mut() = value; self.queue_update_metadata_algorithm(); } diff --git a/components/script/dom/mediasession.rs b/components/script/dom/mediasession.rs index 250f420df60..1523e9a0ae6 100644 --- a/components/script/dom/mediasession.rs +++ b/components/script/dom/mediasession.rs @@ -100,13 +100,6 @@ impl MediaSession { } pub fn send_event(&self, event: MediaSessionEvent) { - match event { - MediaSessionEvent::SetMetadata(ref metadata) => { - *self.metadata.borrow_mut() = Some(metadata.clone()); - }, - _ => (), - } - let global = self.global(); let window = global.as_window(); let pipeline_id = window @@ -114,6 +107,23 @@ impl MediaSession { .expect("Cannot send media session event outside of a pipeline"); window.send_to_constellation(ScriptMsg::MediaSessionEvent(pipeline_id, event)); } + + pub fn update_title(&self, title: String) { + let mut metadata = self.metadata.borrow_mut(); + if let Some(ref mut metadata) = *metadata { + // We only update the title with the data provided by the media + // player and iff the user did not provide a title. + if !metadata.title.is_empty() { + return; + } + metadata.title = title; + } else { + *metadata = Some(EmbedderMediaMetadata::new(title)); + } + self.send_event(MediaSessionEvent::SetMetadata( + metadata.as_ref().unwrap().clone(), + )); + } } impl MediaSessionMethods for MediaSession { @@ -122,8 +132,8 @@ impl MediaSessionMethods for MediaSession { if let Some(ref metadata) = *self.metadata.borrow() { let mut init = MediaMetadataInit::empty(); init.title = DOMString::from_string(metadata.title.clone()); - init.artist = DOMString::from_string(metadata.artist.clone().unwrap_or("".to_owned())); - init.album = DOMString::from_string(metadata.album.clone().unwrap_or("".to_owned())); + init.artist = DOMString::from_string(metadata.artist.clone()); + init.album = DOMString::from_string(metadata.album.clone()); let global = self.global(); Some(MediaMetadata::new(&global.as_window(), &init)) } else { @@ -137,12 +147,27 @@ impl MediaSessionMethods for MediaSession { metadata.set_session(self); } - let _metadata = metadata.map(|m| EmbedderMediaMetadata { - title: m.Title().into(), - artist: Some(m.Artist().into()), - album: Some(m.Album().into()), - }); - *self.metadata.borrow_mut() = _metadata; + let global = self.global(); + let window = global.as_window(); + let _metadata = match metadata { + Some(m) => { + let title = if m.Title().is_empty() { + window.get_url().into_string() + } else { + m.Title().into() + }; + EmbedderMediaMetadata { + title, + artist: m.Artist().into(), + album: m.Album().into(), + } + }, + None => EmbedderMediaMetadata::new(window.get_url().into_string()), + }; + + *self.metadata.borrow_mut() = Some(_metadata.clone()); + + self.send_event(MediaSessionEvent::SetMetadata(_metadata)); } /// https://w3c.github.io/mediasession/#dom-mediasession-playbackstate diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 554cf7ee67c..c7986e05623 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -586,8 +586,8 @@ impl ServoGlue { MediaSessionEvent::SetMetadata(metadata) => { self.callbacks.host_callbacks.on_media_session_metadata( metadata.title, - metadata.artist.unwrap_or("".to_owned()), - metadata.album.unwrap_or("".to_owned()), + metadata.artist, + metadata.album, ) }, MediaSessionEvent::PlaybackStateChange(state) => self From 59f22abb3e8ce75968030152175221bf3a24c907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 18 Nov 2019 11:54:49 +0100 Subject: [PATCH 33/37] Remove stop button for now and show play/pause in compact mode --- .../java/org/mozilla/servo/MediaSession.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java index 6444dff4d99..5b1852d5d3f 100644 --- a/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java +++ b/support/android/apk/servoapp/src/main/java/org/mozilla/servo/MediaSession.java @@ -101,7 +101,6 @@ public class MediaSession { if (mPlaybackState == PLAYBACK_STATE_PLAYING) { filter.addAction(KEY_MEDIA_PAUSE); } - filter.addAction(KEY_MEDIA_STOP); int id; if (mMediaSessionActionReceiver == null) { @@ -116,9 +115,6 @@ public class MediaSession { } else if (intent.getAction().equals(KEY_MEDIA_PLAY)) { mView.mediaSessionAction(ACTION_PLAY); Log.d("MediaSession", "PLAY action"); - } else if (intent.getAction().equals(KEY_MEDIA_STOP)) { - mView.mediaSessionAction(ACTION_STOP); - Log.d("MediaSession", "STOP action"); } } }; @@ -128,18 +124,11 @@ public class MediaSession { mContext.registerReceiver(mMediaSessionActionReceiver, filter); - Intent stopIntent = new Intent(KEY_MEDIA_STOP); - Notification.Action stopAction = - new Notification.Action(R.drawable.media_session_stop, "Stop", - PendingIntent.getBroadcast(mContext, 0, stopIntent, 0)); - Notification.Builder builder = new Notification.Builder(mContext, this.MEDIA_CHANNEL_ID); builder .setSmallIcon(R.drawable.media_session_icon) .setContentTitle(mTitle) - .setVisibility(Notification.VISIBILITY_PUBLIC) - .addAction(stopAction) - .setStyle(new Notification.MediaStyle()); + .setVisibility(Notification.VISIBILITY_PUBLIC); String contentText = new String(); if (mArtist != null && !mArtist.isEmpty()) { @@ -173,6 +162,9 @@ public class MediaSession { builder.addAction(pauseAction); } + builder.setStyle(new Notification.MediaStyle() + .setShowActionsInCompactView(0)); + NotificationManager notificationManager = mContext.getSystemService(NotificationManager.class); notificationManager.notify(id, builder.build()); From ba48e542e4721a92c5375335bb83dc7f38d47ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 18 Nov 2019 12:28:08 +0100 Subject: [PATCH 34/37] Enable MediaSession WPT --- tests/wpt/include.ini | 2 + .../mediasession/idlharness.window.js.ini | 16 +++++++ .../mediasession/mediametadata.html.ini | 43 +++++++++++++++++++ .../mediasession/positionstate.html.ini | 19 ++++++++ .../wpt/mozilla/tests/mozilla/interfaces.html | 2 + 5 files changed, 82 insertions(+) create mode 100644 tests/wpt/metadata/mediasession/idlharness.window.js.ini create mode 100644 tests/wpt/metadata/mediasession/mediametadata.html.ini create mode 100644 tests/wpt/metadata/mediasession/positionstate.html.ini diff --git a/tests/wpt/include.ini b/tests/wpt/include.ini index 6fded483450..2f662095640 100644 --- a/tests/wpt/include.ini +++ b/tests/wpt/include.ini @@ -119,6 +119,8 @@ skip: true skip: true [js] skip: false +[mediasession] + skip: false [navigation-timing] skip: false [offscreen-canvas] diff --git a/tests/wpt/metadata/mediasession/idlharness.window.js.ini b/tests/wpt/metadata/mediasession/idlharness.window.js.ini new file mode 100644 index 00000000000..5e65e535b96 --- /dev/null +++ b/tests/wpt/metadata/mediasession/idlharness.window.js.ini @@ -0,0 +1,16 @@ +[idlharness.window.html] + [MediaSession interface: calling setPositionState(MediaPositionState) on navigator.mediaSession with too few arguments must throw TypeError] + expected: FAIL + + [MediaSession interface: navigator.mediaSession must inherit property "setPositionState(MediaPositionState)" with the proper type] + expected: FAIL + + [MediaSession interface: operation setPositionState(MediaPositionState)] + expected: FAIL + + [MediaMetadata interface: attribute artwork] + expected: FAIL + + [MediaMetadata interface: new MediaMetadata() must inherit property "artwork" with the proper type] + expected: FAIL + diff --git a/tests/wpt/metadata/mediasession/mediametadata.html.ini b/tests/wpt/metadata/mediasession/mediametadata.html.ini new file mode 100644 index 00000000000..2a2b3e73c85 --- /dev/null +++ b/tests/wpt/metadata/mediasession/mediametadata.html.ini @@ -0,0 +1,43 @@ +[mediametadata.html] + [Test that MediaMetadata.artwork is Frozen] + expected: FAIL + + [Test that MediaMetadat.artwork can't be modified] + expected: FAIL + + [Test that resetting metadata to null is reflected] + expected: FAIL + + [Test the default values for MediaMetadata with empty init dictionary] + expected: FAIL + + [Test MediaImage default values] + expected: FAIL + + [Test that mediaSession.metadata is properly set] + expected: FAIL + + [Test that changes to metadata propagate properly] + expected: FAIL + + [Test that MediaMetadata.artwork returns parsed urls] + expected: FAIL + + [Test the different values allowed in MediaMetadata init dictionary] + expected: FAIL + + [Test the default values for MediaMetadata with no init dictionary] + expected: FAIL + + [Test that MediaImage.src is required] + expected: FAIL + + [Test that MediaMetadata throws when setting an invalid url] + expected: FAIL + + [Test that MediaMetadata.artwork will not expose unknown properties] + expected: FAIL + + [Test that the base URL of MediaImage is the base URL of entry setting object] + expected: FAIL + diff --git a/tests/wpt/metadata/mediasession/positionstate.html.ini b/tests/wpt/metadata/mediasession/positionstate.html.ini new file mode 100644 index 00000000000..ca73e31fb84 --- /dev/null +++ b/tests/wpt/metadata/mediasession/positionstate.html.ini @@ -0,0 +1,19 @@ +[positionstate.html] + [Test setPositionState with a null value] + expected: FAIL + + [Test setPositionState with zero duration] + expected: FAIL + + [Test setPositionState with a valid value for forward playback] + expected: FAIL + + [Test setPositionState with optional position] + expected: FAIL + + [Test setPositionState with only duration] + expected: FAIL + + [Test setPositionState with optional playback rate] + expected: FAIL + diff --git a/tests/wpt/mozilla/tests/mozilla/interfaces.html b/tests/wpt/mozilla/tests/mozilla/interfaces.html index 4006cae2d79..f1d58732ada 100644 --- a/tests/wpt/mozilla/tests/mozilla/interfaces.html +++ b/tests/wpt/mozilla/tests/mozilla/interfaces.html @@ -166,8 +166,10 @@ test_interfaces([ "MediaElementAudioSourceNode", "MediaError", "MediaList", + "MediaMetadata", "MediaQueryList", "MediaQueryListEvent", + "MediaSession", "MessageChannel", "MessageEvent", "MessagePort", From ca1a2e052543542e07e3ae6ab379ba942bfcf075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 19 Nov 2019 11:12:01 +0100 Subject: [PATCH 35/37] Update test expectations --- tests/wpt/mozilla/meta/MANIFEST.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/wpt/mozilla/meta/MANIFEST.json b/tests/wpt/mozilla/meta/MANIFEST.json index e675b665aa6..19735a7bdb0 100644 --- a/tests/wpt/mozilla/meta/MANIFEST.json +++ b/tests/wpt/mozilla/meta/MANIFEST.json @@ -19023,7 +19023,7 @@ "testharness" ], "mozilla/interfaces.html": [ - "4006cae2d79ba4ca21c229084fcb528b8a4156f1", + "db1c5f553443121aa8eba862bcbaa3189ee42c2c", "testharness" ], "mozilla/interfaces.js": [ From 6dcdfefd727aec1ae770a0a9e59f2e196906176d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 19 Nov 2019 11:19:39 +0100 Subject: [PATCH 36/37] Fix simpleservo build --- ports/libsimpleservo/capi/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ports/libsimpleservo/capi/src/lib.rs b/ports/libsimpleservo/capi/src/lib.rs index 2d67f088b9a..7f23b1468e7 100644 --- a/ports/libsimpleservo/capi/src/lib.rs +++ b/ports/libsimpleservo/capi/src/lib.rs @@ -17,7 +17,7 @@ use env_logger; use log::LevelFilter; use simpleservo::{self, gl_glue, ServoGlue, SERVO}; use simpleservo::{ - Coordinates, EventLoopWaker, HostTrait, InitOptions, MediaSessionEvent, MouseButton, + Coordinates, EventLoopWaker, HostTrait, InitOptions, MouseButton, VRInitOptions, }; use std::ffi::{CStr, CString}; @@ -716,16 +716,16 @@ impl HostTrait for HostCallbacks { fn on_media_session_metadata( &self, title: String, - artist: Option, - album: Option, + artist: String, + album: String, ) { debug!( "on_media_session_metadata ({:?} {:?} {:?})", title, artist, album ); let title = CString::new(title).expect("Can't create string"); - let artist = CString::new(artist.unwrap_or(String::new())).expect("Can't create string"); - let album = CString::new(album.unwrap_or(String::new())).expect("Can't create string"); + let artist = CString::new(artist).expect("Can't create string"); + let album = CString::new(album).expect("Can't create string"); (self.0.on_media_session_metadata)(title.as_ptr(), artist.as_ptr(), album.as_ptr()); } From 9f77ea11651f2d987d84e01e222f2382d525b868 Mon Sep 17 00:00:00 2001 From: Fernando Jimenez Moreno Date: Wed, 20 Nov 2019 13:17:47 +0100 Subject: [PATCH 37/37] Mach fmt, this time including C++ files --- ports/libsimpleservo/capi/src/lib.rs | 10 ++-------- tests/wpt/mozilla/meta/MANIFEST.json | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/ports/libsimpleservo/capi/src/lib.rs b/ports/libsimpleservo/capi/src/lib.rs index 7f23b1468e7..956c7e3a380 100644 --- a/ports/libsimpleservo/capi/src/lib.rs +++ b/ports/libsimpleservo/capi/src/lib.rs @@ -17,8 +17,7 @@ use env_logger; use log::LevelFilter; use simpleservo::{self, gl_glue, ServoGlue, SERVO}; use simpleservo::{ - Coordinates, EventLoopWaker, HostTrait, InitOptions, MouseButton, - VRInitOptions, + Coordinates, EventLoopWaker, HostTrait, InitOptions, MouseButton, VRInitOptions, }; use std::ffi::{CStr, CString}; #[cfg(target_os = "windows")] @@ -713,12 +712,7 @@ impl HostTrait for HostCallbacks { (self.0.set_clipboard_contents)(contents.as_ptr()); } - fn on_media_session_metadata( - &self, - title: String, - artist: String, - album: String, - ) { + fn on_media_session_metadata(&self, title: String, artist: String, album: String) { debug!( "on_media_session_metadata ({:?} {:?} {:?})", title, artist, album diff --git a/tests/wpt/mozilla/meta/MANIFEST.json b/tests/wpt/mozilla/meta/MANIFEST.json index 19735a7bdb0..409d33b4c58 100644 --- a/tests/wpt/mozilla/meta/MANIFEST.json +++ b/tests/wpt/mozilla/meta/MANIFEST.json @@ -19023,7 +19023,7 @@ "testharness" ], "mozilla/interfaces.html": [ - "db1c5f553443121aa8eba862bcbaa3189ee42c2c", + "f1d58732adafef4afc9f9b7f16d6961e4b74a5e9", "testharness" ], "mozilla/interfaces.js": [