From 518ec87cdd769dada57e391a73960a891a622222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 7 Nov 2019 06:00:11 +0100 Subject: [PATCH 1/6] Update servo-media with MediaElementAudioSourceNode engine --- Cargo.lock | 23 ++++++++++++----------- components/script/dom/bindings/trace.rs | 4 ++-- components/script/dom/htmlmediaelement.rs | 21 +++++++++++---------- components/script/dom/htmlvideoelement.rs | 6 +++--- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a36afea94e2..308b19c2081 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4446,7 +4446,7 @@ dependencies = [ [[package]] name = "servo-media" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "servo-media-audio", "servo-media-player", @@ -4458,7 +4458,7 @@ dependencies = [ [[package]] name = "servo-media-audio" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "boxfnonce", "byte-slice-cast", @@ -4467,6 +4467,7 @@ dependencies = [ "petgraph", "serde", "serde_derive", + "servo-media-player", "servo-media-traits", "servo_media_derive", "smallvec", @@ -4490,7 +4491,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "boxfnonce", "byte-slice-cast", @@ -4526,7 +4527,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "gstreamer", "gstreamer-video", @@ -4536,7 +4537,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render-android" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "glib", "gstreamer", @@ -4549,7 +4550,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render-unix" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "glib", "gstreamer", @@ -4562,7 +4563,7 @@ dependencies = [ [[package]] name = "servo-media-player" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "ipc-channel", "serde", @@ -4574,7 +4575,7 @@ dependencies = [ [[package]] name = "servo-media-streams" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "lazy_static", "uuid", @@ -4583,12 +4584,12 @@ dependencies = [ [[package]] name = "servo-media-traits" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" [[package]] name = "servo-media-webrtc" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "boxfnonce", "log", @@ -4685,7 +4686,7 @@ dependencies = [ [[package]] name = "servo_media_derive" version = "0.1.0" -source = "git+https://github.com/servo/media#5380170bb42286ce9009d83e1034904bea8f00ee" +source = "git+https://github.com/servo/media#220ed1388f2ba008b05f5e94aca21dd14aa37290" dependencies = [ "proc-macro2 1.0.1", "quote 1.0.2", diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 65faac1bb95..301d8fe5390 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -107,7 +107,7 @@ use servo_media::audio::context::AudioContext; use servo_media::audio::graph::NodeId; use servo_media::audio::panner_node::{DistanceModel, PanningModel}; use servo_media::audio::param::ParamType; -use servo_media::player::frame::Frame; +use servo_media::player::video::VideoFrame; use servo_media::player::Player; use servo_media::streams::registry::MediaStreamId; use servo_media::streams::MediaStreamType; @@ -532,7 +532,7 @@ unsafe_no_jsmanaged_fields!(Point2D, Rect); unsafe_no_jsmanaged_fields!(Rect); unsafe_no_jsmanaged_fields!(CascadeData); unsafe_no_jsmanaged_fields!(WindowGLContext); -unsafe_no_jsmanaged_fields!(Frame); +unsafe_no_jsmanaged_fields!(VideoFrame); unsafe_no_jsmanaged_fields!(WebGLContextId); unsafe impl<'a> JSTraceable for &'a str { diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 5afc2deb381..097be0b5388 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -79,7 +79,7 @@ use net_traits::{CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseLis use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType}; use script_layout_interface::HTMLMediaData; use servo_config::pref; -use servo_media::player::frame::{Frame, FrameRenderer}; +use servo_media::player::video::{VideoFrame, VideoFrameRenderer}; use servo_media::player::{PlaybackState, Player, PlayerError, PlayerEvent, SeekLock, StreamType}; use servo_media::{ClientContextId, ServoMedia, SupportsMediaType}; use servo_url::ServoUrl; @@ -100,10 +100,10 @@ enum FrameStatus { Unlocked, } -struct FrameHolder(FrameStatus, Frame); +struct FrameHolder(FrameStatus, VideoFrame); impl FrameHolder { - fn new(frame: Frame) -> FrameHolder { + fn new(frame: VideoFrame) -> FrameHolder { FrameHolder(FrameStatus::Unlocked, frame) } @@ -119,7 +119,7 @@ impl FrameHolder { }; } - fn set(&mut self, new_frame: Frame) { + fn set(&mut self, new_frame: VideoFrame) { if self.0 == FrameStatus::Unlocked { self.1 = new_frame }; @@ -137,7 +137,7 @@ impl FrameHolder { } } - fn get_frame(&self) -> Frame { + fn get_frame(&self) -> VideoFrame { self.1.clone() } } @@ -170,8 +170,8 @@ impl MediaFrameRenderer { } } -impl FrameRenderer for MediaFrameRenderer { - fn render(&mut self, frame: Frame) { +impl VideoFrameRenderer for MediaFrameRenderer { + fn render(&mut self, frame: VideoFrame) { let mut txn = Transaction::new(); if let Some(old_image_key) = mem::replace(&mut self.very_old_frame, self.old_frame.take()) { @@ -1325,7 +1325,7 @@ impl HTMLMediaElement { let window = window_from_node(self); let (action_sender, action_receiver) = ipc::channel::().unwrap(); - let renderer: Option>> = match self.media_type_id() { + let renderer: Option>> = match self.media_type_id() { HTMLMediaElementTypeId::HTMLAudioElement => None, HTMLMediaElementTypeId::HTMLVideoElement => Some(self.frame_renderer.clone()), }; @@ -1340,6 +1340,7 @@ impl HTMLMediaElement { stream_type, action_sender, renderer, + None, Box::new(window.get_player_context()), ); @@ -1527,7 +1528,7 @@ impl HTMLMediaElement { ))); self.upcast::().fire_event(atom!("error")); }, - PlayerEvent::FrameUpdated => { + PlayerEvent::VideoFrameUpdated => { self.upcast::().dirty(NodeDamage::OtherNodeDamage); }, PlayerEvent::MetadataUpdated(ref metadata) => { @@ -1855,7 +1856,7 @@ impl HTMLMediaElement { } } - pub fn get_current_frame(&self) -> Option { + pub fn get_current_frame(&self) -> Option { match self.frame_renderer.lock().unwrap().current_frame_holder { Some(ref holder) => Some(holder.get_frame()), None => return None, diff --git a/components/script/dom/htmlvideoelement.rs b/components/script/dom/htmlvideoelement.rs index 55990c1ffe5..69f8b9a85e6 100644 --- a/components/script/dom/htmlvideoelement.rs +++ b/components/script/dom/htmlvideoelement.rs @@ -35,7 +35,7 @@ use net_traits::{ CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseListener, FetchResponseMsg, }; use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType}; -use servo_media::player::frame::Frame; +use servo_media::player::video::VideoFrame; use servo_url::ServoUrl; use std::cell::Cell; use std::sync::{Arc, Mutex}; @@ -58,8 +58,8 @@ pub struct HTMLVideoElement { /// is being fetched. load_blocker: DomRefCell>, /// A copy of the last frame - #[ignore_malloc_size_of = "Frame"] - last_frame: DomRefCell>, + #[ignore_malloc_size_of = "VideoFrame"] + last_frame: DomRefCell>, } impl HTMLVideoElement { From ea054871bf5157561d7cb28cb202e36046a328fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 7 Nov 2019 16:29:48 +0100 Subject: [PATCH 2/6] Rename frame_renderer to video_renderer --- components/script/dom/htmlmediaelement.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 097be0b5388..53756a10576 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -325,7 +325,7 @@ pub struct HTMLMediaElement { #[ignore_malloc_size_of = "servo_media"] player: DomRefCell>>>, #[ignore_malloc_size_of = "Arc"] - frame_renderer: Arc>, + video_renderer: Arc>, /// https://html.spec.whatwg.org/multipage/#show-poster-flag show_poster: Cell, /// https://html.spec.whatwg.org/multipage/#dom-media-duration @@ -410,7 +410,7 @@ impl HTMLMediaElement { pending_play_promises: Default::default(), in_flight_play_promises_queue: Default::default(), player: Default::default(), - frame_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new( + video_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new( document.window().get_webrender_api_sender(), ))), show_poster: Cell::new(true), @@ -1293,7 +1293,7 @@ impl HTMLMediaElement { // Step 6. if let ImageResponse::Loaded(image, _) = image { - self.frame_renderer + self.video_renderer .lock() .unwrap() .render_poster_frame(image); @@ -1327,7 +1327,7 @@ impl HTMLMediaElement { let (action_sender, action_receiver) = ipc::channel::().unwrap(); let renderer: Option>> = match self.media_type_id() { HTMLMediaElementTypeId::HTMLAudioElement => None, - HTMLMediaElementTypeId::HTMLVideoElement => Some(self.frame_renderer.clone()), + HTMLMediaElementTypeId::HTMLVideoElement => Some(self.video_renderer.clone()), }; let pipeline_id = window @@ -1386,7 +1386,7 @@ impl HTMLMediaElement { .unwrap_or((0, None)); self.id.set(player_id); - self.frame_renderer.lock().unwrap().player_id = Some(player_id); + self.video_renderer.lock().unwrap().player_id = Some(player_id); if let Some(image_receiver) = image_receiver { let trusted_node = Trusted::new(self); @@ -1401,11 +1401,11 @@ impl HTMLMediaElement { if let Err(err) = task_source.queue_with_canceller( task!(handle_glplayer_message: move || { trace!("GLPlayer message {:?}", msg); - let frame_renderer = this.root().frame_renderer.clone(); + let video_renderer = this.root().video_renderer.clone(); match msg { GLPlayerMsgForward::Lock(sender) => { - frame_renderer + video_renderer .lock() .unwrap() .current_frame_holder @@ -1416,7 +1416,7 @@ impl HTMLMediaElement { }); }, GLPlayerMsgForward::Unlock() => { - frame_renderer + video_renderer .lock() .unwrap() .current_frame_holder @@ -1857,7 +1857,7 @@ impl HTMLMediaElement { } pub fn get_current_frame(&self) -> Option { - match self.frame_renderer.lock().unwrap().current_frame_holder { + match self.video_renderer.lock().unwrap().current_frame_holder { Some(ref holder) => Some(holder.get_frame()), None => return None, } @@ -2366,7 +2366,7 @@ impl LayoutHTMLMediaElementHelpers for LayoutDom { fn data(&self) -> HTMLMediaData { let media = unsafe { &*self.unsafe_get() }; HTMLMediaData { - current_frame: media.frame_renderer.lock().unwrap().current_frame.clone(), + current_frame: media.video_renderer.lock().unwrap().current_frame.clone(), } } } From 03485d730ffc3b5c3b9e2c66c2d985c83ccb5665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 7 Nov 2019 16:45:34 +0100 Subject: [PATCH 3/6] MediaElementAudioSourceNode implementation --- components/script/dom/audiocontext.rs | 4 + .../script/dom/mediaelementaudiosourcenode.rs | 81 +++++++++++++++++++ components/script/dom/mod.rs | 1 + .../MediaElementAudioSourceNode.webidl | 17 ++++ 4 files changed, 103 insertions(+) create mode 100644 components/script/dom/mediaelementaudiosourcenode.rs create mode 100644 components/script/dom/webidls/MediaElementAudioSourceNode.webidl diff --git a/components/script/dom/audiocontext.rs b/components/script/dom/audiocontext.rs index 2b6f04b421b..033ac49ed11 100644 --- a/components/script/dom/audiocontext.rs +++ b/components/script/dom/audiocontext.rs @@ -97,6 +97,10 @@ impl AudioContext { self.context.resume(); } } + + pub fn base(&self) -> DomRoot { + DomRoot::from_ref(&self.context) + } } impl AudioContextMethods for AudioContext { diff --git a/components/script/dom/mediaelementaudiosourcenode.rs b/components/script/dom/mediaelementaudiosourcenode.rs new file mode 100644 index 00000000000..8ec25026bc1 --- /dev/null +++ b/components/script/dom/mediaelementaudiosourcenode.rs @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::dom::audiocontext::AudioContext; +use crate::dom::audionode::AudioNode; +use crate::dom::bindings::codegen::Bindings::MediaElementAudioSourceNodeBinding; +use crate::dom::bindings::codegen::Bindings::MediaElementAudioSourceNodeBinding::MediaElementAudioSourceNodeMethods; +use crate::dom::bindings::codegen::Bindings::MediaElementAudioSourceNodeBinding::MediaElementAudioSourceOptions; +use crate::dom::bindings::error::Fallible; +use crate::dom::bindings::reflector::reflect_dom_object; +use crate::dom::bindings::root::{Dom, DomRoot}; +use crate::dom::htmlmediaelement::HTMLMediaElement; +use crate::dom::window::Window; +use dom_struct::dom_struct; +use servo_media::audio::media_element_source_node::MediaElementSourceNodeMessage; +use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage}; +use std::sync::mpsc; + +#[dom_struct] +pub struct MediaElementAudioSourceNode { + node: AudioNode, + media_element: Dom, +} + +impl MediaElementAudioSourceNode { + #[allow(unrooted_must_root)] + fn new_inherited( + window: &Window, + context: &AudioContext, + options: &MediaElementAudioSourceOptions, + ) -> Fallible { + let node = AudioNode::new_inherited( + AudioNodeInit::MediaElementSourceNode, + &*context.base(), + Default::default(), + 0, + 1, + )?; + let (sender, receiver) = mpsc::channel(); + node.message(AudioNodeMessage::MediaElementSourceNode( + MediaElementSourceNodeMessage::GetAudioRenderer(sender), + )); + let audio_renderer = receiver.recv().unwrap(); + // XXX set audio renderer in player. + let media_element = &options.mediaElement; + let media_element = Dom::from_ref(&**media_element); + Ok(MediaElementAudioSourceNode { + node, + media_element, + }) + } + + #[allow(unrooted_must_root)] + pub fn new( + window: &Window, + context: &AudioContext, + options: &MediaElementAudioSourceOptions, + ) -> Fallible> { + let node = MediaElementAudioSourceNode::new_inherited(window, context, options)?; + Ok(reflect_dom_object( + Box::new(node), + window, + MediaElementAudioSourceNodeBinding::Wrap, + )) + } + + pub fn Constructor( + window: &Window, + context: &AudioContext, + options: &MediaElementAudioSourceOptions, + ) -> Fallible> { + MediaElementAudioSourceNode::new(window, context, options) + } +} + +impl MediaElementAudioSourceNodeMethods for MediaElementAudioSourceNode { + fn MediaElement(&self) -> DomRoot { + DomRoot::from_ref(&*self.media_element) + } +} diff --git a/components/script/dom/mod.rs b/components/script/dom/mod.rs index 16d2c6fb686..fbc4d457482 100644 --- a/components/script/dom/mod.rs +++ b/components/script/dom/mod.rs @@ -394,6 +394,7 @@ pub mod inputevent; pub mod keyboardevent; pub mod location; pub mod mediadevices; +pub mod mediaelementaudiosourcenode; pub mod mediaerror; pub mod mediafragmentparser; pub mod medialist; diff --git a/components/script/dom/webidls/MediaElementAudioSourceNode.webidl b/components/script/dom/webidls/MediaElementAudioSourceNode.webidl new file mode 100644 index 00000000000..5afe7775caa --- /dev/null +++ b/components/script/dom/webidls/MediaElementAudioSourceNode.webidl @@ -0,0 +1,17 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +/* + * The origin of this IDL file is + * https://webaudio.github.io/web-audio-api/#mediaelementaudiosourcenode + */ + +dictionary MediaElementAudioSourceOptions { + required HTMLMediaElement mediaElement; +}; + +[Exposed=Window] +interface MediaElementAudioSourceNode : AudioNode { + [Throws] constructor (AudioContext context, MediaElementAudioSourceOptions options); + [SameObject] readonly attribute HTMLMediaElement mediaElement; +}; From a72e46f9c8c897ecff603899704e5b88484c1138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 7 Nov 2019 17:35:10 +0100 Subject: [PATCH 4/6] MediaElementAudioSourceNode: set audio renderer --- components/script/dom/bindings/trace.rs | 2 ++ components/script/dom/htmlmediaelement.rs | 27 ++++++++++++++++--- .../script/dom/mediaelementaudiosourcenode.rs | 8 +++--- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 301d8fe5390..74e6996b67a 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -107,6 +107,7 @@ use servo_media::audio::context::AudioContext; use servo_media::audio::graph::NodeId; use servo_media::audio::panner_node::{DistanceModel, PanningModel}; use servo_media::audio::param::ParamType; +use servo_media::player::audio::AudioRenderer; use servo_media::player::video::VideoFrame; use servo_media::player::Player; use servo_media::streams::registry::MediaStreamId; @@ -534,6 +535,7 @@ unsafe_no_jsmanaged_fields!(CascadeData); unsafe_no_jsmanaged_fields!(WindowGLContext); unsafe_no_jsmanaged_fields!(VideoFrame); unsafe_no_jsmanaged_fields!(WebGLContextId); +unsafe_no_jsmanaged_fields!(Arc>); unsafe impl<'a> JSTraceable for &'a str { #[inline] diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 53756a10576..d10801c5779 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -79,6 +79,7 @@ use net_traits::{CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseLis use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType}; use script_layout_interface::HTMLMediaData; use servo_config::pref; +use servo_media::player::audio::AudioRenderer; use servo_media::player::video::{VideoFrame, VideoFrameRenderer}; use servo_media::player::{PlaybackState, Player, PlayerError, PlayerEvent, SeekLock, StreamType}; use servo_media::{ClientContextId, ServoMedia, SupportsMediaType}; @@ -326,6 +327,8 @@ pub struct HTMLMediaElement { player: DomRefCell>>>, #[ignore_malloc_size_of = "Arc"] video_renderer: Arc>, + #[ignore_malloc_size_of = "Arc"] + audio_renderer: DomRefCell>>>, /// https://html.spec.whatwg.org/multipage/#show-poster-flag show_poster: Cell, /// https://html.spec.whatwg.org/multipage/#dom-media-duration @@ -413,6 +416,7 @@ impl HTMLMediaElement { video_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new( document.window().get_webrender_api_sender(), ))), + audio_renderer: Default::default(), show_poster: Cell::new(true), duration: Cell::new(f64::NAN), playback_position: Cell::new(0.), @@ -1325,11 +1329,14 @@ impl HTMLMediaElement { let window = window_from_node(self); let (action_sender, action_receiver) = ipc::channel::().unwrap(); - let renderer: Option>> = match self.media_type_id() { + let video_renderer: Option>> = match self.media_type_id() + { HTMLMediaElementTypeId::HTMLAudioElement => None, HTMLMediaElementTypeId::HTMLVideoElement => Some(self.video_renderer.clone()), }; + let audio_renderer = self.audio_renderer.borrow().as_ref().map(|r| r.clone()); + let pipeline_id = window .pipeline_id() .expect("Cannot create player outside of a pipeline"); @@ -1339,8 +1346,8 @@ impl HTMLMediaElement { &client_context_id, stream_type, action_sender, - renderer, - None, + video_renderer, + audio_renderer, Box::new(window.get_player_context()), ); @@ -1862,6 +1869,20 @@ impl HTMLMediaElement { None => return None, } } + + /// By default the audio is rendered through the audio sink automatically + /// selected by the servo-media Player instance. However, in some cases, like + /// the WebAudio MediaElementAudioSourceNode, we need to set a custom audio + /// renderer. + pub fn set_audio_renderer(&self, audio_renderer: Arc>) { + *self.audio_renderer.borrow_mut() = Some(audio_renderer); + if let Some(ref player) = *self.player.borrow() { + if let Err(e) = player.lock().unwrap().stop() { + eprintln!("Could not stop player {:?}", e); + } + self.media_element_load_algorithm(); + } + } } // XXX Placeholder for [https://github.com/servo/servo/issues/22293] diff --git a/components/script/dom/mediaelementaudiosourcenode.rs b/components/script/dom/mediaelementaudiosourcenode.rs index 8ec25026bc1..312857569e6 100644 --- a/components/script/dom/mediaelementaudiosourcenode.rs +++ b/components/script/dom/mediaelementaudiosourcenode.rs @@ -26,7 +26,6 @@ pub struct MediaElementAudioSourceNode { impl MediaElementAudioSourceNode { #[allow(unrooted_must_root)] fn new_inherited( - window: &Window, context: &AudioContext, options: &MediaElementAudioSourceOptions, ) -> Fallible { @@ -42,9 +41,8 @@ impl MediaElementAudioSourceNode { MediaElementSourceNodeMessage::GetAudioRenderer(sender), )); let audio_renderer = receiver.recv().unwrap(); - // XXX set audio renderer in player. - let media_element = &options.mediaElement; - let media_element = Dom::from_ref(&**media_element); + options.mediaElement.set_audio_renderer(audio_renderer); + let media_element = Dom::from_ref(&*options.mediaElement); Ok(MediaElementAudioSourceNode { node, media_element, @@ -57,7 +55,7 @@ impl MediaElementAudioSourceNode { context: &AudioContext, options: &MediaElementAudioSourceOptions, ) -> Fallible> { - let node = MediaElementAudioSourceNode::new_inherited(window, context, options)?; + let node = MediaElementAudioSourceNode::new_inherited(context, options)?; Ok(reflect_dom_object( Box::new(node), window, From 04f4da3e135ffac4ecb44b5072db58e36bc9cf01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 7 Nov 2019 18:31:56 +0100 Subject: [PATCH 5/6] Implement AudioContext.createMediaElementAudioSourceNode() --- components/script/dom/audiocontext.rs | 12 ++++++++++++ .../script/dom/mediaelementaudiosourcenode.rs | 13 +++++++------ components/script/dom/webidls/AudioContext.webidl | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/components/script/dom/audiocontext.rs b/components/script/dom/audiocontext.rs index 033ac49ed11..0212cb78ffa 100644 --- a/components/script/dom/audiocontext.rs +++ b/components/script/dom/audiocontext.rs @@ -20,6 +20,8 @@ use crate::dom::bindings::num::Finite; use crate::dom::bindings::refcounted::{Trusted, TrustedPromise}; use crate::dom::bindings::reflector::{reflect_dom_object, DomObject}; use crate::dom::bindings::root::DomRoot; +use crate::dom::htmlmediaelement::HTMLMediaElement; +use crate::dom::mediaelementaudiosourcenode::MediaElementAudioSourceNode; use crate::dom::promise::Promise; use crate::dom::window::Window; use crate::task_source::TaskSource; @@ -244,6 +246,16 @@ impl AudioContextMethods for AudioContext { // Step 6. promise } + + /// https://webaudio.github.io/web-audio-api/#dom-audiocontext-createmediaelementsource + fn CreateMediaElementSource( + &self, + media_element: &HTMLMediaElement, + ) -> Fallible> { + let global = self.global(); + let window = global.as_window(); + MediaElementAudioSourceNode::new(window, self, media_element) + } } impl From for LatencyCategory { diff --git a/components/script/dom/mediaelementaudiosourcenode.rs b/components/script/dom/mediaelementaudiosourcenode.rs index 312857569e6..07ca70bd80e 100644 --- a/components/script/dom/mediaelementaudiosourcenode.rs +++ b/components/script/dom/mediaelementaudiosourcenode.rs @@ -27,7 +27,7 @@ impl MediaElementAudioSourceNode { #[allow(unrooted_must_root)] fn new_inherited( context: &AudioContext, - options: &MediaElementAudioSourceOptions, + media_element: &HTMLMediaElement, ) -> Fallible { let node = AudioNode::new_inherited( AudioNodeInit::MediaElementSourceNode, @@ -41,8 +41,8 @@ impl MediaElementAudioSourceNode { MediaElementSourceNodeMessage::GetAudioRenderer(sender), )); let audio_renderer = receiver.recv().unwrap(); - options.mediaElement.set_audio_renderer(audio_renderer); - let media_element = Dom::from_ref(&*options.mediaElement); + media_element.set_audio_renderer(audio_renderer); + let media_element = Dom::from_ref(media_element); Ok(MediaElementAudioSourceNode { node, media_element, @@ -53,9 +53,9 @@ impl MediaElementAudioSourceNode { pub fn new( window: &Window, context: &AudioContext, - options: &MediaElementAudioSourceOptions, + media_element: &HTMLMediaElement, ) -> Fallible> { - let node = MediaElementAudioSourceNode::new_inherited(context, options)?; + let node = MediaElementAudioSourceNode::new_inherited(context, media_element)?; Ok(reflect_dom_object( Box::new(node), window, @@ -68,11 +68,12 @@ impl MediaElementAudioSourceNode { context: &AudioContext, options: &MediaElementAudioSourceOptions, ) -> Fallible> { - MediaElementAudioSourceNode::new(window, context, options) + MediaElementAudioSourceNode::new(window, context, &*options.mediaElement) } } impl MediaElementAudioSourceNodeMethods for MediaElementAudioSourceNode { + /// https://webaudio.github.io/web-audio-api/#dom-mediaelementaudiosourcenode-mediaelement fn MediaElement(&self) -> DomRoot { DomRoot::from_ref(&*self.media_element) } diff --git a/components/script/dom/webidls/AudioContext.webidl b/components/script/dom/webidls/AudioContext.webidl index 9e5dd6bd556..cd9e18edfa0 100644 --- a/components/script/dom/webidls/AudioContext.webidl +++ b/components/script/dom/webidls/AudioContext.webidl @@ -33,7 +33,7 @@ interface AudioContext : BaseAudioContext { Promise suspend(); Promise close(); - // MediaElementAudioSourceNode createMediaElementSource(HTMLMediaElement mediaElement); + [Throws] MediaElementAudioSourceNode createMediaElementSource(HTMLMediaElement mediaElement); // MediaStreamAudioSourceNode createMediaStreamSource(MediaStream mediaStream); // MediaStreamTrackAudioSourceNode createMediaStreamTrackSource(MediaStreamTrack mediaStreamTrack); // MediaStreamAudioDestinationNode createMediaStreamDestination(); From da2f601e93e8388033b87fa49cfdae4796aaf7cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 19 Nov 2019 10:33:41 +0100 Subject: [PATCH 6/6] Update tests for MediaElementAudioSourceNode --- .../webaudio/idlharness.https.window.js.ini | 108 ------------------ tests/wpt/mozilla/meta/MANIFEST.json | 2 +- .../wpt/mozilla/tests/mozilla/interfaces.html | 1 + 3 files changed, 2 insertions(+), 109 deletions(-) diff --git a/tests/wpt/metadata/webaudio/idlharness.https.window.js.ini b/tests/wpt/metadata/webaudio/idlharness.https.window.js.ini index 37aa74faea8..9ba5b568945 100644 --- a/tests/wpt/metadata/webaudio/idlharness.https.window.js.ini +++ b/tests/wpt/metadata/webaudio/idlharness.https.window.js.ini @@ -2,9 +2,6 @@ [AudioNode interface: calling connect(AudioNode, unsigned long, unsigned long) on new ConvolverNode(context) with too few arguments must throw TypeError] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "disconnect(AudioParam)" with the proper type] - expected: FAIL - [ConvolverNode interface: existence and properties of interface prototype object's @@unscopables property] expected: FAIL @@ -38,9 +35,6 @@ [ScriptProcessorNode interface: context.createScriptProcessor() must inherit property "onaudioprocess" with the proper type] expected: FAIL - [MediaElementAudioSourceNode interface: existence and properties of interface prototype object's @@unscopables property] - expected: FAIL - [AudioNode interface: worklet_node must inherit property "disconnect(AudioParam, unsigned long)" with the proper type] expected: FAIL @@ -59,15 +53,9 @@ [AudioNode interface: calling disconnect(AudioNode) on new WaveShaperNode(context) with too few arguments must throw TypeError] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "disconnect(AudioNode)" with the proper type] - expected: FAIL - [AudioNode interface: context.createScriptProcessor() must inherit property "disconnect()" with the proper type] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "disconnect(AudioNode, unsigned long, unsigned long)" with the proper type] - expected: FAIL - [AudioNode interface: new MediaStreamAudioDestinationNode(context) must inherit property "context" with the proper type] expected: FAIL @@ -116,9 +104,6 @@ [AudioNode interface: calling connect(AudioParam, unsigned long) on new ConvolverNode(context) with too few arguments must throw TypeError] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "connect(AudioNode, unsigned long, unsigned long)" with the proper type] - expected: FAIL - [OscillatorNode interface: operation setPeriodicWave(PeriodicWave)] expected: FAIL @@ -179,15 +164,9 @@ [AudioProcessingEvent interface: existence and properties of interface object] expected: FAIL - [AudioContext interface: operation createMediaElementSource(HTMLMediaElement)] - expected: FAIL - [AudioNode interface: calling disconnect(AudioNode, unsigned long) on new DynamicsCompressorNode(context) with too few arguments must throw TypeError] expected: FAIL - [AudioNode interface: calling connect(AudioParam, unsigned long) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - [BaseAudioContext interface: operation createPeriodicWave([object Object\], [object Object\], PeriodicWaveConstraints)] expected: FAIL @@ -197,9 +176,6 @@ [ScriptProcessorNode interface: existence and properties of interface prototype object] expected: FAIL - [AudioContext interface: calling createMediaElementSource(HTMLMediaElement) on context with too few arguments must throw TypeError] - expected: FAIL - [AudioNode interface: calling connect(AudioNode, unsigned long, unsigned long) on new DynamicsCompressorNode(context) with too few arguments must throw TypeError] expected: FAIL @@ -233,9 +209,6 @@ [AudioNode interface: new DynamicsCompressorNode(context) must inherit property "channelCount" with the proper type] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "channelCountMode" with the proper type] - expected: FAIL - [AudioNode interface: calling connect(AudioParam, unsigned long) on new DelayNode(context) with too few arguments must throw TypeError] expected: FAIL @@ -257,9 +230,6 @@ [AudioNode interface: new DynamicsCompressorNode(context) must inherit property "disconnect(AudioNode, unsigned long)" with the proper type] expected: FAIL - [AudioNode interface: calling disconnect(unsigned long) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - [IIRFilterNode interface: operation getFrequencyResponse(Float32Array, Float32Array, Float32Array)] expected: FAIL @@ -305,9 +275,6 @@ [MediaStreamAudioSourceNode interface object length] expected: FAIL - [AudioNode interface: calling disconnect(AudioParam) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - [AudioWorklet interface object name] expected: FAIL @@ -317,9 +284,6 @@ [AudioNode interface: context.createScriptProcessor() must inherit property "disconnect(AudioParam, unsigned long)" with the proper type] expected: FAIL - [MediaElementAudioSourceNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "mediaElement" with the proper type] - expected: FAIL - [BaseAudioContext interface: operation decodeAudioData(ArrayBuffer, DecodeSuccessCallback, DecodeErrorCallback)] expected: FAIL @@ -335,15 +299,6 @@ [StereoPannerNode interface: existence and properties of interface object] expected: FAIL - [MediaElementAudioSourceNode interface: existence and properties of interface object] - expected: FAIL - - [AudioNode interface: calling connect(AudioNode, unsigned long, unsigned long) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "disconnect()" with the proper type] - expected: FAIL - [MediaStreamAudioSourceNode interface: existence and properties of interface prototype object's "constructor" property] expected: FAIL @@ -419,9 +374,6 @@ [AudioNode interface: worklet_node must inherit property "disconnect()" with the proper type] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "numberOfInputs" with the proper type] - expected: FAIL - [BaseAudioContext interface: attribute audioWorklet] expected: FAIL @@ -455,21 +407,12 @@ [DelayNode interface: existence and properties of interface prototype object] expected: FAIL - [AudioNode interface: calling disconnect(AudioNode, unsigned long, unsigned long) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - [DynamicsCompressorNode interface: existence and properties of interface prototype object's @@unscopables property] expected: FAIL [MediaStreamAudioSourceNode interface: existence and properties of interface object] expected: FAIL - [MediaElementAudioSourceNode interface: existence and properties of interface prototype object] - expected: FAIL - - [AudioNode interface: calling disconnect(AudioParam, unsigned long) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - [AudioNode interface: new DynamicsCompressorNode(context) must inherit property "disconnect(AudioParam, unsigned long)" with the proper type] expected: FAIL @@ -479,9 +422,6 @@ [AudioNode interface: new MediaStreamAudioDestinationNode(context) must inherit property "channelCountMode" with the proper type] expected: FAIL - [MediaElementAudioSourceNode interface object length] - expected: FAIL - [AudioNode interface: new WaveShaperNode(context) must inherit property "disconnect(AudioParam, unsigned long)" with the proper type] expected: FAIL @@ -515,9 +455,6 @@ [AudioProcessingEvent interface: attribute outputBuffer] expected: FAIL - [AudioContext interface: context must inherit property "createMediaElementSource(HTMLMediaElement)" with the proper type] - expected: FAIL - [AudioProcessingEvent interface object length] expected: FAIL @@ -605,9 +542,6 @@ [MediaStreamAudioDestinationNode interface: existence and properties of interface prototype object's "constructor" property] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "channelInterpretation" with the proper type] - expected: FAIL - [BaseAudioContext interface: operation createIIRFilter([object Object\], [object Object\])] expected: FAIL @@ -620,15 +554,9 @@ [AudioNode interface: calling disconnect(AudioNode, unsigned long, unsigned long) on new ConvolverNode(context) with too few arguments must throw TypeError] expected: FAIL - [AudioNode interface: calling disconnect(AudioNode, unsigned long) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - [AudioNode interface: new DynamicsCompressorNode(context) must inherit property "disconnect(AudioParam)" with the proper type] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "disconnect(AudioNode, unsigned long)" with the proper type] - expected: FAIL - [AudioWorkletNode must be primary interface of worklet_node] expected: FAIL @@ -659,9 +587,6 @@ [BaseAudioContext interface: context must inherit property "createScriptProcessor(unsigned long, unsigned long, unsigned long)" with the proper type] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "context" with the proper type] - expected: FAIL - [AudioNode interface: calling disconnect(AudioParam, unsigned long) on new DelayNode(context) with too few arguments must throw TypeError] expected: FAIL @@ -716,9 +641,6 @@ [ConvolverNode interface: attribute buffer] expected: FAIL - [AudioNode interface: calling disconnect(AudioNode) on new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) with too few arguments must throw TypeError] - expected: FAIL - [AudioNode interface: new WaveShaperNode(context) must inherit property "disconnect(AudioNode)" with the proper type] expected: FAIL @@ -731,15 +653,9 @@ [BaseAudioContext interface: calling createPeriodicWave([object Object\], [object Object\], PeriodicWaveConstraints) on new OfflineAudioContext(1, 1, sample_rate) with too few arguments must throw TypeError] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "disconnect(AudioParam, unsigned long)" with the proper type] - expected: FAIL - [AudioNode interface: calling disconnect(AudioParam, unsigned long) on new WaveShaperNode(context) with too few arguments must throw TypeError] expected: FAIL - [MediaElementAudioSourceNode interface object name] - expected: FAIL - [AudioProcessingEvent interface: attribute playbackTime] expected: FAIL @@ -782,9 +698,6 @@ [AudioNode interface: calling disconnect(AudioNode, unsigned long) on new MediaStreamAudioDestinationNode(context) with too few arguments must throw TypeError] expected: FAIL - [MediaElementAudioSourceNode must be primary interface of new MediaElementAudioSourceNode(context, {mediaElement: new Audio})] - expected: FAIL - [ScriptProcessorNode interface: attribute onaudioprocess] expected: FAIL @@ -845,15 +758,9 @@ [OscillatorNode interface: calling setPeriodicWave(PeriodicWave) on new OscillatorNode(context) with too few arguments must throw TypeError] expected: FAIL - [MediaElementAudioSourceNode interface: existence and properties of interface prototype object's "constructor" property] - expected: FAIL - [AudioNode interface: new DynamicsCompressorNode(context) must inherit property "disconnect()" with the proper type] expected: FAIL - [Stringification of new MediaElementAudioSourceNode(context, {mediaElement: new Audio})] - expected: FAIL - [AudioNode interface: calling disconnect(AudioNode, unsigned long) on new DelayNode(context) with too few arguments must throw TypeError] expected: FAIL @@ -890,9 +797,6 @@ [AudioNode interface: new ConvolverNode(context) must inherit property "disconnect(AudioParam)" with the proper type] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "numberOfOutputs" with the proper type] - expected: FAIL - [DynamicsCompressorNode interface: attribute attack] expected: FAIL @@ -923,9 +827,6 @@ [BaseAudioContext interface: calling createScriptProcessor(unsigned long, unsigned long, unsigned long) on context with too few arguments must throw TypeError] expected: FAIL - [MediaElementAudioSourceNode interface: attribute mediaElement] - expected: FAIL - [AudioWorklet interface: existence and properties of interface prototype object's @@unscopables property] expected: FAIL @@ -1013,9 +914,6 @@ [DynamicsCompressorNode interface: existence and properties of interface prototype object] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "channelCount" with the proper type] - expected: FAIL - [AudioNode interface: new DelayNode(context) must inherit property "channelCountMode" with the proper type] expected: FAIL @@ -1055,9 +953,6 @@ [AudioNode interface: new MediaStreamAudioDestinationNode(context) must inherit property "disconnect(AudioParam)" with the proper type] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "connect(AudioParam, unsigned long)" with the proper type] - expected: FAIL - [AudioNode interface: new DelayNode(context) must inherit property "disconnect(AudioNode, unsigned long, unsigned long)" with the proper type] expected: FAIL @@ -1088,9 +983,6 @@ [WaveShaperNode interface object name] expected: FAIL - [AudioNode interface: new MediaElementAudioSourceNode(context, {mediaElement: new Audio}) must inherit property "disconnect(unsigned long)" with the proper type] - expected: FAIL - [BaseAudioContext interface: new OfflineAudioContext(1, 1, sample_rate) must inherit property "createDynamicsCompressor()" with the proper type] expected: FAIL diff --git a/tests/wpt/mozilla/meta/MANIFEST.json b/tests/wpt/mozilla/meta/MANIFEST.json index 8ecff2bc7af..84508fe4a60 100644 --- a/tests/wpt/mozilla/meta/MANIFEST.json +++ b/tests/wpt/mozilla/meta/MANIFEST.json @@ -18989,7 +18989,7 @@ "testharness" ], "mozilla/interfaces.html": [ - "3578a6e01cc3bb6daf8b4942d672ecb7da1ff6b5", + "4006cae2d79ba4ca21c229084fcb528b8a4156f1", "testharness" ], "mozilla/interfaces.js": [ diff --git a/tests/wpt/mozilla/tests/mozilla/interfaces.html b/tests/wpt/mozilla/tests/mozilla/interfaces.html index 3578a6e01cc..4006cae2d79 100644 --- a/tests/wpt/mozilla/tests/mozilla/interfaces.html +++ b/tests/wpt/mozilla/tests/mozilla/interfaces.html @@ -163,6 +163,7 @@ test_interfaces([ "InputEvent", "KeyboardEvent", "Location", + "MediaElementAudioSourceNode", "MediaError", "MediaList", "MediaQueryList",