Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-08-05 05:30:08 +01:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #24687 - ferjm:mediaelementaudionode, r=Manishearth

Browse source 
WebAudio MediaElementAudioSourceNode implementation

- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes fix #22370
- [X] There are tests for these changes

Depends on https://github.com/servo/media/pull/321

Tested with https://ferjm.github.io/webaudio-examples/media-source-buffer/

I am still investigating why it is playing back with a low rate.
This commit is contained in:
bors-servo 2019-11-19 11:17:40 -05:00 • committed by GitHub
commit 03a6fe0b1f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 179 additions and 147 deletions
Download patch file Download diff file Expand all files Collapse all files

16
components/script/dom/audiocontext.rs
View file

@ -20,6 +20,8 @@ use crate::dom::bindings::num::Finite;
use crate::dom::bindings::refcounted::{Trusted, TrustedPromise};
use crate::dom::bindings::reflector::{reflect_dom_object, DomObject};
use crate::dom::bindings::root::DomRoot;
use crate::dom::htmlmediaelement::HTMLMediaElement;
use crate::dom::mediaelementaudiosourcenode::MediaElementAudioSourceNode;
use crate::dom::promise::Promise;
use crate::dom::window::Window;
use crate::task_source::TaskSource;
@ -97,6 +99,10 @@ impl AudioContext {
self.context.resume();
}
}
pub fn base(&self) -> DomRoot<BaseAudioContext> {
DomRoot::from_ref(&self.context)
}
}
impl AudioContextMethods for AudioContext {
@ -240,6 +246,16 @@ impl AudioContextMethods for AudioContext {
// Step 6.
promise
}
/// https://webaudio.github.io/web-audio-api/#dom-audiocontext-createmediaelementsource
fn CreateMediaElementSource(
&self,
media_element: &HTMLMediaElement,
) -> Fallible<DomRoot<MediaElementAudioSourceNode>> {
let global = self.global();
let window = global.as_window();
MediaElementAudioSourceNode::new(window, self, media_element)
}
}
impl From<AudioContextLatencyCategory> for LatencyCategory {

6
components/script/dom/bindings/trace.rs
View file

@ -107,7 +107,8 @@ use servo_media::audio::context::AudioContext;
use servo_media::audio::graph::NodeId;
use servo_media::audio::panner_node::{DistanceModel, PanningModel};
use servo_media::audio::param::ParamType;
use servo_media::player::frame::Frame;
use servo_media::player::audio::AudioRenderer;
use servo_media::player::video::VideoFrame;
use servo_media::player::Player;
use servo_media::streams::registry::MediaStreamId;
use servo_media::streams::MediaStreamType;
@ -532,8 +533,9 @@ unsafe_no_jsmanaged_fields!(Point2D<f32>, Rect<Au>);
unsafe_no_jsmanaged_fields!(Rect<f32>);
unsafe_no_jsmanaged_fields!(CascadeData);
unsafe_no_jsmanaged_fields!(WindowGLContext);
unsafe_no_jsmanaged_fields!(Frame);
unsafe_no_jsmanaged_fields!(VideoFrame);
unsafe_no_jsmanaged_fields!(WebGLContextId);
unsafe_no_jsmanaged_fields!(Arc<Mutex<dyn AudioRenderer>>);
unsafe impl<'a> JSTraceable for &'a str {
#[inline]

64
components/script/dom/htmlmediaelement.rs
View file

@ -79,7 +79,8 @@ use net_traits::{CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseLis
use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType};
use script_layout_interface::HTMLMediaData;
use servo_config::pref;
use servo_media::player::frame::{Frame, FrameRenderer};
use servo_media::player::audio::AudioRenderer;
use servo_media::player::video::{VideoFrame, VideoFrameRenderer};
use servo_media::player::{PlaybackState, Player, PlayerError, PlayerEvent, SeekLock, StreamType};
use servo_media::{ClientContextId, ServoMedia, SupportsMediaType};
use servo_url::ServoUrl;
@ -100,10 +101,10 @@ enum FrameStatus {
Unlocked,
}
struct FrameHolder(FrameStatus, Frame);
struct FrameHolder(FrameStatus, VideoFrame);
impl FrameHolder {
fn new(frame: Frame) -> FrameHolder {
fn new(frame: VideoFrame) -> FrameHolder {
FrameHolder(FrameStatus::Unlocked, frame)
}
@ -119,7 +120,7 @@ impl FrameHolder {
};
}
fn set(&mut self, new_frame: Frame) {
fn set(&mut self, new_frame: VideoFrame) {
if self.0 == FrameStatus::Unlocked {
self.1 = new_frame
};
@ -137,7 +138,7 @@ impl FrameHolder {
}
}
fn get_frame(&self) -> Frame {
fn get_frame(&self) -> VideoFrame {
self.1.clone()
}
}
@ -170,8 +171,8 @@ impl MediaFrameRenderer {
}
}
impl FrameRenderer for MediaFrameRenderer {
fn render(&mut self, frame: Frame) {
impl VideoFrameRenderer for MediaFrameRenderer {
fn render(&mut self, frame: VideoFrame) {
let mut txn = Transaction::new();
if let Some(old_image_key) = mem::replace(&mut self.very_old_frame, self.old_frame.take()) {
@ -325,7 +326,9 @@ pub struct HTMLMediaElement {
#[ignore_malloc_size_of = "servo_media"]
player: DomRefCell<Option<Arc<Mutex<dyn Player>>>>,
#[ignore_malloc_size_of = "Arc"]
frame_renderer: Arc<Mutex<MediaFrameRenderer>>,
video_renderer: Arc<Mutex<MediaFrameRenderer>>,
#[ignore_malloc_size_of = "Arc"]
audio_renderer: DomRefCell<Option<Arc<Mutex<dyn AudioRenderer>>>>,
/// https://html.spec.whatwg.org/multipage/#show-poster-flag
show_poster: Cell<bool>,
/// https://html.spec.whatwg.org/multipage/#dom-media-duration
@ -410,9 +413,10 @@ impl HTMLMediaElement {
pending_play_promises: Default::default(),
in_flight_play_promises_queue: Default::default(),
player: Default::default(),
frame_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new(
video_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new(
document.window().get_webrender_api_sender(),
))),
audio_renderer: Default::default(),
show_poster: Cell::new(true),
duration: Cell::new(f64::NAN),
playback_position: Cell::new(0.),
@ -1293,7 +1297,7 @@ impl HTMLMediaElement {
// Step 6.
if let ImageResponse::Loaded(image, _) = image {
self.frame_renderer
self.video_renderer
.lock()
.unwrap()
.render_poster_frame(image);
@ -1325,11 +1329,14 @@ impl HTMLMediaElement {
let window = window_from_node(self);
let (action_sender, action_receiver) = ipc::channel::<PlayerEvent>().unwrap();
let renderer: Option<Arc<Mutex<dyn FrameRenderer>>> = match self.media_type_id() {
let video_renderer: Option<Arc<Mutex<dyn VideoFrameRenderer>>> = match self.media_type_id()
{
HTMLMediaElementTypeId::HTMLAudioElement => None,
HTMLMediaElementTypeId::HTMLVideoElement => Some(self.frame_renderer.clone()),
HTMLMediaElementTypeId::HTMLVideoElement => Some(self.video_renderer.clone()),
};
let audio_renderer = self.audio_renderer.borrow().as_ref().map(|r| r.clone());
let pipeline_id = window
.pipeline_id()
.expect("Cannot create player outside of a pipeline");
@ -1339,7 +1346,8 @@ impl HTMLMediaElement {
&client_context_id,
stream_type,
action_sender,
renderer,
video_renderer,
audio_renderer,
Box::new(window.get_player_context()),
);
@ -1385,7 +1393,7 @@ impl HTMLMediaElement {
.unwrap_or((0, None));
self.id.set(player_id);
self.frame_renderer.lock().unwrap().player_id = Some(player_id);
self.video_renderer.lock().unwrap().player_id = Some(player_id);
if let Some(image_receiver) = image_receiver {
let trusted_node = Trusted::new(self);
@ -1400,11 +1408,11 @@ impl HTMLMediaElement {
if let Err(err) = task_source.queue_with_canceller(
task!(handle_glplayer_message: move || {
trace!("GLPlayer message {:?}", msg);
let frame_renderer = this.root().frame_renderer.clone();
let video_renderer = this.root().video_renderer.clone();
match msg {
GLPlayerMsgForward::Lock(sender) => {
frame_renderer
video_renderer
.lock()
.unwrap()
.current_frame_holder
@ -1415,7 +1423,7 @@ impl HTMLMediaElement {
});
},
GLPlayerMsgForward::Unlock() => {
frame_renderer
video_renderer
.lock()
.unwrap()
.current_frame_holder
@ -1527,7 +1535,7 @@ impl HTMLMediaElement {
)));
self.upcast::<EventTarget>().fire_event(atom!("error"));
},
PlayerEvent::FrameUpdated => {
PlayerEvent::VideoFrameUpdated => {
self.upcast::<Node>().dirty(NodeDamage::OtherNodeDamage);
},
PlayerEvent::MetadataUpdated(ref metadata) => {
@ -1855,12 +1863,26 @@ impl HTMLMediaElement {
}
}
pub fn get_current_frame(&self) -> Option<Frame> {
match self.frame_renderer.lock().unwrap().current_frame_holder {
pub fn get_current_frame(&self) -> Option<VideoFrame> {
match self.video_renderer.lock().unwrap().current_frame_holder {
Some(ref holder) => Some(holder.get_frame()),
None => return None,
}
}
/// By default the audio is rendered through the audio sink automatically
/// selected by the servo-media Player instance. However, in some cases, like
/// the WebAudio MediaElementAudioSourceNode, we need to set a custom audio
/// renderer.
pub fn set_audio_renderer(&self, audio_renderer: Arc<Mutex<dyn AudioRenderer>>) {
*self.audio_renderer.borrow_mut() = Some(audio_renderer);
if let Some(ref player) = *self.player.borrow() {
if let Err(e) = player.lock().unwrap().stop() {
eprintln!("Could not stop player {:?}", e);
}
self.media_element_load_algorithm();
}
}
}
// XXX Placeholder for [https://github.com/servo/servo/issues/22293]
@ -2365,7 +2387,7 @@ impl LayoutHTMLMediaElementHelpers for LayoutDom<HTMLMediaElement> {
fn data(&self) -> HTMLMediaData {
let media = unsafe { &*self.unsafe_get() };
HTMLMediaData {
current_frame: media.frame_renderer.lock().unwrap().current_frame.clone(),
current_frame: media.video_renderer.lock().unwrap().current_frame.clone(),
}
}
}

6
components/script/dom/htmlvideoelement.rs
View file

@ -35,7 +35,7 @@ use net_traits::{
CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseListener, FetchResponseMsg,
};
use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType};
use servo_media::player::frame::Frame;
use servo_media::player::video::VideoFrame;
use servo_url::ServoUrl;
use std::cell::Cell;
use std::sync::{Arc, Mutex};
@ -58,8 +58,8 @@ pub struct HTMLVideoElement {
/// is being fetched.
load_blocker: DomRefCell<Option<LoadBlocker>>,
/// A copy of the last frame
#[ignore_malloc_size_of = "Frame"]
last_frame: DomRefCell<Option<Frame>>,
#[ignore_malloc_size_of = "VideoFrame"]
last_frame: DomRefCell<Option<VideoFrame>>,
}
impl HTMLVideoElement {

80
components/script/dom/mediaelementaudiosourcenode.rs Normal file
View file

@ -0,0 +1,80 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::dom::audiocontext::AudioContext;
use crate::dom::audionode::AudioNode;
use crate::dom::bindings::codegen::Bindings::MediaElementAudioSourceNodeBinding;
use crate::dom::bindings::codegen::Bindings::MediaElementAudioSourceNodeBinding::MediaElementAudioSourceNodeMethods;
use crate::dom::bindings::codegen::Bindings::MediaElementAudioSourceNodeBinding::MediaElementAudioSourceOptions;
use crate::dom::bindings::error::Fallible;
use crate::dom::bindings::reflector::reflect_dom_object;
use crate::dom::bindings::root::{Dom, DomRoot};
use crate::dom::htmlmediaelement::HTMLMediaElement;
use crate::dom::window::Window;
use dom_struct::dom_struct;
use servo_media::audio::media_element_source_node::MediaElementSourceNodeMessage;
use servo_media::audio::node::{AudioNodeInit, AudioNodeMessage};
use std::sync::mpsc;
#[dom_struct]
pub struct MediaElementAudioSourceNode {
node: AudioNode,
media_element: Dom<HTMLMediaElement>,
}
impl MediaElementAudioSourceNode {
#[allow(unrooted_must_root)]
fn new_inherited(
context: &AudioContext,
media_element: &HTMLMediaElement,
) -> Fallible<MediaElementAudioSourceNode> {
let node = AudioNode::new_inherited(
AudioNodeInit::MediaElementSourceNode,
&*context.base(),
Default::default(),
0,
1,
)?;
let (sender, receiver) = mpsc::channel();
node.message(AudioNodeMessage::MediaElementSourceNode(
MediaElementSourceNodeMessage::GetAudioRenderer(sender),
));
let audio_renderer = receiver.recv().unwrap();
media_element.set_audio_renderer(audio_renderer);
let media_element = Dom::from_ref(media_element);
Ok(MediaElementAudioSourceNode {
node,
media_element,
})
}
#[allow(unrooted_must_root)]
pub fn new(
window: &Window,
context: &AudioContext,
media_element: &HTMLMediaElement,
) -> Fallible<DomRoot<MediaElementAudioSourceNode>> {
let node = MediaElementAudioSourceNode::new_inherited(context, media_element)?;
Ok(reflect_dom_object(
Box::new(node),
window,
MediaElementAudioSourceNodeBinding::Wrap,
))
}
pub fn Constructor(
window: &Window,
context: &AudioContext,
options: &MediaElementAudioSourceOptions,
) -> Fallible<DomRoot<MediaElementAudioSourceNode>> {
MediaElementAudioSourceNode::new(window, context, &*options.mediaElement)
}
}
impl MediaElementAudioSourceNodeMethods for MediaElementAudioSourceNode {
/// https://webaudio.github.io/web-audio-api/#dom-mediaelementaudiosourcenode-mediaelement
fn MediaElement(&self) -> DomRoot<HTMLMediaElement> {
DomRoot::from_ref(&*self.media_element)
}
}

1
components/script/dom/mod.rs
View file

@ -394,6 +394,7 @@ pub mod inputevent;
pub mod keyboardevent;
pub mod location;
pub mod mediadevices;
pub mod mediaelementaudiosourcenode;
pub mod mediaerror;
pub mod mediafragmentparser;
pub mod medialist;

2
components/script/dom/webidls/AudioContext.webidl
View file

@ -33,7 +33,7 @@ interface AudioContext : BaseAudioContext {
Promise<void> suspend();
Promise<void> close();
// MediaElementAudioSourceNode createMediaElementSource(HTMLMediaElement mediaElement);
[Throws] MediaElementAudioSourceNode createMediaElementSource(HTMLMediaElement mediaElement);
// MediaStreamAudioSourceNode createMediaStreamSource(MediaStream mediaStream);
// MediaStreamTrackAudioSourceNode createMediaStreamTrackSource(MediaStreamTrack mediaStreamTrack);
// MediaStreamAudioDestinationNode createMediaStreamDestination();

17
components/script/dom/webidls/MediaElementAudioSourceNode.webidl Normal file
View file

@ -0,0 +1,17 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
/*
* The origin of this IDL file is
* https://webaudio.github.io/web-audio-api/#mediaelementaudiosourcenode
*/
dictionary MediaElementAudioSourceOptions {
required HTMLMediaElement mediaElement;
};
[Exposed=Window]
interface MediaElementAudioSourceNode : AudioNode {
[Throws] constructor (AudioContext context, MediaElementAudioSourceOptions options);
[SameObject] readonly attribute HTMLMediaElement mediaElement;
};