Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-06 16:45:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

MediaStream playback through audio and video elements

Browse source
This commit is contained in:
Fernando Jiménez Moreno 2019-04-03 11:58:49 +02:00
parent f142b1d1c7
commit af242a0571
5 changed files with 116 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

23
Cargo.lock generated
View file

@ -3764,7 +3764,7 @@ dependencies = [
[[package]]
name = "servo-media"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"servo-media-audio 0.1.0 (git+https://github.com/servo/media)",
"servo-media-player 0.1.0 (git+https://github.com/servo/media)",
@ -3775,7 +3775,7 @@ dependencies = [
[[package]]
name = "servo-media-audio"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3791,7 +3791,7 @@ dependencies = [
[[package]]
name = "servo-media-auto"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"servo-media-dummy 0.1.0 (git+https://github.com/servo/media)",
"servo-media-gstreamer 0.1.0 (git+https://github.com/servo/media)",
@ -3800,7 +3800,7 @@ dependencies = [
[[package]]
name = "servo-media-dummy"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3814,7 +3814,7 @@ dependencies = [
[[package]]
name = "servo-media-gstreamer"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3845,22 +3845,27 @@ dependencies = [
[[package]]
name = "servo-media-player"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"ipc-channel 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"servo-media-streams 0.1.0 (git+https://github.com/servo/media)",
]
[[package]]
name = "servo-media-streams"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"uuid 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "servo-media-webrtc"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3959,7 +3964,7 @@ dependencies = [
[[package]]
name = "servo_media_derive"
version = "0.1.0"
source = "git+https://github.com/servo/media#a5a8490087d786a4dbebed43e9705874c4351aa0"
source = "git+https://github.com/servo/media#a7e38bb566e81085eef9410b9dd6bd67873ffdd9"
dependencies = [
"proc-macro2 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)",

4
components/script/dom/bindings/trace.rs
View file

@ -104,7 +104,7 @@ use servo_media::audio::graph::NodeId;
use servo_media::audio::panner_node::{DistanceModel, PanningModel};
use servo_media::audio::param::ParamType;
use servo_media::player::Player;
use servo_media::streams::MediaStream as BackendMediaStream;
use servo_media::streams::registry::MediaStreamId;
use servo_media::webrtc::WebRtcController;
use servo_url::{ImmutableOrigin, MutableOrigin, ServoUrl};
use smallvec::SmallVec;
@ -488,7 +488,7 @@ unsafe_no_jsmanaged_fields!(NodeId);
unsafe_no_jsmanaged_fields!(AnalysisEngine, DistanceModel, PanningModel, ParamType);
unsafe_no_jsmanaged_fields!(dyn Player);
unsafe_no_jsmanaged_fields!(WebRtcController);
unsafe_no_jsmanaged_fields!(dyn BackendMediaStream);
unsafe_no_jsmanaged_fields!(MediaStreamId);
unsafe_no_jsmanaged_fields!(Mutex<MediaFrameRenderer>);
unsafe_no_jsmanaged_fields!(RenderApiSender);
unsafe_no_jsmanaged_fields!(ResourceFetchTiming);

137
components/script/dom/htmlmediaelement.rs
View file

@ -215,7 +215,7 @@ pub struct HTMLMediaElement {
#[ignore_malloc_size_of = "promises are hard"]
in_flight_play_promises_queue: DomRefCell<VecDeque<(Box<[Rc<Promise>]>, ErrorResult)>>,
#[ignore_malloc_size_of = "servo_media"]
player: Box<Player>,
player: DomRefCell<Option<Box<Player>>>,
#[ignore_malloc_size_of = "Arc"]
frame_renderer: Arc<Mutex<MediaFrameRenderer>>,
/// https://html.spec.whatwg.org/multipage/#show-poster-flag
@ -295,7 +295,7 @@ impl HTMLMediaElement {
delaying_the_load_event_flag: Default::default(),
pending_play_promises: Default::default(),
in_flight_play_promises_queue: Default::default(),
player: ServoMedia::get().unwrap().create_player(),
player: Default::default(),
frame_renderer: Arc::new(Mutex::new(MediaFrameRenderer::new(
document.window().get_webrender_api_sender(),
))),
@ -330,11 +330,13 @@ impl HTMLMediaElement {
}
fn play_media(&self) {
if let Err(e) = self.player.set_rate(self.playbackRate.get()) {
warn!("Could not set the playback rate {:?}", e);
}
if let Err(e) = self.player.play() {
warn!("Could not play media {:?}", e);
if let Some(ref player) = *self.player.borrow() {
if let Err(e) = player.set_rate(self.playbackRate.get()) {
warn!("Could not set the playback rate {:?}", e);
}
if let Err(e) = player.play() {
warn!("Could not play media {:?}", e);
}
}
}
@ -398,8 +400,10 @@ impl HTMLMediaElement {
// Step 2.3.2.
this.upcast::<EventTarget>().fire_event(atom!("pause"));
if let Err(e) = this.player.pause() {
eprintln!("Could not pause player {:?}", e);
if let Some(ref player) = *this.player.borrow() {
if let Err(e) = player.pause() {
eprintln!("Could not pause player {:?}", e);
}
}
// Step 2.3.3.
@ -762,7 +766,7 @@ impl HTMLMediaElement {
// https://html.spec.whatwg.org/multipage/#concept-media-load-resource
fn resource_fetch_algorithm(&self, resource: Resource) {
if let Err(e) = self.setup_media_player() {
if let Err(e) = self.setup_media_player(&resource) {
eprintln!("Setup media player error {:?}", e);
self.queue_dedicated_media_source_failure_steps();
return;
@ -827,8 +831,14 @@ impl HTMLMediaElement {
Some(ServoUrl::parse(&blob_url).expect("infallible"));
self.fetch_request(None);
},
SrcObject::MediaStream(_) => {
self.queue_dedicated_media_source_failure_steps();
SrcObject::MediaStream(ref stream) => {
for stream in stream.get_tracks() {
if let Err(_) =
self.player.borrow().as_ref().unwrap().set_stream(&stream)
{
self.queue_dedicated_media_source_failure_steps();
}
}
},
}
}
@ -872,8 +882,10 @@ impl HTMLMediaElement {
// Step 5.
this.upcast::<EventTarget>().fire_event(atom!("error"));
if let Err(e) = this.player.stop() {
eprintln!("Could not stop player {:?}", e);
if let Some(ref player) = *this.player.borrow() {
if let Err(e) = player.stop() {
eprintln!("Could not stop player {:?}", e);
}
}
// Step 6.
@ -1126,8 +1138,10 @@ impl HTMLMediaElement {
task_source.queue_simple_event(self.upcast(), atom!("seeking"), &window);
// Step 11.
if let Err(e) = self.player.seek(time) {
eprintln!("Seek error {:?}", e);
if let Some(ref player) = *self.player.borrow() {
if let Err(e) = player.seek(time) {
eprintln!("Seek error {:?}", e);
}
}
// The rest of the steps are handled when the media engine signals a
@ -1175,12 +1189,28 @@ impl HTMLMediaElement {
}
}
fn setup_media_player(&self) -> Result<(), PlayerError> {
let (action_sender, action_receiver) = ipc::channel().unwrap();
fn setup_media_player(&self, resource: &Resource) -> Result<(), ()> {
let stream_type = match *resource {
Resource::Object => {
if let Some(ref src_object) = *self.src_object.borrow() {
match src_object {
SrcObject::MediaStream(_) => StreamType::Stream,
_ => StreamType::Seekable,
}
} else {
return Err(());
}
},
_ => StreamType::Seekable,
};
self.player.register_event_handler(action_sender);
self.player
.register_frame_renderer(self.frame_renderer.clone());
let player = ServoMedia::get().unwrap().create_player(stream_type);
let (action_sender, action_receiver) = ipc::channel().unwrap();
player.register_event_handler(action_sender);
player.register_frame_renderer(self.frame_renderer.clone());
*self.player.borrow_mut() = Some(player);
let trusted_node = Trusted::new(self);
let window = window_from_node(self);
@ -1536,8 +1566,10 @@ impl HTMLMediaElement {
impl Drop for HTMLMediaElement {
fn drop(&mut self) {
if let Err(err) = self.player.shutdown() {
warn!("Error shutting down player {:?}", err);
if let Some(ref player) = *self.player.borrow() {
if let Err(err) = player.shutdown() {
warn!("Error shutting down player {:?}", err);
}
}
}
}
@ -1579,15 +1611,17 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
if self.muted.get() == value {
return;
}
self.muted.set(value);
let _ = self.player.set_mute(value);
let window = window_from_node(self);
window
.task_manager()
.media_element_task_source()
.queue_simple_event(self.upcast(), atom!("volumechange"), &window);
if !self.is_allowed_to_play() {
self.internal_pause_steps();
if let Some(ref player) = *self.player.borrow() {
self.muted.set(value);
let _ = player.set_mute(value);
let window = window_from_node(self);
window
.task_manager()
.media_element_task_source()
.queue_simple_event(self.upcast(), atom!("volumechange"), &window);
if !self.is_allowed_to_play() {
self.internal_pause_steps();
}
}
}
@ -1790,8 +1824,10 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
self.playbackRate.set(*value);
self.queue_ratechange_event();
if self.is_potentially_playing() {
if let Err(e) = self.player.set_rate(*value) {
warn!("Could not set the playback rate {:?}", e);
if let Some(ref player) = *self.player.borrow() {
if let Err(e) = player.set_rate(*value) {
warn!("Could not set the playback rate {:?}", e);
}
}
}
}
@ -1855,9 +1891,11 @@ impl HTMLMediaElementMethods for HTMLMediaElement {
// https://html.spec.whatwg.org/multipage/#dom-media-buffered
fn Buffered(&self) -> DomRoot<TimeRanges> {
let mut buffered = TimeRangesContainer::new();
if let Ok(ranges) = self.player.buffered() {
for range in ranges {
let _ = buffered.add(range.start as f64, range.end as f64);
if let Some(ref player) = *self.player.borrow() {
if let Ok(ranges) = player.buffered() {
for range in ranges {
let _ = buffered.add(range.start as f64, range.end as f64);
}
}
}
TimeRanges::new(self.global().as_window(), buffered)
@ -2129,7 +2167,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
fn process_response(&mut self, metadata: Result<FetchMetadata, NetworkError>) {
let elem = self.elem.root();
if elem.generation_id.get() != self.generation_id {
if elem.generation_id.get() != self.generation_id || elem.player.borrow().is_none() {
// A new fetch request was triggered, so we ignore this response.
return;
}
@ -2155,7 +2193,13 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
// We only set the expected input size if it changes.
if content_length != self.expected_content_length {
if let Some(content_length) = content_length {
if let Err(e) = elem.player.set_input_size(content_length) {
if let Err(e) = elem
.player
.borrow()
.as_ref()
.unwrap()
.set_input_size(content_length)
{
warn!("Could not set player input size {:?}", e);
} else {
self.expected_content_length = Some(content_length);
@ -2178,10 +2222,6 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
if let Some(ref mut current_fetch_context) = *elem.current_fetch_context.borrow_mut() {
current_fetch_context.set_seekable(true);
}
// and we can safely set the type of stream to Seekable.
if let Err(e) = elem.player.set_stream_type(StreamType::Seekable) {
warn!("Could not set stream type to Seekable. {:?}", e);
}
}
// => "If the media data cannot be fetched at all..."
@ -2199,7 +2239,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
let elem = self.elem.root();
// If an error was received previously or if we triggered a new fetch request,
// we skip processing the payload.
if elem.generation_id.get() != self.generation_id {
if elem.generation_id.get() != self.generation_id || elem.player.borrow().is_none() {
return;
}
if let Some(ref current_fetch_context) = *elem.current_fetch_context.borrow() {
@ -2211,7 +2251,7 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
let payload_len = payload.len() as u64;
// Push input data into the player.
if let Err(e) = elem.player.push_data(payload) {
if let Err(e) = elem.player.borrow().as_ref().unwrap().push_data(payload) {
// If we are pushing too much data and we know that we can
// restart the download later from where we left, we cancel
// the current request. Otherwise, we continue the request
@ -2247,13 +2287,18 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
// https://html.spec.whatwg.org/multipage/#media-data-processing-steps-list
fn process_response_eof(&mut self, status: Result<ResourceFetchTiming, NetworkError>) {
let elem = self.elem.root();
if elem.player.borrow().is_none() {
return;
}
// If an error was previously received and no new fetch request was triggered,
// we skip processing the payload and notify the media backend that we are done
// pushing data.
if elem.generation_id.get() == self.generation_id {
if let Some(ref current_fetch_context) = *elem.current_fetch_context.borrow() {
if let Some(CancelReason::Error) = current_fetch_context.cancel_reason() {
if let Err(e) = elem.player.end_of_stream() {
if let Err(e) = elem.player.borrow().as_ref().unwrap().end_of_stream() {
warn!("Could not signal EOS to player {:?}", e);
}
return;

17
components/script/dom/mediastream.rs
View file

@ -9,25 +9,24 @@ use crate::dom::bindings::root::DomRoot;
use crate::dom::eventtarget::EventTarget;
use crate::dom::globalscope::GlobalScope;
use dom_struct::dom_struct;
use servo_media::streams::MediaStream as BackendMediaStream;
use std::mem;
use servo_media::streams::registry::MediaStreamId;
#[dom_struct]
pub struct MediaStream {
eventtarget: EventTarget,
#[ignore_malloc_size_of = "defined in servo-media"]
tracks: DomRefCell<Vec<Box<BackendMediaStream>>>,
tracks: DomRefCell<Vec<MediaStreamId>>,
}
impl MediaStream {
pub fn new_inherited(tracks: Vec<Box<BackendMediaStream>>) -> MediaStream {
pub fn new_inherited(tracks: Vec<MediaStreamId>) -> MediaStream {
MediaStream {
eventtarget: EventTarget::new_inherited(),
tracks: DomRefCell::new(tracks),
}
}
pub fn new(global: &GlobalScope, tracks: Vec<Box<BackendMediaStream>>) -> DomRoot<MediaStream> {
pub fn new(global: &GlobalScope, tracks: Vec<MediaStreamId>) -> DomRoot<MediaStream> {
reflect_dom_object(
Box::new(MediaStream::new_inherited(tracks)),
global,
@ -35,11 +34,7 @@ impl MediaStream {
)
}
pub fn get_tracks(&self) -> Vec<Box<BackendMediaStream>> {
// XXXManishearth we have hard ownership constraints here so we actually empty the vec,
// ideally we should only have a media stream id here, or servo-media hands
// out Arcs
let mut tracks = self.tracks.borrow_mut();
mem::replace(&mut *tracks, vec![])
pub fn get_tracks(&self) -> Vec<MediaStreamId> {
self.tracks.borrow_mut().clone()
}
}

6
components/script/dom/rtcpeerconnection.rs
View file

@ -35,7 +35,7 @@ use crate::task_source::networking::NetworkingTaskSource;
use crate::task_source::TaskSource;
use dom_struct::dom_struct;
use servo_media::streams::MediaStream as BackendMediaStream;
use servo_media::streams::registry::MediaStreamId;
use servo_media::webrtc::{
BundlePolicy, GatheringState, IceCandidate, IceConnectionState, SdpType, SessionDescription,
SignalingState, WebRtcController, WebRtcSignaller,
@ -128,7 +128,7 @@ impl WebRtcSignaller for RTCSignaller {
);
}
fn on_add_stream(&self, _: Box<BackendMediaStream>) {}
fn on_add_stream(&self, _: &MediaStreamId) {}
fn close(&self) {
// do nothing
@ -567,7 +567,7 @@ impl RTCPeerConnectionMethods for RTCPeerConnection {
fn AddStream(&self, stream: &MediaStream) {
let mut tracks = stream.get_tracks();
for track in tracks.drain(..) {
for ref track in tracks.drain(..) {
self.controller.borrow().as_ref().unwrap().add_stream(track);
}
}