Auto merge of #23342 - Manishearth:webrtc-streams, r=ferjm

Receive streams in WebRTC (and MediaStreamTrack support)

This adds the `ontrack` event handler to webrtc, and all the `MediaStreamTrack` stuff necessary to make it work.

WebRTC has the ability to group media tracks into streams using MSIDs, but I haven't yet figured out how to do this. For now, `ontrack` should work.

This _should_ be complete, but it hasn't yet been tested (hence the WIP)

r? @ferjm or @jdm

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/23342)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2019-05-09 00:26:02 -04:00 committed by GitHub
commit 17590fd48f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 390 additions and 47 deletions

View file

@ -105,6 +105,7 @@ use servo_media::audio::panner_node::{DistanceModel, PanningModel};
use servo_media::audio::param::ParamType;
use servo_media::player::Player;
use servo_media::streams::registry::MediaStreamId;
use servo_media::streams::MediaStreamType;
use servo_media::webrtc::WebRtcController;
use servo_url::{ImmutableOrigin, MutableOrigin, ServoUrl};
use smallvec::SmallVec;
@ -490,7 +491,7 @@ unsafe_no_jsmanaged_fields!(NodeId);
unsafe_no_jsmanaged_fields!(AnalysisEngine, DistanceModel, PanningModel, ParamType);
unsafe_no_jsmanaged_fields!(dyn Player);
unsafe_no_jsmanaged_fields!(WebRtcController);
unsafe_no_jsmanaged_fields!(MediaStreamId);
unsafe_no_jsmanaged_fields!(MediaStreamId, MediaStreamType);
unsafe_no_jsmanaged_fields!(Mutex<MediaFrameRenderer>);
unsafe_no_jsmanaged_fields!(RenderApiSender);
unsafe_no_jsmanaged_fields!(ResourceFetchTiming);

View file

@ -839,9 +839,13 @@ impl HTMLMediaElement {
self.fetch_request(None);
},
SrcObject::MediaStream(ref stream) => {
for stream in stream.get_tracks() {
if let Err(_) =
self.player.borrow().as_ref().unwrap().set_stream(&stream)
for stream in &*stream.get_tracks() {
if let Err(_) = self
.player
.borrow()
.as_ref()
.unwrap()
.set_stream(&stream.id())
{
self.queue_dedicated_media_source_failure_steps();
}

View file

@ -14,9 +14,11 @@ use crate::dom::bindings::root::DomRoot;
use crate::dom::eventtarget::EventTarget;
use crate::dom::globalscope::GlobalScope;
use crate::dom::mediastream::MediaStream;
use crate::dom::mediastreamtrack::MediaStreamTrack;
use crate::dom::promise::Promise;
use dom_struct::dom_struct;
use servo_media::streams::capture::{Constrain, ConstrainRange, MediaTrackConstraintSet};
use servo_media::streams::MediaStreamType;
use servo_media::ServoMedia;
use std::rc::Rc;
@ -51,18 +53,20 @@ impl MediaDevicesMethods for MediaDevices {
InCompartment::Already(&in_compartment_proof),
);
let media = ServoMedia::get().unwrap();
let mut tracks = vec![];
let stream = MediaStream::new(&self.global());
if let Some(constraints) = convert_constraints(&constraints.audio) {
if let Some(audio) = media.create_audioinput_stream(constraints) {
tracks.push(audio)
let track = MediaStreamTrack::new(&self.global(), audio, MediaStreamType::Audio);
stream.add_track(&track);
}
}
if let Some(constraints) = convert_constraints(&constraints.video) {
if let Some(video) = media.create_videoinput_stream(constraints) {
tracks.push(video)
let track = MediaStreamTrack::new(&self.global(), video, MediaStreamType::Video);
stream.add_track(&track);
}
}
let stream = MediaStream::new(&self.global(), tracks);
p.resolve_native(&stream);
p
}

View file

@ -3,38 +3,131 @@
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::dom::bindings::cell::DomRefCell;
use crate::dom::bindings::codegen::Bindings::MediaStreamBinding;
use crate::dom::bindings::reflector::reflect_dom_object;
use crate::dom::bindings::root::DomRoot;
use crate::dom::bindings::codegen::Bindings::MediaStreamBinding::{self, MediaStreamMethods};
use crate::dom::bindings::error::Fallible;
use crate::dom::bindings::reflector::{reflect_dom_object, DomObject};
use crate::dom::bindings::root::{Dom, DomRoot};
use crate::dom::bindings::str::DOMString;
use crate::dom::eventtarget::EventTarget;
use crate::dom::globalscope::GlobalScope;
use crate::dom::mediastreamtrack::MediaStreamTrack;
use crate::dom::window::Window;
use dom_struct::dom_struct;
use servo_media::streams::registry::MediaStreamId;
use servo_media::streams::MediaStreamType;
use std::cell::Ref;
#[dom_struct]
pub struct MediaStream {
eventtarget: EventTarget,
#[ignore_malloc_size_of = "defined in servo-media"]
tracks: DomRefCell<Vec<MediaStreamId>>,
tracks: DomRefCell<Vec<Dom<MediaStreamTrack>>>,
}
impl MediaStream {
pub fn new_inherited(tracks: Vec<MediaStreamId>) -> MediaStream {
pub fn new_inherited() -> MediaStream {
MediaStream {
eventtarget: EventTarget::new_inherited(),
tracks: DomRefCell::new(tracks),
tracks: DomRefCell::new(vec![]),
}
}
pub fn new(global: &GlobalScope, tracks: Vec<MediaStreamId>) -> DomRoot<MediaStream> {
pub fn new(global: &GlobalScope) -> DomRoot<MediaStream> {
reflect_dom_object(
Box::new(MediaStream::new_inherited(tracks)),
Box::new(MediaStream::new_inherited()),
global,
MediaStreamBinding::Wrap,
)
}
pub fn get_tracks(&self) -> Vec<MediaStreamId> {
self.tracks.borrow_mut().clone()
pub fn Constructor(global: &Window) -> Fallible<DomRoot<MediaStream>> {
Ok(MediaStream::new(&global.global()))
}
pub fn Constructor_(_: &Window, stream: &MediaStream) -> Fallible<DomRoot<MediaStream>> {
Ok(stream.Clone())
}
pub fn Constructor__(
global: &Window,
tracks: Vec<DomRoot<MediaStreamTrack>>,
) -> Fallible<DomRoot<MediaStream>> {
let new = MediaStream::new(&global.global());
for track in tracks {
// this is quadratic, but shouldn't matter much
// if this becomes a problem we can use a hash map
new.AddTrack(&track)
}
Ok(new)
}
pub fn get_tracks(&self) -> Ref<[Dom<MediaStreamTrack>]> {
Ref::map(self.tracks.borrow(), |tracks| &**tracks)
}
pub fn add_track(&self, track: &MediaStreamTrack) {
self.tracks.borrow_mut().push(Dom::from_ref(track))
}
}
impl MediaStreamMethods for MediaStream {
/// https://w3c.github.io/mediacapture-main/#dom-mediastream-gettracks
fn GetTracks(&self) -> Vec<DomRoot<MediaStreamTrack>> {
self.tracks
.borrow()
.iter()
.map(|x| DomRoot::from_ref(&**x))
.collect()
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastream-getaudiotracks
fn GetAudioTracks(&self) -> Vec<DomRoot<MediaStreamTrack>> {
self.tracks
.borrow()
.iter()
.filter(|x| x.ty() == MediaStreamType::Audio)
.map(|x| DomRoot::from_ref(&**x))
.collect()
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastream-getvideotracks
fn GetVideoTracks(&self) -> Vec<DomRoot<MediaStreamTrack>> {
self.tracks
.borrow()
.iter()
.filter(|x| x.ty() == MediaStreamType::Video)
.map(|x| DomRoot::from_ref(&**x))
.collect()
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastream-gettrackbyid
fn GetTrackById(&self, id: DOMString) -> Option<DomRoot<MediaStreamTrack>> {
self.tracks
.borrow()
.iter()
.find(|x| x.id().id().to_string() == &*id)
.map(|x| DomRoot::from_ref(&**x))
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastream-addtrack
fn AddTrack(&self, track: &MediaStreamTrack) {
let existing = self.tracks.borrow().iter().find(|x| *x == &track).is_some();
if existing {
return;
}
self.add_track(track)
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastream-removetrack
fn RemoveTrack(&self, track: &MediaStreamTrack) {
self.tracks.borrow_mut().retain(|x| *x != track);
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastream-clone
fn Clone(&self) -> DomRoot<MediaStream> {
let new = MediaStream::new(&self.global());
for track in &*self.tracks.borrow() {
new.add_track(&track)
}
new
}
}

View file

@ -0,0 +1,74 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::dom::bindings::codegen::Bindings::MediaStreamTrackBinding::{
self, MediaStreamTrackMethods,
};
use crate::dom::bindings::reflector::{reflect_dom_object, DomObject};
use crate::dom::bindings::root::DomRoot;
use crate::dom::bindings::str::DOMString;
use crate::dom::eventtarget::EventTarget;
use crate::dom::globalscope::GlobalScope;
use dom_struct::dom_struct;
use servo_media::streams::registry::MediaStreamId;
use servo_media::streams::MediaStreamType;
#[dom_struct]
pub struct MediaStreamTrack {
eventtarget: EventTarget,
#[ignore_malloc_size_of = "defined in servo-media"]
id: MediaStreamId,
#[ignore_malloc_size_of = "defined in servo-media"]
ty: MediaStreamType,
}
impl MediaStreamTrack {
pub fn new_inherited(id: MediaStreamId, ty: MediaStreamType) -> MediaStreamTrack {
MediaStreamTrack {
eventtarget: EventTarget::new_inherited(),
id,
ty,
}
}
pub fn new(
global: &GlobalScope,
id: MediaStreamId,
ty: MediaStreamType,
) -> DomRoot<MediaStreamTrack> {
reflect_dom_object(
Box::new(MediaStreamTrack::new_inherited(id, ty)),
global,
MediaStreamTrackBinding::Wrap,
)
}
pub fn id(&self) -> MediaStreamId {
self.id
}
pub fn ty(&self) -> MediaStreamType {
self.ty
}
}
impl MediaStreamTrackMethods for MediaStreamTrack {
/// https://w3c.github.io/mediacapture-main/#dom-mediastreamtrack-kind
fn Kind(&self) -> DOMString {
match self.ty {
MediaStreamType::Video => "video".into(),
MediaStreamType::Audio => "audio".into(),
}
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastreamtrack-id
fn Id(&self) -> DOMString {
self.id.id().to_string().into()
}
/// https://w3c.github.io/mediacapture-main/#dom-mediastreamtrack-clone
fn Clone(&self) -> DomRoot<MediaStreamTrack> {
MediaStreamTrack::new(&self.global(), self.id, self.ty)
}
}

View file

@ -399,6 +399,7 @@ pub mod medialist;
pub mod mediaquerylist;
pub mod mediaquerylistevent;
pub mod mediastream;
pub mod mediastreamtrack;
pub mod messageevent;
pub mod mimetype;
pub mod mimetypearray;
@ -450,6 +451,7 @@ pub mod rtcicecandidate;
pub mod rtcpeerconnection;
pub mod rtcpeerconnectioniceevent;
pub mod rtcsessiondescription;
pub mod rtctrackevent;
pub mod screen;
pub mod serviceworker;
pub mod serviceworkercontainer;

View file

@ -26,10 +26,12 @@ use crate::dom::event::{Event, EventBubbles, EventCancelable};
use crate::dom::eventtarget::EventTarget;
use crate::dom::globalscope::GlobalScope;
use crate::dom::mediastream::MediaStream;
use crate::dom::mediastreamtrack::MediaStreamTrack;
use crate::dom::promise::Promise;
use crate::dom::rtcicecandidate::RTCIceCandidate;
use crate::dom::rtcpeerconnectioniceevent::RTCPeerConnectionIceEvent;
use crate::dom::rtcsessiondescription::RTCSessionDescription;
use crate::dom::rtctrackevent::RTCTrackEvent;
use crate::dom::window::Window;
use crate::task::TaskCanceller;
use crate::task_source::networking::NetworkingTaskSource;
@ -37,6 +39,7 @@ use crate::task_source::TaskSource;
use dom_struct::dom_struct;
use servo_media::streams::registry::MediaStreamId;
use servo_media::streams::MediaStreamType;
use servo_media::webrtc::{
BundlePolicy, GatheringState, IceCandidate, IceConnectionState, SdpType, SessionDescription,
SignalingState, WebRtcController, WebRtcSignaller,
@ -128,7 +131,17 @@ impl WebRtcSignaller for RTCSignaller {
);
}
fn on_add_stream(&self, _: &MediaStreamId) {}
fn on_add_stream(&self, id: &MediaStreamId, ty: MediaStreamType) {
let this = self.trusted.clone();
let id = *id;
let _ = self.task_source.queue_with_canceller(
task!(on_add_stream: move || {
let this = this.root();
this.on_add_stream(id, ty);
}),
&self.canceller,
);
}
fn close(&self) {
// do nothing
@ -238,6 +251,15 @@ impl RTCPeerConnection {
event.upcast::<Event>().fire(self.upcast());
}
fn on_add_stream(&self, id: MediaStreamId, ty: MediaStreamType) {
if self.closed.get() {
return;
}
let track = MediaStreamTrack::new(&self.global(), id, ty);
let event = RTCTrackEvent::new(&self.global(), atom!("track"), false, false, &track);
event.upcast::<Event>().fire(self.upcast());
}
/// https://www.w3.org/TR/webrtc/#update-ice-gathering-state
fn update_gathering_state(&self, state: GatheringState) {
// step 1
@ -399,6 +421,9 @@ impl RTCPeerConnectionMethods for RTCPeerConnection {
/// https://w3c.github.io/webrtc-pc/#dom-rtcpeerconnection-icecandidate
event_handler!(icecandidate, GetOnicecandidate, SetOnicecandidate);
/// https://www.w3.org/TR/webrtc/#dom-rtcpeerconnection-ontrack
event_handler!(track, GetOntrack, SetOntrack);
/// https://w3c.github.io/webrtc-pc/#dom-rtcpeerconnection-iceconnectionstatechange
event_handler!(
iceconnectionstatechange,
@ -584,10 +609,12 @@ impl RTCPeerConnectionMethods for RTCPeerConnection {
// https://w3c.github.io/webrtc-pc/#legacy-interface-extensions
fn AddStream(&self, stream: &MediaStream) {
let mut tracks = stream.get_tracks();
for ref track in tracks.drain(..) {
self.controller.borrow().as_ref().unwrap().add_stream(track);
for track in &*stream.get_tracks() {
self.controller
.borrow()
.as_ref()
.unwrap()
.add_stream(&track.id());
}
}

View file

@ -0,0 +1,78 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use crate::dom::bindings::codegen::Bindings::EventBinding::EventBinding::EventMethods;
use crate::dom::bindings::codegen::Bindings::RTCTrackEventBinding::{self, RTCTrackEventMethods};
use crate::dom::bindings::error::Fallible;
use crate::dom::bindings::inheritance::Castable;
use crate::dom::bindings::reflector::{reflect_dom_object, DomObject};
use crate::dom::bindings::root::{Dom, DomRoot};
use crate::dom::bindings::str::DOMString;
use crate::dom::event::Event;
use crate::dom::globalscope::GlobalScope;
use crate::dom::mediastreamtrack::MediaStreamTrack;
use crate::dom::window::Window;
use dom_struct::dom_struct;
use servo_atoms::Atom;
#[dom_struct]
pub struct RTCTrackEvent {
event: Event,
track: Dom<MediaStreamTrack>,
}
impl RTCTrackEvent {
#[allow(unrooted_must_root)]
fn new_inherited(track: &MediaStreamTrack) -> RTCTrackEvent {
RTCTrackEvent {
event: Event::new_inherited(),
track: Dom::from_ref(track),
}
}
pub fn new(
global: &GlobalScope,
type_: Atom,
bubbles: bool,
cancelable: bool,
track: &MediaStreamTrack,
) -> DomRoot<RTCTrackEvent> {
let trackevent = reflect_dom_object(
Box::new(RTCTrackEvent::new_inherited(&track)),
global,
RTCTrackEventBinding::Wrap,
);
{
let event = trackevent.upcast::<Event>();
event.init_event(type_, bubbles, cancelable);
}
trackevent
}
pub fn Constructor(
window: &Window,
type_: DOMString,
init: &RTCTrackEventBinding::RTCTrackEventInit,
) -> Fallible<DomRoot<RTCTrackEvent>> {
Ok(RTCTrackEvent::new(
&window.global(),
Atom::from(type_),
init.parent.bubbles,
init.parent.cancelable,
&init.track,
))
}
}
impl RTCTrackEventMethods for RTCTrackEvent {
// https://w3c.github.io/webrtc-pc/#dom-rtctrackevent-track
fn Track(&self) -> DomRoot<MediaStreamTrack> {
DomRoot::from_ref(&*self.track)
}
// https://dom.spec.whatwg.org/#dom-event-istrusted
fn IsTrusted(&self) -> bool {
self.event.IsTrusted()
}
}

View file

@ -4,20 +4,20 @@
// https://w3c.github.io/mediacapture-main/#dom-mediastream
// [Exposed=Window,
// Constructor,
// Constructor(MediaStream stream),
// Constructor(sequence<MediaStreamTrack> tracks)]
[Exposed=Window, Pref="dom.webrtc.enabled"]
[Exposed=Window,
Constructor,
Constructor(MediaStream stream),
Constructor(sequence<MediaStreamTrack> tracks),
Pref="dom.webrtc.enabled"]
interface MediaStream : EventTarget {
// readonly attribute DOMString id;
// sequence<MediaStreamTrack> getAudioTracks();
// sequence<MediaStreamTrack> getVideoTracks();
// sequence<MediaStreamTrack> getTracks();
// MediaStreamTrack? getTrackById(DOMString trackId);
// void addTrack(MediaStreamTrack track);
// void removeTrack(MediaStreamTrack track);
// MediaStream clone();
sequence<MediaStreamTrack> getAudioTracks();
sequence<MediaStreamTrack> getVideoTracks();
sequence<MediaStreamTrack> getTracks();
MediaStreamTrack? getTrackById(DOMString trackId);
void addTrack(MediaStreamTrack track);
void removeTrack(MediaStreamTrack track);
MediaStream clone();
// readonly attribute boolean active;
// attribute EventHandler onaddtrack;
// attribute EventHandler onremovetrack;

View file

@ -0,0 +1,24 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
// https://w3c.github.io/mediacapture-main/#dom-mediastreamtrack
[Exposed=Window, Pref="dom.webrtc.enabled"]
interface MediaStreamTrack : EventTarget {
readonly attribute DOMString kind;
readonly attribute DOMString id;
// readonly attribute DOMString label;
// attribute boolean enabled;
// readonly attribute boolean muted;
// attribute EventHandler onmute;
// attribute EventHandler onunmute;
// readonly attribute MediaStreamTrackState readyState;
// attribute EventHandler onended;
MediaStreamTrack clone();
// void stop();
// MediaTrackCapabilities getCapabilities();
// MediaTrackConstraints getConstraints();
// MediaTrackSettings getSettings();
// Promise<void> applyConstraints(optional MediaTrackConstraints constraints);
};

View file

@ -114,3 +114,15 @@ enum RTCSignalingState {
"have-remote-pranswer",
"closed"
};
partial interface RTCPeerConnection {
// sequence<RTCRtpSender> getSenders();
// sequence<RTCRtpReceiver> getReceivers();
// sequence<RTCRtpTransceiver> getTransceivers();
// RTCRtpSender addTrack(MediaStreamTrack track,
// MediaStream... streams);
// void removeTrack(RTCRtpSender sender);
// RTCRtpTransceiver addTransceiver((MediaStreamTrack or DOMString) trackOrKind,
// optional RTCRtpTransceiverInit init);
attribute EventHandler ontrack;
};

View file

@ -0,0 +1,23 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
// https://w3c.github.io/webrtc-pc/#dom-rtctrackevent
[Constructor(DOMString type, RTCTrackEventInit eventInitDict),
Exposed=Window, Pref="dom.webrtc.enabled"]
interface RTCTrackEvent : Event {
// readonly attribute RTCRtpReceiver receiver;
readonly attribute MediaStreamTrack track;
// [SameObject]
// readonly attribute FrozenArray<MediaStream> streams;
// readonly attribute RTCRtpTransceiver transceiver;
};
// https://www.w3.org/TR/webrtc/#dom-rtctrackeventinit
dictionary RTCTrackEventInit : EventInit {
// required RTCRtpReceiver receiver;
required MediaStreamTrack track;
// sequence<MediaStream> streams = [];
// required RTCRtpTransceiver transceiver;
};