Add media (WindowGLContext) module in canvas_trait

This module adds a structure (WindowGLContext) which holds the
OpenGL parameters that are going to be used by servo-media player
to render video frames using OpenGL.

In order to fill this structure, three new methods were added to
WindowMethods trait. In this patch only the Glutin-based
implementation provides a simple boilerplate.

The WindowGLContext is created in the entry point of libservo, when
the application window is created, and later passed to the
constellation, the pipeline and to the window element in dom, thus
htmlmediaelement has a mean to obtain these parameters via its
window.
This commit is contained in:
Víctor Manuel Jáquez Leal 2019-06-18 17:45:56 +02:00 committed by Fernando Jiménez Moreno
parent e9f46f9d72
commit 9f4f9dc750
16 changed files with 133 additions and 26 deletions

23
Cargo.lock generated
View file

@ -426,6 +426,7 @@ dependencies = [
"pixels 0.0.1",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
"servo-media 0.1.0 (git+https://github.com/servo/media)",
"servo_config 0.0.1",
"typetag 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"webrender_api 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)",
@ -610,6 +611,7 @@ dependencies = [
"pixels 0.0.1",
"profile_traits 0.0.1",
"script_traits 0.0.1",
"servo-media 0.1.0 (git+https://github.com/servo/media)",
"servo_geometry 0.0.1",
"servo_url 0.0.1",
"style_traits 0.0.1",
@ -4033,6 +4035,7 @@ dependencies = [
"osmesa-src 0.1.0 (git+https://github.com/servo/osmesa-src)",
"osmesa-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rust-webvr 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"servo-media 0.1.0 (git+https://github.com/servo/media)",
"sig 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tinyfiledialogs 3.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"webxr 0.0.1 (git+https://github.com/servo/webxr)",
@ -4080,7 +4083,7 @@ dependencies = [
[[package]]
name = "servo-media"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"servo-media-audio 0.1.0 (git+https://github.com/servo/media)",
"servo-media-player 0.1.0 (git+https://github.com/servo/media)",
@ -4091,7 +4094,7 @@ dependencies = [
[[package]]
name = "servo-media-audio"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4107,7 +4110,7 @@ dependencies = [
[[package]]
name = "servo-media-dummy"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"ipc-channel 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4121,7 +4124,7 @@ dependencies = [
[[package]]
name = "servo-media-gstreamer"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4155,7 +4158,7 @@ dependencies = [
[[package]]
name = "servo-media-gstreamer-render"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"gstreamer 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer-video 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4165,7 +4168,7 @@ dependencies = [
[[package]]
name = "servo-media-gstreamer-render-unix"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"glib 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gstreamer 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4178,7 +4181,7 @@ dependencies = [
[[package]]
name = "servo-media-player"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"ipc-channel 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4189,7 +4192,7 @@ dependencies = [
[[package]]
name = "servo-media-streams"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4198,7 +4201,7 @@ dependencies = [
[[package]]
name = "servo-media-webrtc"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -4297,7 +4300,7 @@ dependencies = [
[[package]]
name = "servo_media_derive"
version = "0.1.0"
source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4"
source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5"
dependencies = [
"proc-macro2 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",

View file

@ -25,6 +25,7 @@ pixels = {path = "../pixels"}
serde = "1.0"
serde_bytes = "0.10"
servo_config = {path = "../config"}
servo-media = {git = "https://github.com/servo/media"}
typetag = "0.1"
webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]}
webvr_traits = {path = "../webvr_traits"}

View file

@ -14,6 +14,7 @@ extern crate malloc_size_of_derive;
extern crate serde;
pub mod canvas;
pub mod media;
#[macro_use]
pub mod webgl;
mod webgl_channel;

View file

@ -0,0 +1,29 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */
use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext};
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct WindowGLContext {
/// Application's GL Context
pub gl_context: GlContext,
/// Application's GL Api
pub gl_api: GlApi,
/// Application's native display
pub native_display: NativeDisplay,
}
impl PlayerGLContext for WindowGLContext {
fn get_gl_context(&self) -> GlContext {
self.gl_context.clone()
}
fn get_native_display(&self) -> NativeDisplay {
self.native_display.clone()
}
fn get_gl_api(&self) -> GlApi {
self.gl_api.clone()
}
}

View file

@ -33,6 +33,7 @@ pixels = {path = "../pixels", optional = true}
profile_traits = {path = "../profile_traits"}
script_traits = {path = "../script_traits"}
servo_geometry = {path = "../geometry"}
servo-media = {git = "https://github.com/servo/media"}
servo_url = {path = "../url"}
style_traits = {path = "../style_traits"}
time = "0.1.17"

View file

@ -12,6 +12,7 @@ use keyboard_types::KeyboardEvent;
use msg::constellation_msg::{PipelineId, TopLevelBrowsingContextId, TraversalDirection};
use script_traits::{MouseButton, TouchEventType, TouchId, WheelDelta};
use servo_geometry::DeviceIndependentPixel;
use servo_media::player::context::{GlApi, GlContext, NativeDisplay};
use servo_url::ServoUrl;
use std::fmt::{Debug, Error, Formatter};
#[cfg(feature = "gl")]
@ -157,6 +158,12 @@ pub trait WindowMethods {
/// will want to avoid blocking on UI events, and just
/// run the event loop at the vsync interval.
fn set_animation_state(&self, _state: AnimationState);
/// Get the GL context
fn get_gl_context(&self) -> GlContext;
/// Get the native display
fn get_native_display(&self) -> NativeDisplay;
/// Get the GL api
fn get_gl_api(&self) -> GlApi;
}
pub trait EmbedderMethods {

View file

@ -107,6 +107,7 @@ use canvas::canvas_paint_thread::CanvasPaintThread;
use canvas::webgl_thread::WebGLThreads;
use canvas_traits::canvas::CanvasId;
use canvas_traits::canvas::CanvasMsg;
use canvas_traits::media::WindowGLContext;
use compositing::compositor_thread::CompositorProxy;
use compositing::compositor_thread::Msg as ToCompositorMsg;
use compositing::SendableFrameTree;
@ -409,6 +410,9 @@ pub struct Constellation<Message, LTF, STF> {
/// Like --disable-text-aa, this is useful for reftests where pixel perfect
/// results are required.
enable_canvas_antialiasing: bool,
/// Application window's GL Context for Media player
player_context: WindowGLContext,
}
/// State needed to construct a constellation.
@ -457,6 +461,9 @@ pub struct InitialConstellationState {
/// The XR device registry
pub webxr_registry: webxr_api::Registry,
/// Application window's GL Context for Media player
pub player_context: WindowGLContext,
}
/// Data needed for webdriver
@ -753,6 +760,7 @@ where
is_running_problem_test,
hard_fail,
enable_canvas_antialiasing,
player_context: state.player_context,
};
constellation.run();
@ -994,6 +1002,7 @@ where
.map(|threads| threads.pipeline()),
webvr_chan: self.webvr_chan.clone(),
webxr_registry: self.webxr_registry.clone(),
player_context: self.player_context.clone(),
});
let pipeline = match result {

View file

@ -5,6 +5,7 @@
use crate::event_loop::EventLoop;
use background_hang_monitor::HangMonitorRegister;
use bluetooth_traits::BluetoothRequest;
use canvas_traits::media::WindowGLContext;
use canvas_traits::webgl::WebGLPipeline;
use compositing::compositor_thread::Msg as CompositorMsg;
use compositing::CompositionPipeline;
@ -191,6 +192,9 @@ pub struct InitialPipelineState {
/// The XR device registry
pub webxr_registry: webxr_api::Registry,
/// Application window's GL Context for Media player
pub player_context: WindowGLContext,
}
pub struct NewPipeline {
@ -309,6 +313,7 @@ impl Pipeline {
webgl_chan: state.webgl_chan,
webvr_chan: state.webvr_chan,
webxr_registry: state.webxr_registry,
player_context: state.player_context,
};
// Spawn the child process.
@ -515,6 +520,7 @@ pub struct UnprivilegedPipelineContent {
webgl_chan: Option<WebGLPipeline>,
webvr_chan: Option<IpcSender<WebVRMsg>>,
webxr_registry: webxr_api::Registry,
player_context: WindowGLContext,
}
impl UnprivilegedPipelineContent {
@ -563,6 +569,7 @@ impl UnprivilegedPipelineContent {
webrender_document: self.webrender_document,
webrender_api_sender: self.webrender_api_sender.clone(),
layout_is_busy: layout_thread_busy_flag.clone(),
player_context: self.player_context.clone(),
},
self.load_data.clone(),
self.opts.profile_script_events,

View file

@ -71,7 +71,6 @@ use net_traits::{CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseLis
use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType};
use script_layout_interface::HTMLMediaData;
use servo_config::pref;
use servo_media::player::context::{GlContext, NativeDisplay, PlayerGLContext};
use servo_media::player::frame::{Frame, FrameRenderer};
use servo_media::player::{PlaybackState, Player, PlayerError, PlayerEvent, StreamType};
use servo_media::{ServoMedia, SupportsMediaType};
@ -162,16 +161,6 @@ impl FrameRenderer for MediaFrameRenderer {
}
}
struct PlayerContextDummy();
impl PlayerGLContext for PlayerContextDummy {
fn get_gl_context(&self) -> GlContext {
return GlContext::Unknown;
}
fn get_native_display(&self) -> NativeDisplay {
return NativeDisplay::Unknown;
}
}
#[must_root]
#[derive(JSTraceable, MallocSizeOf)]
enum SrcObject {
@ -1222,22 +1211,23 @@ impl HTMLMediaElement {
_ => StreamType::Seekable,
};
let window = window_from_node(self);
let (action_sender, action_receiver) = ipc::channel().unwrap();
let renderer: Option<Arc<Mutex<dyn FrameRenderer>>> = match self.media_type_id() {
HTMLMediaElementTypeId::HTMLAudioElement => None,
HTMLMediaElementTypeId::HTMLVideoElement => Some(self.frame_renderer.clone()),
};
let player = ServoMedia::get().unwrap().create_player(
stream_type,
action_sender,
renderer,
Box::new(PlayerContextDummy()),
Box::new(window.get_player_context()),
);
*self.player.borrow_mut() = Some(player);
let trusted_node = Trusted::new(self);
let window = window_from_node(self);
let (task_source, canceller) = window
.task_manager()
.media_element_task_source_with_canceller();

View file

@ -68,6 +68,7 @@ use crate::webdriver_handlers::jsval_to_webdriver;
use app_units::Au;
use base64;
use bluetooth_traits::BluetoothRequest;
use canvas_traits::media::WindowGLContext;
use canvas_traits::webgl::WebGLChan;
use crossbeam_channel::{unbounded, Sender, TryRecvError};
use cssparser::{Parser, ParserInput, SourceLocation};
@ -318,6 +319,10 @@ pub struct Window {
/// Replace unpaired surrogates in DOM strings with U+FFFD.
/// See <https://github.com/servo/servo/issues/6564>
replace_surrogates: bool,
/// Window's GL context from application
#[ignore_malloc_size_of = "defined in script_thread"]
player_context: WindowGLContext,
}
impl Window {
@ -481,6 +486,10 @@ impl Window {
pub fn unminify_js(&self) -> bool {
self.unminify_js
}
pub fn get_player_context(&self) -> WindowGLContext {
self.player_context.clone()
}
}
// https://html.spec.whatwg.org/multipage/#atob
@ -2074,6 +2083,7 @@ impl Window {
is_headless: bool,
replace_surrogates: bool,
user_agent: Cow<'static, str>,
player_context: WindowGLContext,
) -> DomRoot<Self> {
let layout_rpc: Box<dyn LayoutRPC + Send> = {
let (rpc_send, rpc_recv) = unbounded();
@ -2154,6 +2164,7 @@ impl Window {
unminify_js,
userscripts_path,
replace_surrogates,
player_context,
});
unsafe { WindowBinding::Wrap(runtime.cx(), win) }

View file

@ -86,6 +86,7 @@ use crate::task_source::websocket::WebsocketTaskSource;
use crate::task_source::TaskSourceName;
use crate::webdriver_handlers;
use bluetooth_traits::BluetoothRequest;
use canvas_traits::media::WindowGLContext;
use canvas_traits::webgl::WebGLPipeline;
use crossbeam_channel::{unbounded, Receiver, Sender};
use devtools_traits::CSSError;
@ -512,6 +513,8 @@ unsafe_no_jsmanaged_fields!(TaskQueue<MainThreadScriptMsg>);
unsafe_no_jsmanaged_fields!(dyn BackgroundHangMonitorRegister);
unsafe_no_jsmanaged_fields!(dyn BackgroundHangMonitor);
unsafe_no_jsmanaged_fields!(WindowGLContext);
#[derive(JSTraceable)]
// ScriptThread instances are rooted on creation, so this is okay
#[allow(unrooted_must_root)]
@ -646,7 +649,7 @@ pub struct ScriptThread {
/// The Webrender Document ID associated with this thread.
webrender_document: DocumentId,
/// FIXME(victor):
/// Webrender API sender.
webrender_api_sender: RenderApiSender,
/// Periodically print out on which events script threads spend their processing time.
@ -678,6 +681,9 @@ pub struct ScriptThread {
/// An optional string allowing the user agent to be set for testing.
user_agent: Cow<'static, str>,
/// Application window's GL Context for Media player
player_context: WindowGLContext,
}
/// In the event of thread panic, all data on the stack runs its destructor. However, there
@ -1239,6 +1245,7 @@ impl ScriptThread {
headless,
replace_surrogates,
user_agent,
player_context: state.player_context,
}
}
@ -2981,6 +2988,7 @@ impl ScriptThread {
self.headless,
self.replace_surrogates,
self.user_agent.clone(),
self.player_context.clone(),
);
// Initialize the browsing context for the window.

View file

@ -21,6 +21,7 @@ pub mod webdriver_msg;
use crate::webdriver_msg::{LoadStatus, WebDriverScriptCommand};
use bluetooth_traits::BluetoothRequest;
use canvas_traits::media::WindowGLContext;
use canvas_traits::webgl::WebGLPipeline;
use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
use devtools_traits::{DevtoolScriptControlMsg, ScriptToDevtoolsControlMsg, WorkerId};
@ -636,6 +637,8 @@ pub struct InitialScriptState {
pub webrender_api_sender: RenderApiSender,
/// Flag to indicate if the layout thread is busy handling a request.
pub layout_is_busy: Arc<AtomicBool>,
/// Application window's GL Context for Media player
pub player_context: WindowGLContext,
}
/// This trait allows creating a `ScriptThread` without depending on the `script`

View file

@ -64,6 +64,7 @@ use bluetooth::BluetoothThreadFactory;
use bluetooth_traits::BluetoothRequest;
use canvas::gl_context::GLContextFactory;
use canvas::webgl_thread::WebGLThreads;
use canvas_traits::media::WindowGLContext;
use compositing::compositor_thread::{
CompositorProxy, CompositorReceiver, InitialCompositorState, Msg,
};
@ -303,6 +304,12 @@ where
None
};
let player_context = WindowGLContext {
gl_context: window.get_gl_context(),
native_display: window.get_native_display(),
gl_api: window.get_gl_api(),
};
// Create the constellation, which maintains the engine
// pipelines, including the script and layout threads, as well
// as the navigation context.
@ -321,6 +328,7 @@ where
window.gl(),
webvr_services,
webxr_registry,
player_context,
);
// Send the constellation's swmanager sender to service worker manager thread
@ -631,6 +639,7 @@ fn create_constellation(
window_gl: Rc<dyn gl::Gl>,
webvr_services: Option<VRServiceManager>,
webxr_registry: webxr_api::Registry,
player_context: WindowGLContext,
) -> (Sender<ConstellationMsg>, SWManagerSenders) {
// Global configuration options, parsed from the command line.
let opts = opts::get();
@ -712,6 +721,7 @@ fn create_constellation(
webgl_threads,
webvr_chan,
webxr_registry,
player_context,
};
let (constellation_chan, from_swmanager_sender) = Constellation::<
script_layout_interface::message::Msg,

View file

@ -44,6 +44,7 @@ webrender_debugger = ["libservo/webrender_debugger"]
[target.'cfg(not(target_os = "android"))'.dependencies]
backtrace = "0.3"
bitflags = "1.0"
clipboard = "0.5"
crossbeam-channel = "0.3"
euclid = "0.19"
gleam = "0.6"
@ -54,9 +55,9 @@ libservo = {path = "../../components/servo"}
libc = "0.2"
log = "0.4"
rust-webvr = { version = "0.13", features = ["glwindow"] }
webxr = { git = "https://github.com/servo/webxr", features = ["glwindow"] }
servo-media = {git = "https://github.com/servo/media"}
tinyfiledialogs = "3.0"
clipboard = "0.5"
webxr = { git = "https://github.com/servo/webxr", features = ["glwindow"] }
[target.'cfg(any(target_os = "linux", target_os = "windows"))'.dependencies]
image = "0.21"

View file

@ -30,6 +30,7 @@ use servo::style_traits::DevicePixel;
use servo::webrender_api::{
DeviceIntPoint, DeviceIntRect, DeviceIntSize, FramebufferIntSize, ScrollLocation,
};
use servo_media::player::context as MediaPlayerCtxt;
use std::cell::{Cell, RefCell};
use std::mem;
use std::rc::Rc;
@ -524,6 +525,18 @@ impl WindowMethods for Window {
fn prepare_for_composite(&self) {
self.gl_context.borrow_mut().make_current();
}
fn get_gl_context(&self) -> MediaPlayerCtxt::GlContext {
MediaPlayerCtxt::GlContext::Unknown
}
fn get_native_display(&self) -> MediaPlayerCtxt::NativeDisplay {
MediaPlayerCtxt::NativeDisplay::Unknown
}
fn get_gl_api(&self) -> MediaPlayerCtxt::GlApi {
MediaPlayerCtxt::GlApi::None
}
}
fn winit_phase_to_touch_event_type(phase: TouchPhase) -> TouchEventType {

View file

@ -14,6 +14,7 @@ use servo::servo_config::opts;
use servo::servo_geometry::DeviceIndependentPixel;
use servo::style_traits::DevicePixel;
use servo::webrender_api::{DeviceIntRect, FramebufferIntSize};
use servo_media::player::context as MediaPlayerCtxt;
use std::cell::Cell;
#[cfg(any(target_os = "linux", target_os = "macos"))]
use std::ffi::CString;
@ -195,4 +196,16 @@ impl WindowMethods for Window {
}
fn prepare_for_composite(&self) { }
fn get_gl_context(&self) -> MediaPlayerCtxt::GlContext {
MediaPlayerCtxt::GlContext::Unknown
}
fn get_native_display(&self) -> MediaPlayerCtxt::NativeDisplay {
MediaPlayerCtxt::NativeDisplay::Unknown
}
fn get_gl_api(&self) -> MediaPlayerCtxt::GlApi {
MediaPlayerCtxt::GlApi::None
}
}