From e9f46f9d7249999279af60b8b78479f6525757bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 10 May 2019 18:51:20 +0200 Subject: [PATCH 01/31] Fix code style --- components/script/dom/window.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/components/script/dom/window.rs b/components/script/dom/window.rs index 596fd711599..7d5f0fdedbf 100644 --- a/components/script/dom/window.rs +++ b/components/script/dom/window.rs @@ -288,6 +288,7 @@ pub struct Window { /// Flag to identify whether mutation observers are present(true)/absent(false) exists_mut_observer: Cell, + /// Webrender API Sender #[ignore_malloc_size_of = "defined in webrender_api"] webrender_api_sender: RenderApiSender, From 9f4f9dc750ea98e70dbdab8b2a0e5ef6c98dd889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Tue, 18 Jun 2019 17:45:56 +0200 Subject: [PATCH 02/31] Add media (WindowGLContext) module in canvas_trait This module adds a structure (WindowGLContext) which holds the OpenGL parameters that are going to be used by servo-media player to render video frames using OpenGL. In order to fill this structure, three new methods were added to WindowMethods trait. In this patch only the Glutin-based implementation provides a simple boilerplate. The WindowGLContext is created in the entry point of libservo, when the application window is created, and later passed to the constellation, the pipeline and to the window element in dom, thus htmlmediaelement has a mean to obtain these parameters via its window. --- Cargo.lock | 23 ++++++++++-------- components/canvas_traits/Cargo.toml | 1 + components/canvas_traits/lib.rs | 1 + components/canvas_traits/media.rs | 29 +++++++++++++++++++++++ components/compositing/Cargo.toml | 1 + components/compositing/windowing.rs | 7 ++++++ components/constellation/constellation.rs | 9 +++++++ components/constellation/pipeline.rs | 7 ++++++ components/script/dom/htmlmediaelement.rs | 16 +++---------- components/script/dom/window.rs | 11 +++++++++ components/script/script_thread.rs | 10 +++++++- components/script_traits/lib.rs | 3 +++ components/servo/lib.rs | 10 ++++++++ ports/glutin/Cargo.toml | 5 ++-- ports/glutin/headed_window.rs | 13 ++++++++++ ports/glutin/headless_window.rs | 13 ++++++++++ 16 files changed, 133 insertions(+), 26 deletions(-) create mode 100644 components/canvas_traits/media.rs diff --git a/Cargo.lock b/Cargo.lock index 181afad5ac1..0432a88cb4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -426,6 +426,7 @@ dependencies = [ "pixels 0.0.1", "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "servo-media 0.1.0 (git+https://github.com/servo/media)", "servo_config 0.0.1", "typetag 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "webrender_api 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", @@ -610,6 +611,7 @@ dependencies = [ "pixels 0.0.1", "profile_traits 0.0.1", "script_traits 0.0.1", + "servo-media 0.1.0 (git+https://github.com/servo/media)", "servo_geometry 0.0.1", "servo_url 0.0.1", "style_traits 0.0.1", @@ -4033,6 +4035,7 @@ dependencies = [ "osmesa-src 0.1.0 (git+https://github.com/servo/osmesa-src)", "osmesa-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "rust-webvr 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "servo-media 0.1.0 (git+https://github.com/servo/media)", "sig 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "tinyfiledialogs 3.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "webxr 0.0.1 (git+https://github.com/servo/webxr)", @@ -4080,7 +4083,7 @@ dependencies = [ [[package]] name = "servo-media" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "servo-media-audio 0.1.0 (git+https://github.com/servo/media)", "servo-media-player 0.1.0 (git+https://github.com/servo/media)", @@ -4091,7 +4094,7 @@ dependencies = [ [[package]] name = "servo-media-audio" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4107,7 +4110,7 @@ dependencies = [ [[package]] name = "servo-media-dummy" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ipc-channel 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4121,7 +4124,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "byte-slice-cast 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4155,7 +4158,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "gstreamer 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer-video 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4165,7 +4168,7 @@ dependencies = [ [[package]] name = "servo-media-gstreamer-render-unix" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "glib 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4178,7 +4181,7 @@ dependencies = [ [[package]] name = "servo-media-player" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "ipc-channel 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4189,7 +4192,7 @@ dependencies = [ [[package]] name = "servo-media-streams" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4198,7 +4201,7 @@ dependencies = [ [[package]] name = "servo-media-webrtc" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "boxfnonce 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4297,7 +4300,7 @@ dependencies = [ [[package]] name = "servo_media_derive" version = "0.1.0" -source = "git+https://github.com/servo/media#2dabf1ab7e3b6d3b6764eebdf8855431367752c4" +source = "git+https://github.com/servo/media#c145e2097b027ca82ac767f4820693868a408bb5" dependencies = [ "proc-macro2 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/components/canvas_traits/Cargo.toml b/components/canvas_traits/Cargo.toml index 52388be8863..fabe074409c 100644 --- a/components/canvas_traits/Cargo.toml +++ b/components/canvas_traits/Cargo.toml @@ -25,6 +25,7 @@ pixels = {path = "../pixels"} serde = "1.0" serde_bytes = "0.10" servo_config = {path = "../config"} +servo-media = {git = "https://github.com/servo/media"} typetag = "0.1" webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]} webvr_traits = {path = "../webvr_traits"} diff --git a/components/canvas_traits/lib.rs b/components/canvas_traits/lib.rs index c19ca073658..2a52409a0e1 100644 --- a/components/canvas_traits/lib.rs +++ b/components/canvas_traits/lib.rs @@ -14,6 +14,7 @@ extern crate malloc_size_of_derive; extern crate serde; pub mod canvas; +pub mod media; #[macro_use] pub mod webgl; mod webgl_channel; diff --git a/components/canvas_traits/media.rs b/components/canvas_traits/media.rs new file mode 100644 index 00000000000..19be3e09a5d --- /dev/null +++ b/components/canvas_traits/media.rs @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct WindowGLContext { + /// Application's GL Context + pub gl_context: GlContext, + /// Application's GL Api + pub gl_api: GlApi, + /// Application's native display + pub native_display: NativeDisplay, +} + +impl PlayerGLContext for WindowGLContext { + fn get_gl_context(&self) -> GlContext { + self.gl_context.clone() + } + + fn get_native_display(&self) -> NativeDisplay { + self.native_display.clone() + } + + fn get_gl_api(&self) -> GlApi { + self.gl_api.clone() + } +} diff --git a/components/compositing/Cargo.toml b/components/compositing/Cargo.toml index 6ac076e3362..2d5fa600376 100644 --- a/components/compositing/Cargo.toml +++ b/components/compositing/Cargo.toml @@ -33,6 +33,7 @@ pixels = {path = "../pixels", optional = true} profile_traits = {path = "../profile_traits"} script_traits = {path = "../script_traits"} servo_geometry = {path = "../geometry"} +servo-media = {git = "https://github.com/servo/media"} servo_url = {path = "../url"} style_traits = {path = "../style_traits"} time = "0.1.17" diff --git a/components/compositing/windowing.rs b/components/compositing/windowing.rs index 9db0286bd5a..0cbb51249c6 100644 --- a/components/compositing/windowing.rs +++ b/components/compositing/windowing.rs @@ -12,6 +12,7 @@ use keyboard_types::KeyboardEvent; use msg::constellation_msg::{PipelineId, TopLevelBrowsingContextId, TraversalDirection}; use script_traits::{MouseButton, TouchEventType, TouchId, WheelDelta}; use servo_geometry::DeviceIndependentPixel; +use servo_media::player::context::{GlApi, GlContext, NativeDisplay}; use servo_url::ServoUrl; use std::fmt::{Debug, Error, Formatter}; #[cfg(feature = "gl")] @@ -157,6 +158,12 @@ pub trait WindowMethods { /// will want to avoid blocking on UI events, and just /// run the event loop at the vsync interval. fn set_animation_state(&self, _state: AnimationState); + /// Get the GL context + fn get_gl_context(&self) -> GlContext; + /// Get the native display + fn get_native_display(&self) -> NativeDisplay; + /// Get the GL api + fn get_gl_api(&self) -> GlApi; } pub trait EmbedderMethods { diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 3fd21e735e4..dbdd54ba5ce 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -107,6 +107,7 @@ use canvas::canvas_paint_thread::CanvasPaintThread; use canvas::webgl_thread::WebGLThreads; use canvas_traits::canvas::CanvasId; use canvas_traits::canvas::CanvasMsg; +use canvas_traits::media::WindowGLContext; use compositing::compositor_thread::CompositorProxy; use compositing::compositor_thread::Msg as ToCompositorMsg; use compositing::SendableFrameTree; @@ -409,6 +410,9 @@ pub struct Constellation { /// Like --disable-text-aa, this is useful for reftests where pixel perfect /// results are required. enable_canvas_antialiasing: bool, + + /// Application window's GL Context for Media player + player_context: WindowGLContext, } /// State needed to construct a constellation. @@ -457,6 +461,9 @@ pub struct InitialConstellationState { /// The XR device registry pub webxr_registry: webxr_api::Registry, + + /// Application window's GL Context for Media player + pub player_context: WindowGLContext, } /// Data needed for webdriver @@ -753,6 +760,7 @@ where is_running_problem_test, hard_fail, enable_canvas_antialiasing, + player_context: state.player_context, }; constellation.run(); @@ -994,6 +1002,7 @@ where .map(|threads| threads.pipeline()), webvr_chan: self.webvr_chan.clone(), webxr_registry: self.webxr_registry.clone(), + player_context: self.player_context.clone(), }); let pipeline = match result { diff --git a/components/constellation/pipeline.rs b/components/constellation/pipeline.rs index 9b1f0e7b336..049ac82b035 100644 --- a/components/constellation/pipeline.rs +++ b/components/constellation/pipeline.rs @@ -5,6 +5,7 @@ use crate::event_loop::EventLoop; use background_hang_monitor::HangMonitorRegister; use bluetooth_traits::BluetoothRequest; +use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLPipeline; use compositing::compositor_thread::Msg as CompositorMsg; use compositing::CompositionPipeline; @@ -191,6 +192,9 @@ pub struct InitialPipelineState { /// The XR device registry pub webxr_registry: webxr_api::Registry, + + /// Application window's GL Context for Media player + pub player_context: WindowGLContext, } pub struct NewPipeline { @@ -309,6 +313,7 @@ impl Pipeline { webgl_chan: state.webgl_chan, webvr_chan: state.webvr_chan, webxr_registry: state.webxr_registry, + player_context: state.player_context, }; // Spawn the child process. @@ -515,6 +520,7 @@ pub struct UnprivilegedPipelineContent { webgl_chan: Option, webvr_chan: Option>, webxr_registry: webxr_api::Registry, + player_context: WindowGLContext, } impl UnprivilegedPipelineContent { @@ -563,6 +569,7 @@ impl UnprivilegedPipelineContent { webrender_document: self.webrender_document, webrender_api_sender: self.webrender_api_sender.clone(), layout_is_busy: layout_thread_busy_flag.clone(), + player_context: self.player_context.clone(), }, self.load_data.clone(), self.opts.profile_script_events, diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 690dd928f45..a1ab6d08e68 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -71,7 +71,6 @@ use net_traits::{CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseLis use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType}; use script_layout_interface::HTMLMediaData; use servo_config::pref; -use servo_media::player::context::{GlContext, NativeDisplay, PlayerGLContext}; use servo_media::player::frame::{Frame, FrameRenderer}; use servo_media::player::{PlaybackState, Player, PlayerError, PlayerEvent, StreamType}; use servo_media::{ServoMedia, SupportsMediaType}; @@ -162,16 +161,6 @@ impl FrameRenderer for MediaFrameRenderer { } } -struct PlayerContextDummy(); -impl PlayerGLContext for PlayerContextDummy { - fn get_gl_context(&self) -> GlContext { - return GlContext::Unknown; - } - fn get_native_display(&self) -> NativeDisplay { - return NativeDisplay::Unknown; - } -} - #[must_root] #[derive(JSTraceable, MallocSizeOf)] enum SrcObject { @@ -1222,22 +1211,23 @@ impl HTMLMediaElement { _ => StreamType::Seekable, }; + let window = window_from_node(self); let (action_sender, action_receiver) = ipc::channel().unwrap(); let renderer: Option>> = match self.media_type_id() { HTMLMediaElementTypeId::HTMLAudioElement => None, HTMLMediaElementTypeId::HTMLVideoElement => Some(self.frame_renderer.clone()), }; + let player = ServoMedia::get().unwrap().create_player( stream_type, action_sender, renderer, - Box::new(PlayerContextDummy()), + Box::new(window.get_player_context()), ); *self.player.borrow_mut() = Some(player); let trusted_node = Trusted::new(self); - let window = window_from_node(self); let (task_source, canceller) = window .task_manager() .media_element_task_source_with_canceller(); diff --git a/components/script/dom/window.rs b/components/script/dom/window.rs index 7d5f0fdedbf..b834ebc8d6e 100644 --- a/components/script/dom/window.rs +++ b/components/script/dom/window.rs @@ -68,6 +68,7 @@ use crate::webdriver_handlers::jsval_to_webdriver; use app_units::Au; use base64; use bluetooth_traits::BluetoothRequest; +use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLChan; use crossbeam_channel::{unbounded, Sender, TryRecvError}; use cssparser::{Parser, ParserInput, SourceLocation}; @@ -318,6 +319,10 @@ pub struct Window { /// Replace unpaired surrogates in DOM strings with U+FFFD. /// See replace_surrogates: bool, + + /// Window's GL context from application + #[ignore_malloc_size_of = "defined in script_thread"] + player_context: WindowGLContext, } impl Window { @@ -481,6 +486,10 @@ impl Window { pub fn unminify_js(&self) -> bool { self.unminify_js } + + pub fn get_player_context(&self) -> WindowGLContext { + self.player_context.clone() + } } // https://html.spec.whatwg.org/multipage/#atob @@ -2074,6 +2083,7 @@ impl Window { is_headless: bool, replace_surrogates: bool, user_agent: Cow<'static, str>, + player_context: WindowGLContext, ) -> DomRoot { let layout_rpc: Box = { let (rpc_send, rpc_recv) = unbounded(); @@ -2154,6 +2164,7 @@ impl Window { unminify_js, userscripts_path, replace_surrogates, + player_context, }); unsafe { WindowBinding::Wrap(runtime.cx(), win) } diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 5afe7f1c2d1..6e72e8f0e9e 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -86,6 +86,7 @@ use crate::task_source::websocket::WebsocketTaskSource; use crate::task_source::TaskSourceName; use crate::webdriver_handlers; use bluetooth_traits::BluetoothRequest; +use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLPipeline; use crossbeam_channel::{unbounded, Receiver, Sender}; use devtools_traits::CSSError; @@ -512,6 +513,8 @@ unsafe_no_jsmanaged_fields!(TaskQueue); unsafe_no_jsmanaged_fields!(dyn BackgroundHangMonitorRegister); unsafe_no_jsmanaged_fields!(dyn BackgroundHangMonitor); +unsafe_no_jsmanaged_fields!(WindowGLContext); + #[derive(JSTraceable)] // ScriptThread instances are rooted on creation, so this is okay #[allow(unrooted_must_root)] @@ -646,7 +649,7 @@ pub struct ScriptThread { /// The Webrender Document ID associated with this thread. webrender_document: DocumentId, - /// FIXME(victor): + /// Webrender API sender. webrender_api_sender: RenderApiSender, /// Periodically print out on which events script threads spend their processing time. @@ -678,6 +681,9 @@ pub struct ScriptThread { /// An optional string allowing the user agent to be set for testing. user_agent: Cow<'static, str>, + + /// Application window's GL Context for Media player + player_context: WindowGLContext, } /// In the event of thread panic, all data on the stack runs its destructor. However, there @@ -1239,6 +1245,7 @@ impl ScriptThread { headless, replace_surrogates, user_agent, + player_context: state.player_context, } } @@ -2981,6 +2988,7 @@ impl ScriptThread { self.headless, self.replace_surrogates, self.user_agent.clone(), + self.player_context.clone(), ); // Initialize the browsing context for the window. diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index 40e710846f9..8a7dd06e025 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -21,6 +21,7 @@ pub mod webdriver_msg; use crate::webdriver_msg::{LoadStatus, WebDriverScriptCommand}; use bluetooth_traits::BluetoothRequest; +use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLPipeline; use crossbeam_channel::{Receiver, RecvTimeoutError, Sender}; use devtools_traits::{DevtoolScriptControlMsg, ScriptToDevtoolsControlMsg, WorkerId}; @@ -636,6 +637,8 @@ pub struct InitialScriptState { pub webrender_api_sender: RenderApiSender, /// Flag to indicate if the layout thread is busy handling a request. pub layout_is_busy: Arc, + /// Application window's GL Context for Media player + pub player_context: WindowGLContext, } /// This trait allows creating a `ScriptThread` without depending on the `script` diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 9ef5013a7b2..1e01b2fd39e 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -64,6 +64,7 @@ use bluetooth::BluetoothThreadFactory; use bluetooth_traits::BluetoothRequest; use canvas::gl_context::GLContextFactory; use canvas::webgl_thread::WebGLThreads; +use canvas_traits::media::WindowGLContext; use compositing::compositor_thread::{ CompositorProxy, CompositorReceiver, InitialCompositorState, Msg, }; @@ -303,6 +304,12 @@ where None }; + let player_context = WindowGLContext { + gl_context: window.get_gl_context(), + native_display: window.get_native_display(), + gl_api: window.get_gl_api(), + }; + // Create the constellation, which maintains the engine // pipelines, including the script and layout threads, as well // as the navigation context. @@ -321,6 +328,7 @@ where window.gl(), webvr_services, webxr_registry, + player_context, ); // Send the constellation's swmanager sender to service worker manager thread @@ -631,6 +639,7 @@ fn create_constellation( window_gl: Rc, webvr_services: Option, webxr_registry: webxr_api::Registry, + player_context: WindowGLContext, ) -> (Sender, SWManagerSenders) { // Global configuration options, parsed from the command line. let opts = opts::get(); @@ -712,6 +721,7 @@ fn create_constellation( webgl_threads, webvr_chan, webxr_registry, + player_context, }; let (constellation_chan, from_swmanager_sender) = Constellation::< script_layout_interface::message::Msg, diff --git a/ports/glutin/Cargo.toml b/ports/glutin/Cargo.toml index 37feefddead..592c079dda2 100644 --- a/ports/glutin/Cargo.toml +++ b/ports/glutin/Cargo.toml @@ -44,6 +44,7 @@ webrender_debugger = ["libservo/webrender_debugger"] [target.'cfg(not(target_os = "android"))'.dependencies] backtrace = "0.3" bitflags = "1.0" +clipboard = "0.5" crossbeam-channel = "0.3" euclid = "0.19" gleam = "0.6" @@ -54,9 +55,9 @@ libservo = {path = "../../components/servo"} libc = "0.2" log = "0.4" rust-webvr = { version = "0.13", features = ["glwindow"] } -webxr = { git = "https://github.com/servo/webxr", features = ["glwindow"] } +servo-media = {git = "https://github.com/servo/media"} tinyfiledialogs = "3.0" -clipboard = "0.5" +webxr = { git = "https://github.com/servo/webxr", features = ["glwindow"] } [target.'cfg(any(target_os = "linux", target_os = "windows"))'.dependencies] image = "0.21" diff --git a/ports/glutin/headed_window.rs b/ports/glutin/headed_window.rs index 3026170645e..b409304c93b 100644 --- a/ports/glutin/headed_window.rs +++ b/ports/glutin/headed_window.rs @@ -30,6 +30,7 @@ use servo::style_traits::DevicePixel; use servo::webrender_api::{ DeviceIntPoint, DeviceIntRect, DeviceIntSize, FramebufferIntSize, ScrollLocation, }; +use servo_media::player::context as MediaPlayerCtxt; use std::cell::{Cell, RefCell}; use std::mem; use std::rc::Rc; @@ -524,6 +525,18 @@ impl WindowMethods for Window { fn prepare_for_composite(&self) { self.gl_context.borrow_mut().make_current(); } + + fn get_gl_context(&self) -> MediaPlayerCtxt::GlContext { + MediaPlayerCtxt::GlContext::Unknown + } + + fn get_native_display(&self) -> MediaPlayerCtxt::NativeDisplay { + MediaPlayerCtxt::NativeDisplay::Unknown + } + + fn get_gl_api(&self) -> MediaPlayerCtxt::GlApi { + MediaPlayerCtxt::GlApi::None + } } fn winit_phase_to_touch_event_type(phase: TouchPhase) -> TouchEventType { diff --git a/ports/glutin/headless_window.rs b/ports/glutin/headless_window.rs index 6357ad04a9c..63b45cc5de7 100644 --- a/ports/glutin/headless_window.rs +++ b/ports/glutin/headless_window.rs @@ -14,6 +14,7 @@ use servo::servo_config::opts; use servo::servo_geometry::DeviceIndependentPixel; use servo::style_traits::DevicePixel; use servo::webrender_api::{DeviceIntRect, FramebufferIntSize}; +use servo_media::player::context as MediaPlayerCtxt; use std::cell::Cell; #[cfg(any(target_os = "linux", target_os = "macos"))] use std::ffi::CString; @@ -195,4 +196,16 @@ impl WindowMethods for Window { } fn prepare_for_composite(&self) { } + + fn get_gl_context(&self) -> MediaPlayerCtxt::GlContext { + MediaPlayerCtxt::GlContext::Unknown + } + + fn get_native_display(&self) -> MediaPlayerCtxt::NativeDisplay { + MediaPlayerCtxt::NativeDisplay::Unknown + } + + fn get_gl_api(&self) -> MediaPlayerCtxt::GlApi { + MediaPlayerCtxt::GlApi::None + } } From e0a5abf7df32def09795447bc28207d19a4a1387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Mon, 13 May 2019 20:06:46 +0200 Subject: [PATCH 03/31] Implement Window's GL context for Glutin --- ports/glutin/context.rs | 60 ++++++++++++++++++++++++++++++- ports/glutin/headed_window.rs | 68 ++++++++++++++++++++++++++++++----- 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/ports/glutin/context.rs b/ports/glutin/context.rs index 5e5e86cb461..4328f9e47ea 100644 --- a/ports/glutin/context.rs +++ b/ports/glutin/context.rs @@ -2,7 +2,10 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use glutin::{WindowedContext, NotCurrent, PossiblyCurrent}; +use glutin::os::ContextTraitExt; +use glutin::{NotCurrent, PossiblyCurrent, WindowedContext}; +use servo_media::player::context::GlContext as RawContext; +use std::os::raw; pub enum GlContext { Current(WindowedContext), @@ -71,4 +74,59 @@ impl GlContext { GlContext::None => unreachable!(), }; } + pub fn raw_context(&self) -> RawContext { + match self { + GlContext::Current(c) => { + let raw_handle = unsafe { c.raw_handle() }; + + #[cfg(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" + ))] + { + use glutin::os::unix::RawHandle; + + return match raw_handle { + RawHandle::Egl(handle) => RawContext::Egl(handle as usize), + RawHandle::Glx(handle) => RawContext::Glx(handle as usize), + }; + } + + #[cfg(not(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" + )))] + unimplemented!() + } + GlContext::NotCurrent(_) => { + error!("Context is not current."); + RawContext::Unknown + } + GlContext::None => unreachable!(), + } + } + pub fn egl_display(&self) -> Option<*const raw::c_void> { + match self { + GlContext::Current(c) => unsafe { c.get_egl_display() }, + GlContext::NotCurrent(_) => { + error!("Context is not current."); + None + }, + GlContext::None => unreachable!(), + } + } + + pub fn get_api(&self) -> glutin::Api { + match self { + GlContext::Current(c) => c.get_api(), + GlContext::NotCurrent(c) => c.get_api(), + GlContext::None => unreachable!(), + } + } } diff --git a/ports/glutin/headed_window.rs b/ports/glutin/headed_window.rs index b409304c93b..1e1801c2d49 100644 --- a/ports/glutin/headed_window.rs +++ b/ports/glutin/headed_window.rs @@ -13,9 +13,9 @@ use gleam::gl; use glutin::dpi::{LogicalPosition, LogicalSize, PhysicalSize}; #[cfg(target_os = "macos")] use glutin::os::macos::{ActivationPolicy, WindowBuilderExt}; +use glutin::Api; #[cfg(any(target_os = "linux", target_os = "windows"))] use glutin::Icon; -use glutin::Api; use glutin::{ElementState, KeyboardInput, MouseButton, MouseScrollDelta, TouchPhase}; #[cfg(any(target_os = "linux", target_os = "windows"))] use image; @@ -30,7 +30,7 @@ use servo::style_traits::DevicePixel; use servo::webrender_api::{ DeviceIntPoint, DeviceIntRect, DeviceIntSize, FramebufferIntSize, ScrollLocation, }; -use servo_media::player::context as MediaPlayerCtxt; +use servo_media::player::context::{GlApi, GlContext as PlayerGLContext, NativeDisplay}; use std::cell::{Cell, RefCell}; use std::mem; use std::rc::Rc; @@ -526,16 +526,68 @@ impl WindowMethods for Window { self.gl_context.borrow_mut().make_current(); } - fn get_gl_context(&self) -> MediaPlayerCtxt::GlContext { - MediaPlayerCtxt::GlContext::Unknown + fn get_gl_context(&self) -> PlayerGLContext { + self.gl_context.borrow().raw_context() } - fn get_native_display(&self) -> MediaPlayerCtxt::NativeDisplay { - MediaPlayerCtxt::NativeDisplay::Unknown + fn get_native_display(&self) -> NativeDisplay { + #[cfg(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" + ))] + let native_display = { + if let Some(display) = self.gl_context.borrow().egl_display() { + NativeDisplay::Egl(display as usize) + } else { + use glutin::os::unix::WindowExt; + + if let Some(display) = self.gl_context.borrow().window().get_wayland_display() { + NativeDisplay::Wayland(display as usize) + } else if let Some(display) = self.gl_context.borrow().window().get_xlib_display() { + NativeDisplay::X11(display as usize) + } else { + NativeDisplay::Unknown + } + } + }; + + #[cfg(not(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" + )))] + let native_display = NativeDisplay::Unknown; + + native_display } - fn get_gl_api(&self) -> MediaPlayerCtxt::GlApi { - MediaPlayerCtxt::GlApi::None + fn get_gl_api(&self) -> GlApi { + let api = self.gl_context.borrow().get_api(); + + let version = self.gl.get_string(gl::VERSION); + let version = version.trim_start_matches("OpenGL ES "); + let mut values = version.split(&['.', ' '][..]); + let major = values + .next() + .and_then(|v| v.parse::().ok()) + .unwrap_or(1); + let minor = values + .next() + .and_then(|v| v.parse::().ok()) + .unwrap_or(20); + + match api { + glutin::Api::OpenGl if major >= 3 && minor >= 2 => GlApi::OpenGL3, + glutin::Api::OpenGl => GlApi::OpenGL, + glutin::Api::OpenGlEs if major > 1 => GlApi::Gles2, + glutin::Api::OpenGlEs => GlApi::Gles1, + _ => GlApi::None, + } } } From 08d812ec0ed9e2f3d0a287ba5c7e0b926ae0f89a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 20 May 2019 14:19:08 +0200 Subject: [PATCH 04/31] Implement WindowMethods for Android --- Cargo.lock | 1 + ports/libsimpleservo/api/Cargo.toml | 1 + ports/libsimpleservo/api/src/gl_glue.rs | 20 +++++++++++++++---- ports/libsimpleservo/api/src/lib.rs | 26 ++++++++++++++++++++++++- ports/libsimpleservo/jniapi/src/lib.rs | 12 ++++++++---- 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0432a88cb4b..e6f4452f7c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4404,6 +4404,7 @@ dependencies = [ "libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "libservo 0.0.1", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "servo-media 0.1.0 (git+https://github.com/servo/media)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/ports/libsimpleservo/api/Cargo.toml b/ports/libsimpleservo/api/Cargo.toml index 6dd0c7c014a..241074ac593 100644 --- a/ports/libsimpleservo/api/Cargo.toml +++ b/ports/libsimpleservo/api/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] libservo = { path = "../../../components/servo" } log = "0.4" +servo-media = { git = "https://github.com/servo/media" } [target.'cfg(not(target_os = "macos"))'.dependencies] libc = "0.2" diff --git a/ports/libsimpleservo/api/src/gl_glue.rs b/ports/libsimpleservo/api/src/gl_glue.rs index 126de03789d..3f69ef15d46 100644 --- a/ports/libsimpleservo/api/src/gl_glue.rs +++ b/ports/libsimpleservo/api/src/gl_glue.rs @@ -18,6 +18,7 @@ pub mod egl { pub type khronos_uint64_t = libc::uint64_t; pub type khronos_ssize_t = libc::c_long; pub type EGLint = libc::int32_t; + pub type EGLContext = *const libc::c_void; pub type EGLNativeDisplayType = *const libc::c_void; pub type EGLNativePixmapType = *const libc::c_void; pub type NativeDisplayType = EGLNativeDisplayType; @@ -26,12 +27,19 @@ pub mod egl { include!(concat!(env!("OUT_DIR"), "/egl_bindings.rs")); - pub fn init() -> Result { + pub struct EGLInitResult { + pub gl_wrapper: crate::gl_glue::ServoGl, + pub gl_context: EGLContext, + pub display: EGLNativeDisplayType, + } + + #[cfg(target_os = "android")] + pub fn init() -> Result { info!("Loading EGL..."); unsafe { let egl = Egl; - let d = egl.GetCurrentDisplay(); - egl.SwapInterval(d, 1); + let display = egl.GetCurrentDisplay(); + egl.SwapInterval(display, 1); let egl = GlesFns::load_with(|addr| { let addr = CString::new(addr.as_bytes()).unwrap(); let addr = addr.as_ptr(); @@ -39,7 +47,11 @@ pub mod egl { egl.GetProcAddress(addr) as *const c_void }); info!("EGL loaded"); - Ok(egl) + Ok(EGLInitResult { + gl_wrapper: egl, + gl_context: Egl.GetCurrentContext(), + display, + }) } } } diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 1adf4d14eea..50284dc1111 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -25,7 +25,7 @@ use servo::servo_url::ServoUrl; use servo::webrender_api::{DevicePixel, FramebufferPixel, ScrollLocation}; use servo::webvr::{VRExternalShmemPtr, VRMainThreadHeartbeat, VRService, VRServiceManager}; use servo::{self, gl, BrowserId, Servo}; - +use servo_media::player::context as MediaPlayerContext; use std::cell::RefCell; use std::mem; use std::os::raw::c_void; @@ -48,6 +48,8 @@ pub struct InitOptions { pub density: f32, pub vr_init: VRInitOptions, pub enable_subpixel_text_antialiasing: bool, + pub gl_context_pointer: Option<*const c_void>, + pub native_display_pointer: Option<*const c_void>, } pub enum VRInitOptions { @@ -187,6 +189,8 @@ pub fn init( host_callbacks: callbacks, coordinates: RefCell::new(init_opts.coordinates), density: init_opts.density, + gl_context_pointer: init_opts.gl_context_pointer, + native_display_pointer: init_opts.native_display_pointer, }); let embedder_callbacks = Box::new(ServoEmbedderCallbacks { @@ -583,6 +587,8 @@ struct ServoWindowCallbacks { host_callbacks: Box, coordinates: RefCell, density: f32, + gl_context_pointer: Option<*const libc::c_void>, + native_display_pointer: Option<*const libc::c_void>, } impl EmbedderMethods for ServoEmbedderCallbacks { @@ -643,6 +649,24 @@ impl WindowMethods for ServoWindowCallbacks { hidpi_factor: TypedScale::new(self.density), } } + + fn get_gl_context(&self) -> MediaPlayerContext::GlContext { + match self.gl_context_pointer { + Some(context) => MediaPlayerContext::GlContext::Egl(context as usize), + None => MediaPlayerContext::GlContext::Unknown, + } + } + + fn get_native_display(&self) -> MediaPlayerContext::NativeDisplay { + match self.native_display_pointer { + Some(display) => MediaPlayerContext::NativeDisplay::Egl(display as usize), + None => MediaPlayerContext::NativeDisplay::Unknown, + } + } + + fn get_gl_api(&self) -> MediaPlayerContext::GlApi { + MediaPlayerContext::GlApi::Gles2 + } } struct ResourceReaderInstance; diff --git a/ports/libsimpleservo/jniapi/src/lib.rs b/ports/libsimpleservo/jniapi/src/lib.rs index d342a1a6c29..b392be62f66 100644 --- a/ports/libsimpleservo/jniapi/src/lib.rs +++ b/ports/libsimpleservo/jniapi/src/lib.rs @@ -52,7 +52,7 @@ pub fn Java_org_mozilla_servoview_JNIServo_init( opts: JObject, callbacks_obj: JObject, ) { - let (opts, log, log_str) = match get_options(&env, opts) { + let (mut opts, log, log_str) = match get_options(&env, opts) { Ok((opts, log, log_str)) => (opts, log, log_str), Err(err) => { throw(&env, &err); @@ -104,9 +104,11 @@ pub fn Java_org_mozilla_servoview_JNIServo_init( let wakeup = Box::new(WakeupCallback::new(callbacks_ref.clone(), &env)); let callbacks = Box::new(HostCallbacks::new(callbacks_ref, &env)); - if let Err(err) = - gl_glue::egl::init().and_then(|gl| simpleservo::init(opts, gl, wakeup, callbacks)) - { + if let Err(err) = gl_glue::egl::init().and_then(|egl_init| { + opts.gl_context_pointer = Some(egl_init.gl_context); + opts.native_display_pointer = Some(egl_init.display); + simpleservo::init(opts, egl_init.gl_wrapper, wakeup, callbacks) + }) { throw(&env, err) }; } @@ -726,6 +728,8 @@ fn get_options(env: &JNIEnv, opts: JObject) -> Result<(InitOptions, bool, Option } else { VRInitOptions::VRExternal(vr_pointer) }, + gl_context_pointer: None, + native_display_pointer: None, }; Ok((opts, log, log_str)) } From da8eb18763e5b0b9bd46ee9f6aa5f50e7c053b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Tue, 4 Jun 2019 17:46:58 +0200 Subject: [PATCH 05/31] Add media.glvideo preference --- components/config/prefs.rs | 3 +++ ports/glutin/headed_window.rs | 12 ++++++++++-- resources/prefs.json | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/components/config/prefs.rs b/components/config/prefs.rs index 0c6e394ffaa..1bd1a745799 100644 --- a/components/config/prefs.rs +++ b/components/config/prefs.rs @@ -446,6 +446,9 @@ mod gen { } }, media: { + glvideo: { + enabled: bool, + }, testing: { enabled: bool, } diff --git a/ports/glutin/headed_window.rs b/ports/glutin/headed_window.rs index 1e1801c2d49..9f934029aa6 100644 --- a/ports/glutin/headed_window.rs +++ b/ports/glutin/headed_window.rs @@ -24,7 +24,7 @@ use servo::compositing::windowing::{AnimationState, MouseWindowEvent, WindowEven use servo::compositing::windowing::{EmbedderCoordinates, WindowMethods}; use servo::embedder_traits::Cursor; use servo::script_traits::{TouchEventType, WheelMode, WheelDelta}; -use servo::servo_config::opts; +use servo::servo_config::{opts, pref}; use servo::servo_geometry::DeviceIndependentPixel; use servo::style_traits::DevicePixel; use servo::webrender_api::{ @@ -527,10 +527,18 @@ impl WindowMethods for Window { } fn get_gl_context(&self) -> PlayerGLContext { - self.gl_context.borrow().raw_context() + if pref!(media.glvideo.enabled) { + self.gl_context.borrow().raw_context() + } else { + PlayerGLContext::Unknown + } } fn get_native_display(&self) -> NativeDisplay { + if !pref!(media.glvideo.enabled) { + return NativeDisplay::Unknown; + } + #[cfg(any( target_os = "linux", target_os = "dragonfly", diff --git a/resources/prefs.json b/resources/prefs.json index 22c4e22b4db..8e7efaf6069 100644 --- a/resources/prefs.json +++ b/resources/prefs.json @@ -84,6 +84,7 @@ "layout.threads": 3, "layout.viewport.enabled": false, "layout.writing-mode.enabled": false, + "media.glvideo.enabled": false, "media.testing.enabled": false, "network.http-cache.disabled": false, "network.mime.sniff": false, From e36c0489bf3eff9b775ebc064fbd1a401d7634a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 21:09:33 +0200 Subject: [PATCH 06/31] Ignore frame if it is a GL texture And mock what would be if we handle GL textures. --- components/script/dom/htmlmediaelement.rs | 74 ++++++++++++++++++----- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index a1ab6d08e68..b66c5dd42fb 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -82,6 +82,7 @@ use std::mem; use std::rc::Rc; use std::sync::{Arc, Mutex}; use time::{self, Duration, Timespec}; +//use webrender_api::{ExternalImageData, ExternalImageId, ExternalImageType, TextureTarget}; use webrender_api::{ImageData, ImageDescriptor, ImageFormat, ImageKey, RenderApi}; use webrender_api::{RenderApiSender, Transaction}; @@ -111,6 +112,12 @@ impl MediaFrameRenderer { impl FrameRenderer for MediaFrameRenderer { fn render(&mut self, frame: Frame) { + let mut txn = Transaction::new(); + + if let Some(old_image_key) = mem::replace(&mut self.very_old_frame, self.old_frame.take()) { + txn.delete_image(old_image_key); + } + let descriptor = ImageDescriptor::new( frame.get_width(), frame.get_height(), @@ -119,24 +126,18 @@ impl FrameRenderer for MediaFrameRenderer { false, ); - let mut txn = Transaction::new(); - - let image_data = ImageData::Raw(frame.get_data()); - - if let Some(old_image_key) = mem::replace(&mut self.very_old_frame, self.old_frame.take()) { - txn.delete_image(old_image_key); - } - match self.current_frame { Some((ref image_key, ref mut width, ref mut height)) if *width == frame.get_width() && *height == frame.get_height() => { - txn.update_image( - *image_key, - descriptor, - image_data, - &webrender_api::DirtyRect::All, - ); + if !frame.is_gl_texture() { + txn.update_image( + *image_key, + descriptor, + ImageData::Raw(frame.get_data()), + &webrender_api::DirtyRect::All, + ); + } if let Some(old_image_key) = self.old_frame.take() { txn.delete_image(old_image_key); @@ -146,14 +147,55 @@ impl FrameRenderer for MediaFrameRenderer { self.old_frame = Some(*image_key); let new_image_key = self.api.generate_image_key(); - txn.add_image(new_image_key, descriptor, image_data, None); + + if !frame.is_gl_texture() { + txn.add_image( + new_image_key, + descriptor, + ImageData::Raw(frame.get_data()), + None, + ); + } else { + // txn.add_image( + // new_image_key, + // descriptor, + // ImageData::External(ExternalImageData { + // id: ExternalImageId(0), // let's try to fool webgl + // channel_index: 0, + // image_type: ExternalImageType::TextureHandle(TextureTarget::Default), + // }), + // None, + // ); + } + + /* update current_frame */ *image_key = new_image_key; *width = frame.get_width(); *height = frame.get_height(); }, None => { let image_key = self.api.generate_image_key(); - txn.add_image(image_key, descriptor, image_data, None); + + if !frame.is_gl_texture() { + txn.add_image( + image_key, + descriptor, + ImageData::Raw(frame.get_data()), + None, + ); + } else { + // txn.add_image( + // image_key, + // descriptor, + // ImageData::External(ExternalImageData { + // id: ExternalImageId(0), // let's try to fool webgl + // channel_index: 0, + // image_type: ExternalImageType::TextureHandle(TextureTarget::Default), + // }), + // None, + // ); + } + self.current_frame = Some((image_key, frame.get_width(), frame.get_height())); }, } From 0d52d5d30458503fbac4408a31b4c0e136edba06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Sat, 15 Jun 2019 13:17:53 +0200 Subject: [PATCH 07/31] Add GLPlayerMsg and glplayer_channel in canvas_traits GLPlayerMsg enum values are going to be the commands to send to the glplayer_thread. glplayer_channel mod is a copy of webgl_channel. --- components/canvas_traits/lib.rs | 1 + components/canvas_traits/media.rs | 54 +++++++++ components/canvas_traits/media_channel/ipc.rs | 14 +++ components/canvas_traits/media_channel/mod.rs | 106 ++++++++++++++++++ .../canvas_traits/media_channel/mpsc.rs | 58 ++++++++++ 5 files changed, 233 insertions(+) create mode 100644 components/canvas_traits/media_channel/ipc.rs create mode 100644 components/canvas_traits/media_channel/mod.rs create mode 100644 components/canvas_traits/media_channel/mpsc.rs diff --git a/components/canvas_traits/lib.rs b/components/canvas_traits/lib.rs index 2a52409a0e1..15d15a6ea68 100644 --- a/components/canvas_traits/lib.rs +++ b/components/canvas_traits/lib.rs @@ -15,6 +15,7 @@ extern crate serde; pub mod canvas; pub mod media; +mod media_channel; #[macro_use] pub mod webgl; mod webgl_channel; diff --git a/components/canvas_traits/media.rs b/components/canvas_traits/media.rs index 19be3e09a5d..4b5576dce91 100644 --- a/components/canvas_traits/media.rs +++ b/components/canvas_traits/media.rs @@ -2,8 +2,62 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +use euclid::Size2D; use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; +/// Helper function that creates a GLPlayer channel (GLPlayerSender, +/// GLPlayerReceiver) to be used in GLPlayerMsg. +pub use crate::media_channel::glplayer_channel; +/// Entry point channel type used for sending GLPlayerMsg messages to +/// the GLPlayer thread. +pub use crate::media_channel::GLPlayerChan; +/// Entry point type used in a Script Pipeline to get the GLPlayerChan +/// to be used in that thread. +pub use crate::media_channel::GLPlayerPipeline; +/// Receiver type used in GLPlayerMsg. +pub use crate::media_channel::GLPlayerReceiver; +/// Result type for send()/recv() calls in in GLPlayerMsg. +pub use crate::media_channel::GLPlayerSendResult; +/// Sender type used in GLPlayerMsg. +pub use crate::media_channel::GLPlayerSender; + +/// GLPlayer thread Message API +/// +/// These are the message that the thread will receive from the +/// constellation, the webrender::ExternalImageHandle multiplexor +/// implementation, or a htmlmediaelement +#[derive(Debug, Deserialize, Serialize)] +pub enum GLPlayerMsg { + /// Registers an instantiated player in DOM + RegisterPlayer(GLPlayerSender), + /// Unregisters a player's ID + UnregisterPlayer(u64), + /// Locks a specific texture from a player. Lock messages are used + /// for a correct synchronization with WebRender external image + /// API. + /// + /// WR locks a external texture when it wants to use the shared + /// texture contents. + /// + /// The WR client should not change the shared texture content + /// until the Unlock call. + /// + /// Currently OpenGL Sync Objects are used to implement the + /// synchronization mechanism. + Lock(u64, GLPlayerSender<(u32, Size2D, usize)>), + /// Unlocks a specific texture from a player. Unlock messages are + /// used for a correct synchronization with WebRender external + /// image API. + /// + /// The WR unlocks a context when it finished reading the shared + /// texture contents. + /// + /// Unlock messages are always sent after a Lock message. + Unlock(u64), + /// Frees all resources and closes the thread. + Exit, +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct WindowGLContext { /// Application's GL Context diff --git a/components/canvas_traits/media_channel/ipc.rs b/components/canvas_traits/media_channel/ipc.rs new file mode 100644 index 00000000000..67548a8ddce --- /dev/null +++ b/components/canvas_traits/media_channel/ipc.rs @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use serde::{Deserialize, Serialize}; +use std::io; + +pub type GLPlayerSender = ipc_channel::ipc::IpcSender; +pub type GLPlayerReceiver = ipc_channel::ipc::IpcReceiver; + +pub fn glplayer_channel Deserialize<'de>>( +) -> Result<(GLPlayerSender, GLPlayerReceiver), io::Error> { + ipc_channel::ipc::channel() +} diff --git a/components/canvas_traits/media_channel/mod.rs b/components/canvas_traits/media_channel/mod.rs new file mode 100644 index 00000000000..e6e7adffaa1 --- /dev/null +++ b/components/canvas_traits/media_channel/mod.rs @@ -0,0 +1,106 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +//! Enum wrappers to be able to select different channel implementations at runtime. + +mod ipc; +mod mpsc; + +use crate::media::GLPlayerMsg; +use serde::{Deserialize, Serialize}; +use servo_config::opts; +use std::fmt; + +lazy_static! { + static ref IS_MULTIPROCESS: bool = { opts::multiprocess() }; +} + +#[derive(Deserialize, Serialize)] +pub enum GLPlayerSender { + Ipc(ipc::GLPlayerSender), + Mpsc(mpsc::GLPlayerSender), +} + +impl Clone for GLPlayerSender +where + T: Serialize, +{ + fn clone(&self) -> Self { + match *self { + GLPlayerSender::Ipc(ref chan) => GLPlayerSender::Ipc(chan.clone()), + GLPlayerSender::Mpsc(ref chan) => GLPlayerSender::Mpsc(chan.clone()), + } + } +} + +impl fmt::Debug for GLPlayerSender { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "GLPlayerSender(..)") + } +} + +impl GLPlayerSender { + #[inline] + pub fn send(&self, msg: T) -> GLPlayerSendResult { + match *self { + GLPlayerSender::Ipc(ref sender) => sender.send(msg).map_err(|_| ()), + GLPlayerSender::Mpsc(ref sender) => sender.send(msg).map_err(|_| ()), + } + } +} + +pub type GLPlayerSendResult = Result<(), ()>; + +pub enum GLPlayerReceiver +where + T: for<'de> Deserialize<'de> + Serialize, +{ + Ipc(ipc::GLPlayerReceiver), + Mpsc(mpsc::GLPlayerReceiver), +} + +impl GLPlayerReceiver +where + T: for<'de> Deserialize<'de> + Serialize, +{ + pub fn recv(&self) -> Result { + match *self { + GLPlayerReceiver::Ipc(ref receiver) => receiver.recv().map_err(|_| ()), + GLPlayerReceiver::Mpsc(ref receiver) => receiver.recv().map_err(|_| ()), + } + } +} + +pub fn glplayer_channel() -> Result<(GLPlayerSender, GLPlayerReceiver), ()> +where + T: for<'de> Deserialize<'de> + Serialize, +{ + if *IS_MULTIPROCESS { + ipc::glplayer_channel() + .map(|(tx, rx)| (GLPlayerSender::Ipc(tx), GLPlayerReceiver::Ipc(rx))) + .map_err(|_| ()) + } else { + mpsc::glplayer_channel() + .map(|(tx, rx)| (GLPlayerSender::Mpsc(tx), GLPlayerReceiver::Mpsc(rx))) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct GLPlayerChan(pub GLPlayerSender); + +impl GLPlayerChan { + #[inline] + pub fn send(&self, msg: GLPlayerMsg) -> GLPlayerSendResult { + self.0.send(msg) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct GLPlayerPipeline(pub GLPlayerChan); + +impl GLPlayerPipeline { + pub fn channel(&self) -> GLPlayerChan { + self.0.clone() + } +} diff --git a/components/canvas_traits/media_channel/mpsc.rs b/components/canvas_traits/media_channel/mpsc.rs new file mode 100644 index 00000000000..7d6a396185a --- /dev/null +++ b/components/canvas_traits/media_channel/mpsc.rs @@ -0,0 +1,58 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use serde::{Deserialize, Serialize}; +use serde::{Deserializer, Serializer}; +use std::sync::mpsc; + +#[macro_use] +macro_rules! unreachable_serializable { + ($name:ident) => { + impl Serialize for $name { + fn serialize(&self, _: S) -> Result { + unreachable!(); + } + } + + impl<'a, T> Deserialize<'a> for $name { + fn deserialize(_: D) -> Result<$name, D::Error> + where + D: Deserializer<'a>, + { + unreachable!(); + } + } + }; +} + +pub struct GLPlayerSender(mpsc::Sender); +pub struct GLPlayerReceiver(mpsc::Receiver); + +impl Clone for GLPlayerSender { + fn clone(&self) -> Self { + GLPlayerSender(self.0.clone()) + } +} + +impl GLPlayerSender { + #[inline] + pub fn send(&self, data: T) -> Result<(), mpsc::SendError> { + self.0.send(data) + } +} + +impl GLPlayerReceiver { + #[inline] + pub fn recv(&self) -> Result { + self.0.recv() + } +} + +pub fn glplayer_channel() -> Result<(GLPlayerSender, GLPlayerReceiver), ()> { + let (sender, receiver) = mpsc::channel(); + Ok((GLPlayerSender(sender), GLPlayerReceiver(receiver))) +} + +unreachable_serializable!(GLPlayerReceiver); +unreachable_serializable!(GLPlayerSender); From 43467b4290ad5b8b735de1321d64ce4b5aa7cab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Tue, 11 Jun 2019 14:47:05 +0200 Subject: [PATCH 08/31] Add GLPlayerThreads in canvas This factory will launch a GLPlayerThread from the application main thread. And add GLPlayerThread, the multiplexor for media players' video renderers. This thread will receive commands from htmlmedialement and webrenderer. This code is also inspired by webgl_threads and WebGLThread. --- components/canvas/lib.rs | 2 + components/canvas/media_mode/inprocess.rs | 30 +++++++++++++ components/canvas/media_mode/mod.rs | 6 +++ components/canvas/media_thread.rs | 53 +++++++++++++++++++++++ 4 files changed, 91 insertions(+) create mode 100644 components/canvas/media_mode/inprocess.rs create mode 100644 components/canvas/media_mode/mod.rs create mode 100644 components/canvas/media_thread.rs diff --git a/components/canvas/lib.rs b/components/canvas/lib.rs index ba10f02636c..47628d57fbf 100644 --- a/components/canvas/lib.rs +++ b/components/canvas/lib.rs @@ -16,5 +16,7 @@ mod raqote_backend; pub mod canvas_data; pub mod canvas_paint_thread; pub mod gl_context; +mod media_mode; +pub mod media_thread; mod webgl_mode; pub mod webgl_thread; diff --git a/components/canvas/media_mode/inprocess.rs b/components/canvas/media_mode/inprocess.rs new file mode 100644 index 00000000000..0fa132efd16 --- /dev/null +++ b/components/canvas/media_mode/inprocess.rs @@ -0,0 +1,30 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use crate::media_thread::GLPlayerThread; +use canvas_traits::media::{GLPlayerChan, GLPlayerMsg, GLPlayerPipeline, GLPlayerSender}; + +/// GLPlayer Threading API entry point that lives in the constellation. +pub struct GLPlayerThreads(GLPlayerSender); + +impl GLPlayerThreads { + pub fn new() -> GLPlayerThreads { + let channel = GLPlayerThread::start(); + GLPlayerThreads(channel) + } + + /// Gets the GLPlayerThread handle for each script pipeline. + pub fn pipeline(&self) -> GLPlayerPipeline { + // This mode creates a single thread, so the existing + // GLPlayerChan is just cloned. + GLPlayerPipeline(GLPlayerChan(self.0.clone())) + } + + /// Sends an exit message to close the GLPlayerThreads + pub fn exit(&self) -> Result<(), &'static str> { + self.0 + .send(GLPlayerMsg::Exit) + .map_err(|_| "Failed to send Exit message") + } +} diff --git a/components/canvas/media_mode/mod.rs b/components/canvas/media_mode/mod.rs new file mode 100644 index 00000000000..7541463b2bb --- /dev/null +++ b/components/canvas/media_mode/mod.rs @@ -0,0 +1,6 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +mod inprocess; +pub use self::inprocess::GLPlayerThreads; diff --git a/components/canvas/media_thread.rs b/components/canvas/media_thread.rs new file mode 100644 index 00000000000..7ab530adbd0 --- /dev/null +++ b/components/canvas/media_thread.rs @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use canvas_traits::media::*; +use std::thread; + +/// GL player threading API entry point that lives in the +/// constellation. +/// +/// It allows to get a GLPlayerThead handle for each script pipeline. +pub use crate::media_mode::GLPlayerThreads; + +/// A GLPlayerThrx1ead manages the life cycle and message multiplexign of +/// a set of video players with GL render. +pub struct GLPlayerThread (); + +impl GLPlayerThread { + pub fn new() -> Self { + GLPlayerThread() + } + + pub fn start() -> GLPlayerSender { + let (sender, receiver) = glplayer_channel::().unwrap(); + thread::Builder::new() + .name("GLPlayerThread".to_owned()) + .spawn(move || { + let renderer = GLPlayerThread::new(); + loop { + let msg = receiver.recv().unwrap(); + let exit = renderer.handle_msg(msg); + if exit { + return; + } + } + }) + .expect("Thread spawning failed"); + + sender + } + + /// Handles a generic WebGLMsg message + #[inline] + fn handle_msg(&self, msg: GLPlayerMsg) -> bool { + trace!("processing {:?}", msg); + match msg { + GLPlayerMsg::Exit => return true, + _ => (), + } + + false + } +} From dd01728d530183b9150699018db941768c26ad0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Tue, 11 Jun 2019 20:19:15 +0200 Subject: [PATCH 09/31] Pass GLPlayerThreads to constellation Create the thread only if the GL context is known. --- components/canvas_traits/media.rs | 2 ++ components/constellation/constellation.rs | 14 ++++++++++++++ components/servo/lib.rs | 13 ++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/components/canvas_traits/media.rs b/components/canvas_traits/media.rs index 4b5576dce91..82f2e8e6248 100644 --- a/components/canvas_traits/media.rs +++ b/components/canvas_traits/media.rs @@ -66,6 +66,8 @@ pub struct WindowGLContext { pub gl_api: GlApi, /// Application's native display pub native_display: NativeDisplay, + /// A channel to the GLPlayer thread. + pub glplayer_chan: Option, } impl PlayerGLContext for WindowGLContext { diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index dbdd54ba5ce..b2a6729e681 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -104,6 +104,7 @@ use background_hang_monitor::HangMonitorRegister; use backtrace::Backtrace; use bluetooth_traits::BluetoothRequest; use canvas::canvas_paint_thread::CanvasPaintThread; +use canvas::media_thread::GLPlayerThreads; use canvas::webgl_thread::WebGLThreads; use canvas_traits::canvas::CanvasId; use canvas_traits::canvas::CanvasMsg; @@ -411,6 +412,9 @@ pub struct Constellation { /// results are required. enable_canvas_antialiasing: bool, + /// Entry point to create and get channels to a GLPlayerThread. + glplayer_threads: Option, + /// Application window's GL Context for Media player player_context: WindowGLContext, } @@ -461,6 +465,8 @@ pub struct InitialConstellationState { /// The XR device registry pub webxr_registry: webxr_api::Registry, + + pub glplayer_threads: Option, /// Application window's GL Context for Media player pub player_context: WindowGLContext, @@ -760,6 +766,7 @@ where is_running_problem_test, hard_fail, enable_canvas_antialiasing, + glplayer_threads: state.glplayer_threads, player_context: state.player_context, }; @@ -1804,6 +1811,13 @@ where } } + debug!("Exiting GLPlayer thread."); + if let Some(glplayer_threads) = self.glplayer_threads.as_ref() { + if let Err(e) = glplayer_threads.exit() { + warn!("Exit GLPlayer Thread failed ({})", e); + } + } + debug!("Exiting timer scheduler."); if let Err(e) = self.scheduler_chan.send(TimerSchedulerMsg::Exit) { warn!("Exit timer scheduler failed ({})", e); diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 1e01b2fd39e..f95a2f804de 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -63,6 +63,7 @@ fn webdriver(_port: u16, _constellation: Sender) {} use bluetooth::BluetoothThreadFactory; use bluetooth_traits::BluetoothRequest; use canvas::gl_context::GLContextFactory; +use canvas::media_thread::GLPlayerThreads; use canvas::webgl_thread::WebGLThreads; use canvas_traits::media::WindowGLContext; use compositing::compositor_thread::{ @@ -104,6 +105,7 @@ use profile_traits::time; use script_traits::{ConstellationMsg, SWManagerSenders, ScriptToConstellationChan}; use servo_config::opts; use servo_config::{pref, prefs}; +use servo_media::player::context::GlContext; use servo_media::ServoMedia; use std::borrow::Cow; use std::cmp::max; @@ -304,10 +306,16 @@ where None }; + let gl_context = window.get_gl_context(); + let glplayer_threads = match gl_context { + GlContext::Unknown => None, + _ => Some(GLPlayerThreads::new()), + }; let player_context = WindowGLContext { - gl_context: window.get_gl_context(), + gl_context, native_display: window.get_native_display(), gl_api: window.get_gl_api(), + glplayer_chan: glplayer_threads.as_ref().map(|threads| threads.pipeline()), }; // Create the constellation, which maintains the engine @@ -328,6 +336,7 @@ where window.gl(), webvr_services, webxr_registry, + glplayer_threads, player_context, ); @@ -639,6 +648,7 @@ fn create_constellation( window_gl: Rc, webvr_services: Option, webxr_registry: webxr_api::Registry, + glplayer_threads: Option, player_context: WindowGLContext, ) -> (Sender, SWManagerSenders) { // Global configuration options, parsed from the command line. @@ -721,6 +731,7 @@ fn create_constellation( webgl_threads, webvr_chan, webxr_registry, + glplayer_threads, player_context, }; let (constellation_chan, from_swmanager_sender) = Constellation::< From fe860f3aad8d165850a43e5b8c8baea47b736887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 11:13:30 +0200 Subject: [PATCH 10/31] Define the channel type for player events --- components/script/dom/htmlmediaelement.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index b66c5dd42fb..72ab28f1df0 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -1254,7 +1254,7 @@ impl HTMLMediaElement { }; let window = window_from_node(self); - let (action_sender, action_receiver) = ipc::channel().unwrap(); + let (action_sender, action_receiver) = ipc::channel::().unwrap(); let renderer: Option>> = match self.media_type_id() { HTMLMediaElementTypeId::HTMLAudioElement => None, HTMLMediaElementTypeId::HTMLVideoElement => Some(self.frame_renderer.clone()), @@ -1276,7 +1276,7 @@ impl HTMLMediaElement { ROUTER.add_route( action_receiver.to_opaque(), Box::new(move |message| { - let event: PlayerEvent = message.to().unwrap(); + let event = message.to().unwrap(); trace!("Player event {:?}", event); let this = trusted_node.clone(); if let Err(err) = task_source.queue_with_canceller( From 59aacb0076b3ed2a09f46c89dcfb21b6b0d9add8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 11:14:26 +0200 Subject: [PATCH 11/31] Add GLPlayerMsgForward enum These are the messages which are going to be sended/forwarded to the player. --- components/canvas_traits/media.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/canvas_traits/media.rs b/components/canvas_traits/media.rs index 82f2e8e6248..4eac95c6290 100644 --- a/components/canvas_traits/media.rs +++ b/components/canvas_traits/media.rs @@ -21,6 +21,15 @@ pub use crate::media_channel::GLPlayerSendResult; /// Sender type used in GLPlayerMsg. pub use crate::media_channel::GLPlayerSender; +/// These are the messages that the GLPlayer thread will forward to +/// the video player which lives in htmlmediaelement +#[derive(Debug, Deserialize, Serialize)] +pub enum GLPlayerMsgForward { + PlayerId(u64), + Lock(GLPlayerSender<(u32, Size2D, usize)>), + Unlock(), +} + /// GLPlayer thread Message API /// /// These are the message that the thread will receive from the @@ -29,7 +38,7 @@ pub use crate::media_channel::GLPlayerSender; #[derive(Debug, Deserialize, Serialize)] pub enum GLPlayerMsg { /// Registers an instantiated player in DOM - RegisterPlayer(GLPlayerSender), + RegisterPlayer(GLPlayerSender), /// Unregisters a player's ID UnregisterPlayer(u64), /// Locks a specific texture from a player. Lock messages are used From 38eb48441c4691e85cf7fba6bedcb99abfd8a7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 11:16:02 +0200 Subject: [PATCH 12/31] GLPlayer thread API implementation --- components/canvas/media_thread.rs | 39 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/components/canvas/media_thread.rs b/components/canvas/media_thread.rs index 7ab530adbd0..efcd6792d36 100644 --- a/components/canvas/media_thread.rs +++ b/components/canvas/media_thread.rs @@ -3,6 +3,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ use canvas_traits::media::*; +use fnv::FnvHashMap; use std::thread; /// GL player threading API entry point that lives in the @@ -13,11 +14,19 @@ pub use crate::media_mode::GLPlayerThreads; /// A GLPlayerThrx1ead manages the life cycle and message multiplexign of /// a set of video players with GL render. -pub struct GLPlayerThread (); +pub struct GLPlayerThread { + // Map of live players. + players: FnvHashMap>, + /// Id generator for new WebGLContexts. + next_player_id: u64, +} impl GLPlayerThread { pub fn new() -> Self { - GLPlayerThread() + GLPlayerThread { + players: Default::default(), + next_player_id: 1, + } } pub fn start() -> GLPlayerSender { @@ -25,7 +34,7 @@ impl GLPlayerThread { thread::Builder::new() .name("GLPlayerThread".to_owned()) .spawn(move || { - let renderer = GLPlayerThread::new(); + let mut renderer = GLPlayerThread::new(); loop { let msg = receiver.recv().unwrap(); let exit = renderer.handle_msg(msg); @@ -41,11 +50,31 @@ impl GLPlayerThread { /// Handles a generic WebGLMsg message #[inline] - fn handle_msg(&self, msg: GLPlayerMsg) -> bool { + fn handle_msg(&mut self, msg: GLPlayerMsg) -> bool { trace!("processing {:?}", msg); match msg { + GLPlayerMsg::RegisterPlayer(sender) => { + let id = self.next_player_id; + self.players.insert(id, sender.clone()); + sender.send(GLPlayerMsgForward::PlayerId(id)).unwrap(); + self.next_player_id += 1; + }, + GLPlayerMsg::UnregisterPlayer(id) => { + if self.players.remove(&id).is_none() { + warn!("Tried to remove an unknown player"); + } + }, + GLPlayerMsg::Lock(id, handler_sender) => { + self.players.get(&id).map(|sender| { + sender.send(GLPlayerMsgForward::Lock(handler_sender)).ok(); + }); + }, + GLPlayerMsg::Unlock(id) => { + self.players.get(&id).map(|sender| { + sender.send(GLPlayerMsgForward::Unlock()).ok(); + }); + }, GLPlayerMsg::Exit => return true, - _ => (), } false From 586d3f8da597df97bfe6ba6642dc73271e65bb19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 11:51:59 +0200 Subject: [PATCH 13/31] Register and unregister a player in GLPlayer thread --- components/script/dom/htmlmediaelement.rs | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 72ab28f1df0..dc7a797abe1 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -58,6 +58,7 @@ use crate::microtask::{Microtask, MicrotaskRunnable}; use crate::network_listener::{self, NetworkListener, PreInvoke, ResourceTimingListener}; use crate::script_thread::ScriptThread; use crate::task_source::TaskSource; +use canvas_traits::media::*; use dom_struct::dom_struct; use headers::{ContentLength, ContentRange, HeaderMapExt}; use html5ever::{LocalName, Prefix}; @@ -294,6 +295,8 @@ pub struct HTMLMediaElement { next_timeupdate_event: Cell, /// Latest fetch request context. current_fetch_context: DomRefCell>, + /// Player Id reported the player thread + id: Cell, } /// @@ -355,6 +358,7 @@ impl HTMLMediaElement { text_tracks_list: Default::default(), next_timeupdate_event: Cell::new(time::get_time() + Duration::milliseconds(250)), current_fetch_context: DomRefCell::new(None), + id: Cell::new(0), } } @@ -1290,6 +1294,25 @@ impl HTMLMediaElement { }), ); + // GLPlayer thread setup + let player_id = window + .get_player_context() + .glplayer_chan + .map(|pipeline| { + let (image_sender, image_receiver) = + glplayer_channel::().unwrap(); + pipeline + .channel() + .send(GLPlayerMsg::RegisterPlayer(image_sender)) + .unwrap(); + match image_receiver.recv().unwrap() { + GLPlayerMsgForward::PlayerId(id) => id, + _ => unreachable!(), + } + }) + .unwrap_or(0); + self.id.set(player_id); + Ok(()) } @@ -1617,6 +1640,16 @@ impl HTMLMediaElement { impl Drop for HTMLMediaElement { fn drop(&mut self) { + let window = window_from_node(self); + window.get_player_context().glplayer_chan.map(|pipeline| { + if let Err(err) = pipeline + .channel() + .send(GLPlayerMsg::UnregisterPlayer(self.id.get())) + { + warn!("GLPlayer disappeared!: {:?}", err); + } + }); + if let Some(ref player) = *self.player.borrow() { if let Err(err) = player.shutdown() { warn!("Error shutting down player {:?}", err); From 65f9e2161c11941aafc7ae6e40dc8fee834ff137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 12:21:22 +0200 Subject: [PATCH 14/31] Add texture image in MediaFrameRenderer --- components/script/dom/htmlmediaelement.rs | 49 ++++++++++++----------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index dc7a797abe1..e0dd3d57c8f 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -83,11 +83,12 @@ use std::mem; use std::rc::Rc; use std::sync::{Arc, Mutex}; use time::{self, Duration, Timespec}; -//use webrender_api::{ExternalImageData, ExternalImageId, ExternalImageType, TextureTarget}; +use webrender_api::{ExternalImageData, ExternalImageId, ExternalImageType, TextureTarget}; use webrender_api::{ImageData, ImageDescriptor, ImageFormat, ImageKey, RenderApi}; use webrender_api::{RenderApiSender, Transaction}; pub struct MediaFrameRenderer { + id: u64, api: RenderApi, current_frame: Option<(ImageKey, i32, i32)>, old_frame: Option, @@ -97,6 +98,7 @@ pub struct MediaFrameRenderer { impl MediaFrameRenderer { fn new(render_api_sender: RenderApiSender) -> Self { Self { + id: 0, api: render_api_sender.create_api(), current_frame: None, old_frame: None, @@ -156,17 +158,17 @@ impl FrameRenderer for MediaFrameRenderer { ImageData::Raw(frame.get_data()), None, ); - } else { - // txn.add_image( - // new_image_key, - // descriptor, - // ImageData::External(ExternalImageData { - // id: ExternalImageId(0), // let's try to fool webgl - // channel_index: 0, - // image_type: ExternalImageType::TextureHandle(TextureTarget::Default), - // }), - // None, - // ); + } else if self.id != 0 { + txn.add_image( + new_image_key, + descriptor, + ImageData::External(ExternalImageData { + id: ExternalImageId(self.id), + channel_index: 0, + image_type: ExternalImageType::TextureHandle(TextureTarget::Default), + }), + None, + ); } /* update current_frame */ @@ -184,17 +186,17 @@ impl FrameRenderer for MediaFrameRenderer { ImageData::Raw(frame.get_data()), None, ); - } else { - // txn.add_image( - // image_key, - // descriptor, - // ImageData::External(ExternalImageData { - // id: ExternalImageId(0), // let's try to fool webgl - // channel_index: 0, - // image_type: ExternalImageType::TextureHandle(TextureTarget::Default), - // }), - // None, - // ); + } else if self.id != 0 { + txn.add_image( + image_key, + descriptor, + ImageData::External(ExternalImageData { + id: ExternalImageId(self.id), + channel_index: 0, + image_type: ExternalImageType::TextureHandle(TextureTarget::Default), + }), + None, + ); } self.current_frame = Some((image_key, frame.get_width(), frame.get_height())); @@ -1312,6 +1314,7 @@ impl HTMLMediaElement { }) .unwrap_or(0); self.id.set(player_id); + self.frame_renderer.lock().unwrap().id = player_id; Ok(()) } From e000c14eb27dd6d1a0014cf428ad28518dd8cdcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 14:39:49 +0200 Subject: [PATCH 15/31] Implement webrender::ExternalImageHandler for player Added trait GLPlayerExternalImageApi and its implementation Implemented webrender::ExternalImageHandler using GLPlayerExternalImageApi --- components/canvas/media_mode/inprocess.rs | 55 ++++++++++++++++++++- components/canvas/media_thread.rs | 58 +++++++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/components/canvas/media_mode/inprocess.rs b/components/canvas/media_mode/inprocess.rs index 0fa132efd16..7a1c781bfee 100644 --- a/components/canvas/media_mode/inprocess.rs +++ b/components/canvas/media_mode/inprocess.rs @@ -2,8 +2,12 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use crate::media_thread::GLPlayerThread; -use canvas_traits::media::{GLPlayerChan, GLPlayerMsg, GLPlayerPipeline, GLPlayerSender}; +use crate::media_thread::{GLPlayerExternalImageApi, GLPlayerExternalImageHandler, GLPlayerThread}; +use canvas_traits::media::glplayer_channel; +use canvas_traits::media::{ + GLPlayerChan, GLPlayerMsg, GLPlayerPipeline, GLPlayerReceiver, GLPlayerSender, +}; +use euclid::Size2D; /// GLPlayer Threading API entry point that lives in the constellation. pub struct GLPlayerThreads(GLPlayerSender); @@ -28,3 +32,50 @@ impl GLPlayerThreads { .map_err(|_| "Failed to send Exit message") } } + +/// Bridge between the webrender::ExternalImage callbacks and the +/// GLPlayerThreads. +struct GLPlayerExternalImages { + // @FIXME(victor): this should be added when GstGLSyncMeta is + // added + //webrender_gl: Rc, + glplayer_channel: GLPlayerSender, + // Used to avoid creating a new channel on each received WebRender + // request. + lock_channel: ( + GLPlayerSender<(u32, Size2D, usize)>, + GLPlayerReceiver<(u32, Size2D, usize)>, + ), +} + +impl GLPlayerExternalImages { + fn new(channel: GLPlayerSender) -> Self { + Self { + glplayer_channel: channel, + lock_channel: glplayer_channel().unwrap(), + } + } +} + +impl GLPlayerExternalImageApi for GLPlayerExternalImages { + fn lock(&mut self, id: u64) -> (u32, Size2D) { + // The GLPlayerMsgForward::Lock message inserts a fence in the + // GLPlayer command queue. + self.glplayer_channel + .send(GLPlayerMsg::Lock(id, self.lock_channel.0.clone())) + .unwrap(); + let (image_id, size, _gl_sync) = self.lock_channel.1.recv().unwrap(); + // The next glWaitSync call is run on the WR thread and it's + // used to synchronize the two flows of OpenGL commands in + // order to avoid WR using a semi-ready GLPlayer texture. + // glWaitSync doesn't block WR thread, it affects only + // internal OpenGL subsystem. + //self.webrender_gl + // .wait_sync(gl_sync as gl::GLsync, 0, gl::TIMEOUT_IGNORED); + (image_id, size) + } + + fn unlock(&mut self, id: u64) { + self.glplayer_channel.send(GLPlayerMsg::Unlock(id)).unwrap(); + } +} diff --git a/components/canvas/media_thread.rs b/components/canvas/media_thread.rs index efcd6792d36..c8a104a4658 100644 --- a/components/canvas/media_thread.rs +++ b/components/canvas/media_thread.rs @@ -3,6 +3,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ use canvas_traits::media::*; +use euclid::Size2D; use fnv::FnvHashMap; use std::thread; @@ -80,3 +81,60 @@ impl GLPlayerThread { false } } + +/// This trait is used as a bridge between the `GLPlayerThreads` +/// implementation and the WR ExternalImageHandler API implemented in +/// the `GLPlayerExternalImageHandler` struct. +// +/// `GLPlayerExternalImageHandler` takes care of type conversions +/// between WR and GLPlayer info (e.g keys, uvs). +// +/// It uses this trait to notify lock/unlock messages and get the +/// required info that WR needs. +// +/// `GLPlayerThreads` receives lock/unlock message notifications and +/// takes care of sending the unlock/lock messages to the appropiate +/// `GLPlayerThread`. +pub trait GLPlayerExternalImageApi { + fn lock(&mut self, id: u64) -> (u32, Size2D); + fn unlock(&mut self, id: u64); +} + +/// WebRender External Image Handler implementation +pub struct GLPlayerExternalImageHandler { + handler: T, +} + +impl GLPlayerExternalImageHandler { + pub fn new(handler: T) -> Self { + Self { handler: handler } + } +} + +impl webrender::ExternalImageHandler + for GLPlayerExternalImageHandler +{ + /// Lock the external image. Then, WR could start to read the + /// image content. + /// The WR client should not change the image content until the + /// unlock() call. + fn lock( + &mut self, + key: webrender_api::ExternalImageId, + _channel_index: u8, + _rendering: webrender_api::ImageRendering, + ) -> webrender::ExternalImage { + let (texture_id, size) = self.handler.lock(key.0); + + webrender::ExternalImage { + uv: webrender_api::TexelRect::new(0.0, 0.0, size.width as f32, size.height as f32), + source: webrender::ExternalImageSource::NativeTexture(texture_id), + } + } + + /// Unlock the external image. The WR should not read the image + /// content after this call. + fn unlock(&mut self, key: webrender_api::ExternalImageId, _channel_index: u8) { + self.handler.unlock(key.0); + } +} From a9ad088e707a637f9f7e223cc2a0419af41f726d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 14:41:36 +0200 Subject: [PATCH 16/31] Instanciate and use the implemented webrender::ExternalImageHandler --- components/canvas/media_mode/inprocess.rs | 6 ++++-- components/servo/lib.rs | 11 ++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/components/canvas/media_mode/inprocess.rs b/components/canvas/media_mode/inprocess.rs index 7a1c781bfee..86ce201020b 100644 --- a/components/canvas/media_mode/inprocess.rs +++ b/components/canvas/media_mode/inprocess.rs @@ -13,9 +13,11 @@ use euclid::Size2D; pub struct GLPlayerThreads(GLPlayerSender); impl GLPlayerThreads { - pub fn new() -> GLPlayerThreads { + pub fn new() -> (GLPlayerThreads, Box) { let channel = GLPlayerThread::start(); - GLPlayerThreads(channel) + let external = + GLPlayerExternalImageHandler::new(GLPlayerExternalImages::new(channel.clone())); + (GLPlayerThreads(channel), Box::new(external)) } /// Gets the GLPlayerThread handle for each script pipeline. diff --git a/components/servo/lib.rs b/components/servo/lib.rs index f95a2f804de..185d5c5838e 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -309,8 +309,13 @@ where let gl_context = window.get_gl_context(); let glplayer_threads = match gl_context { GlContext::Unknown => None, - _ => Some(GLPlayerThreads::new()), + _ => { + let (glplayer_threads, image_handler) = GLPlayerThreads::new(); + webrender.set_external_image_handler(image_handler); + Some(glplayer_threads) + }, }; + let player_context = WindowGLContext { gl_context, native_display: window.get_native_display(), @@ -697,7 +702,7 @@ fn create_constellation( // Initialize WebGL Thread entry point. let webgl_threads = gl_factory.map(|factory| { - let (webgl_threads, image_handler, output_handler) = WebGLThreads::new( + let (webgl_threads, _image_handler, output_handler) = WebGLThreads::new( factory, window_gl, webrender_api_sender.clone(), @@ -705,7 +710,7 @@ fn create_constellation( ); // Set webrender external image handler for WebGL textures - webrender.set_external_image_handler(image_handler); + //webrender.set_external_image_handler(image_handler); // Set DOM to texture handler, if enabled. if let Some(output_handler) = output_handler { From 95c3d52e7bbc400e1109887a39a561840ec6119a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 14:42:44 +0200 Subject: [PATCH 17/31] Add a task source for messages from GLPlayer thread --- components/canvas_traits/media_channel/mod.rs | 10 ++++++- components/script/dom/htmlmediaelement.rs | 29 +++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/components/canvas_traits/media_channel/mod.rs b/components/canvas_traits/media_channel/mod.rs index e6e7adffaa1..6ce9a867598 100644 --- a/components/canvas_traits/media_channel/mod.rs +++ b/components/canvas_traits/media_channel/mod.rs @@ -70,13 +70,21 @@ where GLPlayerReceiver::Mpsc(ref receiver) => receiver.recv().map_err(|_| ()), } } + + pub fn to_opaque(self) -> ipc_channel::ipc::OpaqueIpcReceiver { + match self { + GLPlayerReceiver::Ipc(receiver) => receiver.to_opaque(), + _ => unreachable!(), + } + } } pub fn glplayer_channel() -> Result<(GLPlayerSender, GLPlayerReceiver), ()> where T: for<'de> Deserialize<'de> + Serialize, { - if *IS_MULTIPROCESS { + // Let's use Ipc until we move the Player instance into GPlayerThread + if true { ipc::glplayer_channel() .map(|(tx, rx)| (GLPlayerSender::Ipc(tx), GLPlayerReceiver::Ipc(rx))) .map_err(|_| ()) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index e0dd3d57c8f..cee7259af06 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -1297,7 +1297,7 @@ impl HTMLMediaElement { ); // GLPlayer thread setup - let player_id = window + let (player_id, image_receiver) = window .get_player_context() .glplayer_chan .map(|pipeline| { @@ -1308,14 +1308,37 @@ impl HTMLMediaElement { .send(GLPlayerMsg::RegisterPlayer(image_sender)) .unwrap(); match image_receiver.recv().unwrap() { - GLPlayerMsgForward::PlayerId(id) => id, + GLPlayerMsgForward::PlayerId(id) => (id, Some(image_receiver)), _ => unreachable!(), } }) - .unwrap_or(0); + .unwrap_or((0, None)); + self.id.set(player_id); self.frame_renderer.lock().unwrap().id = player_id; + if let Some(image_receiver) = image_receiver { + let trusted_node = Trusted::new(self); + let (task_source, canceller) = window + .task_manager() + .media_element_task_source_with_canceller(); + ROUTER.add_route( + image_receiver.to_opaque(), + Box::new(move |message| { + let msg: GLPlayerMsgForward = message.to().unwrap(); + let _this = trusted_node.clone(); + if let Err(err) = task_source.queue_with_canceller( + task!(handle_glplayer_message: move || { + trace!("GLPlayer message {:?}", msg); + }), + &canceller, + ) { + warn!("Could not queue GL player message handler task {:?}", err); + } + }), + ); + } + Ok(()) } From 6e2ee394c966a1119b4e03ae855b7319f48087e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 14 Jun 2019 18:02:47 +0200 Subject: [PATCH 18/31] Handle WR's lock/unlock logic under FrameRenderer --- components/script/dom/htmlmediaelement.rs | 128 ++++++++++++++++++---- 1 file changed, 104 insertions(+), 24 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index cee7259af06..abd6613f340 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -60,6 +60,7 @@ use crate::script_thread::ScriptThread; use crate::task_source::TaskSource; use canvas_traits::media::*; use dom_struct::dom_struct; +use euclid::Size2D; use headers::{ContentLength, ContentRange, HeaderMapExt}; use html5ever::{LocalName, Prefix}; use http::header::{self, HeaderMap, HeaderValue}; @@ -87,12 +88,57 @@ use webrender_api::{ExternalImageData, ExternalImageId, ExternalImageType, Textu use webrender_api::{ImageData, ImageDescriptor, ImageFormat, ImageKey, RenderApi}; use webrender_api::{RenderApiSender, Transaction}; +#[derive(PartialEq)] +enum FrameStatus { + Locked, + Unlocked, +} + +struct FrameHolder(FrameStatus, Frame); + +impl FrameHolder { + fn new(frame: Frame) -> FrameHolder { + FrameHolder(FrameStatus::Unlocked, frame) + } + + fn lock(&mut self) { + if self.0 == FrameStatus::Unlocked { + self.0 = FrameStatus::Locked; + }; + } + + fn unlock(&mut self) { + if self.0 == FrameStatus::Locked { + self.0 = FrameStatus::Unlocked; + }; + } + + fn set(&mut self, new_frame: Frame) { + if self.0 == FrameStatus::Unlocked { + self.1 = new_frame + }; + } + + fn get(&self) -> (u32, Size2D, usize) { + if self.0 == FrameStatus::Locked { + ( + self.1.get_texture_id(), + Size2D::new(self.1.get_width(), self.1.get_height()), + 0, + ) + } else { + unreachable!(); + } + } +} + pub struct MediaFrameRenderer { id: u64, api: RenderApi, current_frame: Option<(ImageKey, i32, i32)>, old_frame: Option, very_old_frame: Option, + current_frame_holder: Option, } impl MediaFrameRenderer { @@ -103,6 +149,7 @@ impl MediaFrameRenderer { current_frame: None, old_frame: None, very_old_frame: None, + current_frame_holder: None, } } @@ -140,6 +187,10 @@ impl FrameRenderer for MediaFrameRenderer { ImageData::Raw(frame.get_data()), &webrender_api::DirtyRect::All, ); + } else if self.id != 0 { + self.current_frame_holder + .get_or_insert_with(|| FrameHolder::new(frame.clone())) + .set(frame); } if let Some(old_image_key) = self.old_frame.take() { @@ -151,33 +202,38 @@ impl FrameRenderer for MediaFrameRenderer { let new_image_key = self.api.generate_image_key(); - if !frame.is_gl_texture() { - txn.add_image( - new_image_key, - descriptor, - ImageData::Raw(frame.get_data()), - None, - ); - } else if self.id != 0 { - txn.add_image( - new_image_key, - descriptor, - ImageData::External(ExternalImageData { - id: ExternalImageId(self.id), - channel_index: 0, - image_type: ExternalImageType::TextureHandle(TextureTarget::Default), - }), - None, - ); - } - /* update current_frame */ *image_key = new_image_key; *width = frame.get_width(); *height = frame.get_height(); + + if !frame.is_gl_texture() { + txn.add_image( + new_image_key, + descriptor, + ImageData::Raw(frame.get_data()), + None, + ); + } else if self.id != 0 { + txn.add_image( + new_image_key, + descriptor, + ImageData::External(ExternalImageData { + id: ExternalImageId(self.id), + channel_index: 0, + image_type: ExternalImageType::TextureHandle(TextureTarget::Default), + }), + None, + ); + + self.current_frame_holder + .get_or_insert_with(|| FrameHolder::new(frame.clone())) + .set(frame); + } }, None => { let image_key = self.api.generate_image_key(); + self.current_frame = Some((image_key, frame.get_width(), frame.get_height())); if !frame.is_gl_texture() { txn.add_image( @@ -197,9 +253,9 @@ impl FrameRenderer for MediaFrameRenderer { }), None, ); - } - self.current_frame = Some((image_key, frame.get_width(), frame.get_height())); + self.current_frame_holder = Some(FrameHolder::new(frame)); + } }, } self.api.update_resources(txn.resource_updates); @@ -1325,11 +1381,35 @@ impl HTMLMediaElement { ROUTER.add_route( image_receiver.to_opaque(), Box::new(move |message| { - let msg: GLPlayerMsgForward = message.to().unwrap(); - let _this = trusted_node.clone(); + let msg = message.to().unwrap(); + let this = trusted_node.clone(); if let Err(err) = task_source.queue_with_canceller( task!(handle_glplayer_message: move || { trace!("GLPlayer message {:?}", msg); + let frame_renderer = this.root().frame_renderer.clone(); + + match msg { + GLPlayerMsgForward::Lock(sender) => { + frame_renderer + .lock() + .unwrap() + .current_frame_holder + .as_mut() + .map(|holder| { + holder.lock(); + sender.send(holder.get()).unwrap(); + }); + }, + GLPlayerMsgForward::Unlock() => { + frame_renderer + .lock() + .unwrap() + .current_frame_holder + .as_mut() + .map(|holder| holder.unlock()); + }, + _ => (), + } }), &canceller, ) { From eb3857237c2760ed17a2888f01c069566def509f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Mon, 17 Jun 2019 11:46:24 +0200 Subject: [PATCH 19/31] Add more raw context handler for glutin port --- ports/glutin/context.rs | 32 ++++++++++++++++++++++---- ports/glutin/headed_window.rs | 42 +++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/ports/glutin/context.rs b/ports/glutin/context.rs index 4328f9e47ea..ed56a7afc76 100644 --- a/ports/glutin/context.rs +++ b/ports/glutin/context.rs @@ -77,30 +77,53 @@ impl GlContext { pub fn raw_context(&self) -> RawContext { match self { GlContext::Current(c) => { - let raw_handle = unsafe { c.raw_handle() }; - #[cfg(any( target_os = "linux", target_os = "dragonfly", target_os = "freebsd", target_os = "netbsd", - target_os = "openbsd" + target_os = "openbsd", ))] { use glutin::os::unix::RawHandle; + let raw_handle = unsafe { c.raw_handle() }; return match raw_handle { RawHandle::Egl(handle) => RawContext::Egl(handle as usize), RawHandle::Glx(handle) => RawContext::Glx(handle as usize), }; } + #[cfg(target_os = "windows")] + { + use glutin::os::windows::RawHandle; + + let raw_handle = unsafe { c.raw_handle() }; + return match raw_handle { + RawHandle::Egl(handle) => RawContext::Egl(handle as usize), + // @TODO(victor): RawContext::Wgl in servo-media + RawHandle::Wgl(_) => unimplemented!(), + } + } + + #[cfg(target_os = "android")] + { + let raw_handle = unsafe { c.raw_handle() }; + return RawContext::Egl(raw_handle as usize); + } + + #[cfg(target_os = "macos")] + return unimplemeneted!(); // @TODO(victor): RawContext::Cocoa in servo-media + #[cfg(not(any( target_os = "linux", target_os = "dragonfly", target_os = "freebsd", target_os = "netbsd", - target_os = "openbsd" + target_os = "openbsd", + target_os = "windows", + target_os = "android", + target_os = "macos", )))] unimplemented!() } @@ -111,6 +134,7 @@ impl GlContext { GlContext::None => unreachable!(), } } + pub fn egl_display(&self) -> Option<*const raw::c_void> { match self { GlContext::Current(c) => unsafe { c.get_egl_display() }, diff --git a/ports/glutin/headed_window.rs b/ports/glutin/headed_window.rs index 9f934029aa6..c19d2bdb63b 100644 --- a/ports/glutin/headed_window.rs +++ b/ports/glutin/headed_window.rs @@ -544,21 +544,43 @@ impl WindowMethods for Window { target_os = "dragonfly", target_os = "freebsd", target_os = "netbsd", - target_os = "openbsd" + target_os = "openbsd", + target_os = "windows", + target_os = "android", ))] let native_display = { if let Some(display) = self.gl_context.borrow().egl_display() { NativeDisplay::Egl(display as usize) } else { - use glutin::os::unix::WindowExt; + #[cfg(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + ))] + { + use glutin::os::unix::WindowExt; - if let Some(display) = self.gl_context.borrow().window().get_wayland_display() { - NativeDisplay::Wayland(display as usize) - } else if let Some(display) = self.gl_context.borrow().window().get_xlib_display() { - NativeDisplay::X11(display as usize) - } else { - NativeDisplay::Unknown + if let Some(display) = self.gl_context.borrow().window().get_wayland_display() { + NativeDisplay::Wayland(display as usize) + } else if let Some(display) = + self.gl_context.borrow().window().get_xlib_display() + { + NativeDisplay::X11(display as usize) + } else { + NativeDisplay::Unknown + } } + + #[cfg(not(any( + target_os = "linux", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + )))] + NativeDisplay::Unknown } }; @@ -567,7 +589,9 @@ impl WindowMethods for Window { target_os = "dragonfly", target_os = "freebsd", target_os = "netbsd", - target_os = "openbsd" + target_os = "openbsd", + target_os = "windows", + target_os = "android", )))] let native_display = NativeDisplay::Unknown; From db4fff173d11b7b7a67775e3a7274a84aa0340d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Wed, 19 Jun 2019 18:40:19 +0200 Subject: [PATCH 20/31] Create the WindowGLContext at create_constellation() --- components/servo/lib.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 185d5c5838e..9917c8bea9a 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -306,21 +306,11 @@ where None }; - let gl_context = window.get_gl_context(); - let glplayer_threads = match gl_context { - GlContext::Unknown => None, - _ => { - let (glplayer_threads, image_handler) = GLPlayerThreads::new(); - webrender.set_external_image_handler(image_handler); - Some(glplayer_threads) - }, - }; - let player_context = WindowGLContext { - gl_context, + gl_context: window.get_gl_context(), native_display: window.get_native_display(), gl_api: window.get_gl_api(), - glplayer_chan: glplayer_threads.as_ref().map(|threads| threads.pipeline()), + glplayer_chan: None, }; // Create the constellation, which maintains the engine @@ -341,7 +331,6 @@ where window.gl(), webvr_services, webxr_registry, - glplayer_threads, player_context, ); @@ -653,7 +642,6 @@ fn create_constellation( window_gl: Rc, webvr_services: Option, webxr_registry: webxr_api::Registry, - glplayer_threads: Option, player_context: WindowGLContext, ) -> (Sender, SWManagerSenders) { // Global configuration options, parsed from the command line. @@ -720,6 +708,20 @@ fn create_constellation( webgl_threads }); + let glplayer_threads = match player_context.gl_context { + GlContext::Unknown => None, + _ => { + let (glplayer_threads, image_handler) = GLPlayerThreads::new(); + webrender.set_external_image_handler(image_handler); + Some(glplayer_threads) + }, + }; + + let player_context = WindowGLContext { + glplayer_chan: glplayer_threads.as_ref().map(|threads| threads.pipeline()), + ..player_context + }; + let initial_state = InitialConstellationState { compositor_proxy, embedder_proxy, From 2b3a8bf4909a5705e62ff2872253e89f2975f666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 21 Jun 2019 18:00:45 +0200 Subject: [PATCH 21/31] Fix OSX build --- ports/glutin/context.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/glutin/context.rs b/ports/glutin/context.rs index ed56a7afc76..5e62c2b3e92 100644 --- a/ports/glutin/context.rs +++ b/ports/glutin/context.rs @@ -113,7 +113,7 @@ impl GlContext { } #[cfg(target_os = "macos")] - return unimplemeneted!(); // @TODO(victor): RawContext::Cocoa in servo-media + return unimplemented!(); // @TODO(victor): RawContext::Cocoa in servo-media #[cfg(not(any( target_os = "linux", From 7d589ed4f5762ee185b60a34a76bb59cdf05a536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 24 Jun 2019 18:45:31 +0200 Subject: [PATCH 22/31] Media crate --- Cargo.lock | 21 +++- components/canvas/lib.rs | 2 - components/canvas/media_mode/mod.rs | 6 -- components/canvas_traits/Cargo.toml | 1 - components/canvas_traits/lib.rs | 2 - components/canvas_traits/media.rs | 94 ------------------ components/constellation/Cargo.toml | 1 + components/constellation/constellation.rs | 3 +- components/constellation/pipeline.rs | 2 +- components/media/Cargo.toml | 23 +++++ .../media_mode/inprocess.rs => media/lib.rs} | 97 ++++++++++++++++++- .../media_channel/ipc.rs | 0 .../media_channel/mod.rs | 2 +- .../media_channel/mpsc.rs | 0 components/{canvas => media}/media_thread.rs | 11 +-- components/script/Cargo.toml | 1 + components/script/dom/htmlmediaelement.rs | 2 +- components/script/dom/window.rs | 2 +- components/script/script_thread.rs | 2 +- components/script_traits/Cargo.toml | 1 + components/script_traits/lib.rs | 2 +- components/servo/Cargo.toml | 1 + components/servo/lib.rs | 4 +- 23 files changed, 152 insertions(+), 128 deletions(-) delete mode 100644 components/canvas/media_mode/mod.rs delete mode 100644 components/canvas_traits/media.rs create mode 100644 components/media/Cargo.toml rename components/{canvas/media_mode/inprocess.rs => media/lib.rs} (52%) rename components/{canvas_traits => media}/media_channel/ipc.rs (100%) rename components/{canvas_traits => media}/media_channel/mod.rs (99%) rename components/{canvas_traits => media}/media_channel/mpsc.rs (100%) rename components/{canvas => media}/media_thread.rs (96%) diff --git a/Cargo.lock b/Cargo.lock index e6f4452f7c7..864ca86d567 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -426,7 +426,6 @@ dependencies = [ "pixels 0.0.1", "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", - "servo-media 0.1.0 (git+https://github.com/servo/media)", "servo_config 0.0.1", "typetag 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "webrender_api 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", @@ -648,6 +647,7 @@ dependencies = [ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "layout_traits 0.0.1", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "media 0.0.1", "metrics 0.0.1", "msg 0.0.1", "net 0.0.1", @@ -2518,6 +2518,7 @@ dependencies = [ "keyboard-types 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "layout_thread 0.0.1", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "media 0.0.1", "mozangle 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "msg 0.0.1", "net 0.0.1", @@ -2710,6 +2711,22 @@ name = "matches" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "media" +version = "0.0.1" +dependencies = [ + "euclid 0.19.8 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ipc-channel 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "servo-media 0.1.0 (git+https://github.com/servo/media)", + "servo_config 0.0.1", + "webrender 0.60.0 (git+https://github.com/servo/webrender)", + "webrender_api 0.60.0 (git+https://github.com/servo/webrender)", +] + [[package]] name = "memchr" version = "2.2.0" @@ -3804,6 +3821,7 @@ dependencies = [ "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "malloc_size_of 0.0.1", "malloc_size_of_derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "media 0.0.1", "metrics 0.0.1", "mime 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 2.0.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3929,6 +3947,7 @@ dependencies = [ "libc 0.2.53 (registry+https://github.com/rust-lang/crates.io-index)", "malloc_size_of 0.0.1", "malloc_size_of_derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "media 0.0.1", "msg 0.0.1", "net_traits 0.0.1", "pixels 0.0.1", diff --git a/components/canvas/lib.rs b/components/canvas/lib.rs index 47628d57fbf..ba10f02636c 100644 --- a/components/canvas/lib.rs +++ b/components/canvas/lib.rs @@ -16,7 +16,5 @@ mod raqote_backend; pub mod canvas_data; pub mod canvas_paint_thread; pub mod gl_context; -mod media_mode; -pub mod media_thread; mod webgl_mode; pub mod webgl_thread; diff --git a/components/canvas/media_mode/mod.rs b/components/canvas/media_mode/mod.rs deleted file mode 100644 index 7541463b2bb..00000000000 --- a/components/canvas/media_mode/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -mod inprocess; -pub use self::inprocess::GLPlayerThreads; diff --git a/components/canvas_traits/Cargo.toml b/components/canvas_traits/Cargo.toml index fabe074409c..52388be8863 100644 --- a/components/canvas_traits/Cargo.toml +++ b/components/canvas_traits/Cargo.toml @@ -25,7 +25,6 @@ pixels = {path = "../pixels"} serde = "1.0" serde_bytes = "0.10" servo_config = {path = "../config"} -servo-media = {git = "https://github.com/servo/media"} typetag = "0.1" webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]} webvr_traits = {path = "../webvr_traits"} diff --git a/components/canvas_traits/lib.rs b/components/canvas_traits/lib.rs index 15d15a6ea68..c19ca073658 100644 --- a/components/canvas_traits/lib.rs +++ b/components/canvas_traits/lib.rs @@ -14,8 +14,6 @@ extern crate malloc_size_of_derive; extern crate serde; pub mod canvas; -pub mod media; -mod media_channel; #[macro_use] pub mod webgl; mod webgl_channel; diff --git a/components/canvas_traits/media.rs b/components/canvas_traits/media.rs deleted file mode 100644 index 4eac95c6290..00000000000 --- a/components/canvas_traits/media.rs +++ /dev/null @@ -1,94 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ - -use euclid::Size2D; -use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; - -/// Helper function that creates a GLPlayer channel (GLPlayerSender, -/// GLPlayerReceiver) to be used in GLPlayerMsg. -pub use crate::media_channel::glplayer_channel; -/// Entry point channel type used for sending GLPlayerMsg messages to -/// the GLPlayer thread. -pub use crate::media_channel::GLPlayerChan; -/// Entry point type used in a Script Pipeline to get the GLPlayerChan -/// to be used in that thread. -pub use crate::media_channel::GLPlayerPipeline; -/// Receiver type used in GLPlayerMsg. -pub use crate::media_channel::GLPlayerReceiver; -/// Result type for send()/recv() calls in in GLPlayerMsg. -pub use crate::media_channel::GLPlayerSendResult; -/// Sender type used in GLPlayerMsg. -pub use crate::media_channel::GLPlayerSender; - -/// These are the messages that the GLPlayer thread will forward to -/// the video player which lives in htmlmediaelement -#[derive(Debug, Deserialize, Serialize)] -pub enum GLPlayerMsgForward { - PlayerId(u64), - Lock(GLPlayerSender<(u32, Size2D, usize)>), - Unlock(), -} - -/// GLPlayer thread Message API -/// -/// These are the message that the thread will receive from the -/// constellation, the webrender::ExternalImageHandle multiplexor -/// implementation, or a htmlmediaelement -#[derive(Debug, Deserialize, Serialize)] -pub enum GLPlayerMsg { - /// Registers an instantiated player in DOM - RegisterPlayer(GLPlayerSender), - /// Unregisters a player's ID - UnregisterPlayer(u64), - /// Locks a specific texture from a player. Lock messages are used - /// for a correct synchronization with WebRender external image - /// API. - /// - /// WR locks a external texture when it wants to use the shared - /// texture contents. - /// - /// The WR client should not change the shared texture content - /// until the Unlock call. - /// - /// Currently OpenGL Sync Objects are used to implement the - /// synchronization mechanism. - Lock(u64, GLPlayerSender<(u32, Size2D, usize)>), - /// Unlocks a specific texture from a player. Unlock messages are - /// used for a correct synchronization with WebRender external - /// image API. - /// - /// The WR unlocks a context when it finished reading the shared - /// texture contents. - /// - /// Unlock messages are always sent after a Lock message. - Unlock(u64), - /// Frees all resources and closes the thread. - Exit, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct WindowGLContext { - /// Application's GL Context - pub gl_context: GlContext, - /// Application's GL Api - pub gl_api: GlApi, - /// Application's native display - pub native_display: NativeDisplay, - /// A channel to the GLPlayer thread. - pub glplayer_chan: Option, -} - -impl PlayerGLContext for WindowGLContext { - fn get_gl_context(&self) -> GlContext { - self.gl_context.clone() - } - - fn get_native_display(&self) -> NativeDisplay { - self.native_display.clone() - } - - fn get_gl_api(&self) -> GlApi { - self.gl_api.clone() - } -} diff --git a/components/constellation/Cargo.toml b/components/constellation/Cargo.toml index f91396343d8..f5270aa2d90 100644 --- a/components/constellation/Cargo.toml +++ b/components/constellation/Cargo.toml @@ -35,6 +35,7 @@ ipc-channel = "0.11" layout_traits = {path = "../layout_traits"} keyboard-types = "0.4.3" log = "0.4" +media = {path = "../media"} metrics = {path = "../metrics"} msg = {path = "../msg"} net = {path = "../net"} diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index b2a6729e681..1d6bef75b4e 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -104,11 +104,9 @@ use background_hang_monitor::HangMonitorRegister; use backtrace::Backtrace; use bluetooth_traits::BluetoothRequest; use canvas::canvas_paint_thread::CanvasPaintThread; -use canvas::media_thread::GLPlayerThreads; use canvas::webgl_thread::WebGLThreads; use canvas_traits::canvas::CanvasId; use canvas_traits::canvas::CanvasMsg; -use canvas_traits::media::WindowGLContext; use compositing::compositor_thread::CompositorProxy; use compositing::compositor_thread::Msg as ToCompositorMsg; use compositing::SendableFrameTree; @@ -125,6 +123,7 @@ use keyboard_types::webdriver::Event as WebDriverInputEvent; use keyboard_types::KeyboardEvent; use layout_traits::LayoutThreadFactory; use log::{Level, LevelFilter, Log, Metadata, Record}; +use media::{GLPlayerThreads, WindowGLContext}; use msg::constellation_msg::{BackgroundHangMonitorRegister, HangMonitorAlert, SamplerControlMsg}; use msg::constellation_msg::{ BrowsingContextGroupId, BrowsingContextId, HistoryStateId, PipelineId, diff --git a/components/constellation/pipeline.rs b/components/constellation/pipeline.rs index 049ac82b035..07df7a4b2c9 100644 --- a/components/constellation/pipeline.rs +++ b/components/constellation/pipeline.rs @@ -5,7 +5,6 @@ use crate::event_loop::EventLoop; use background_hang_monitor::HangMonitorRegister; use bluetooth_traits::BluetoothRequest; -use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLPipeline; use compositing::compositor_thread::Msg as CompositorMsg; use compositing::CompositionPipeline; @@ -18,6 +17,7 @@ use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use ipc_channel::router::ROUTER; use ipc_channel::Error; use layout_traits::LayoutThreadFactory; +use media::WindowGLContext; use metrics::PaintTimeMetrics; use msg::constellation_msg::TopLevelBrowsingContextId; use msg::constellation_msg::{BackgroundHangMonitorRegister, HangMonitorAlert, SamplerControlMsg}; diff --git a/components/media/Cargo.toml b/components/media/Cargo.toml new file mode 100644 index 00000000000..53f531a26cf --- /dev/null +++ b/components/media/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "media" +version = "0.0.1" +authors = ["The Servo Project Developers"] +license = "MPL-2.0" +edition = "2018" +publish = false + +[lib] +name = "media" +path = "lib.rs" + +[dependencies] +euclid = "0.19" +fnv = "1.0" +ipc-channel = "0.11" +lazy_static = "1" +log = "0.4" +serde = "1.0" +servo_config = {path = "../config"} +servo-media = {git = "https://github.com/servo/media"} +webrender = {git = "https://github.com/servo/webrender"} +webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]} diff --git a/components/canvas/media_mode/inprocess.rs b/components/media/lib.rs similarity index 52% rename from components/canvas/media_mode/inprocess.rs rename to components/media/lib.rs index 86ce201020b..c98d97df597 100644 --- a/components/canvas/media_mode/inprocess.rs +++ b/components/media/lib.rs @@ -2,12 +2,99 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use crate::media_thread::{GLPlayerExternalImageApi, GLPlayerExternalImageHandler, GLPlayerThread}; -use canvas_traits::media::glplayer_channel; -use canvas_traits::media::{ - GLPlayerChan, GLPlayerMsg, GLPlayerPipeline, GLPlayerReceiver, GLPlayerSender, -}; +#![crate_name = "media"] +#![crate_type = "rlib"] +#![deny(unsafe_code)] + +#[macro_use] +extern crate lazy_static; +#[macro_use] +extern crate log; +#[macro_use] +extern crate serde; + use euclid::Size2D; +use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; + +mod media_channel; +mod media_thread; + +pub use crate::media_channel::glplayer_channel; + +use crate::media_channel::{GLPlayerChan, GLPlayerPipeline, GLPlayerReceiver, GLPlayerSender}; +use crate::media_thread::{GLPlayerExternalImageApi, GLPlayerExternalImageHandler, GLPlayerThread}; + +/// These are the messages that the GLPlayer thread will forward to +/// the video player which lives in htmlmediaelement +#[derive(Debug, Deserialize, Serialize)] +pub enum GLPlayerMsgForward { + PlayerId(u64), + Lock(GLPlayerSender<(u32, Size2D, usize)>), + Unlock(), +} + +/// GLPlayer thread Message API +/// +/// These are the message that the thread will receive from the +/// constellation, the webrender::ExternalImageHandle multiplexor +/// implementation, or a htmlmediaelement +#[derive(Debug, Deserialize, Serialize)] +pub enum GLPlayerMsg { + /// Registers an instantiated player in DOM + RegisterPlayer(GLPlayerSender), + /// Unregisters a player's ID + UnregisterPlayer(u64), + /// Locks a specific texture from a player. Lock messages are used + /// for a correct synchronization with WebRender external image + /// API. + /// + /// WR locks a external texture when it wants to use the shared + /// texture contents. + /// + /// The WR client should not change the shared texture content + /// until the Unlock call. + /// + /// Currently OpenGL Sync Objects are used to implement the + /// synchronization mechanism. + Lock(u64, GLPlayerSender<(u32, Size2D, usize)>), + /// Unlocks a specific texture from a player. Unlock messages are + /// used for a correct synchronization with WebRender external + /// image API. + /// + /// The WR unlocks a context when it finished reading the shared + /// texture contents. + /// + /// Unlock messages are always sent after a Lock message. + Unlock(u64), + /// Frees all resources and closes the thread. + Exit, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct WindowGLContext { + /// Application's GL Context + pub gl_context: GlContext, + /// Application's GL Api + pub gl_api: GlApi, + /// Application's native display + pub native_display: NativeDisplay, + /// A channel to the GLPlayer thread. + pub glplayer_chan: Option, +} + +impl PlayerGLContext for WindowGLContext { + fn get_gl_context(&self) -> GlContext { + self.gl_context.clone() + } + + fn get_native_display(&self) -> NativeDisplay { + self.native_display.clone() + } + + fn get_gl_api(&self) -> GlApi { + self.gl_api.clone() + } +} /// GLPlayer Threading API entry point that lives in the constellation. pub struct GLPlayerThreads(GLPlayerSender); diff --git a/components/canvas_traits/media_channel/ipc.rs b/components/media/media_channel/ipc.rs similarity index 100% rename from components/canvas_traits/media_channel/ipc.rs rename to components/media/media_channel/ipc.rs diff --git a/components/canvas_traits/media_channel/mod.rs b/components/media/media_channel/mod.rs similarity index 99% rename from components/canvas_traits/media_channel/mod.rs rename to components/media/media_channel/mod.rs index 6ce9a867598..5616a8479fd 100644 --- a/components/canvas_traits/media_channel/mod.rs +++ b/components/media/media_channel/mod.rs @@ -7,7 +7,7 @@ mod ipc; mod mpsc; -use crate::media::GLPlayerMsg; +use crate::GLPlayerMsg; use serde::{Deserialize, Serialize}; use servo_config::opts; use std::fmt; diff --git a/components/canvas_traits/media_channel/mpsc.rs b/components/media/media_channel/mpsc.rs similarity index 100% rename from components/canvas_traits/media_channel/mpsc.rs rename to components/media/media_channel/mpsc.rs diff --git a/components/canvas/media_thread.rs b/components/media/media_thread.rs similarity index 96% rename from components/canvas/media_thread.rs rename to components/media/media_thread.rs index c8a104a4658..e47e7c799f9 100644 --- a/components/canvas/media_thread.rs +++ b/components/media/media_thread.rs @@ -2,17 +2,14 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -use canvas_traits::media::*; +use crate::media_channel::{glplayer_channel, GLPlayerSender}; +/// GL player threading API entry point that lives in the +/// constellation. +use crate::{GLPlayerMsg, GLPlayerMsgForward}; use euclid::Size2D; use fnv::FnvHashMap; use std::thread; -/// GL player threading API entry point that lives in the -/// constellation. -/// -/// It allows to get a GLPlayerThead handle for each script pipeline. -pub use crate::media_mode::GLPlayerThreads; - /// A GLPlayerThrx1ead manages the life cycle and message multiplexign of /// a set of video players with GL render. pub struct GLPlayerThread { diff --git a/components/script/Cargo.toml b/components/script/Cargo.toml index fd270a74fc8..e0f11c85434 100644 --- a/components/script/Cargo.toml +++ b/components/script/Cargo.toml @@ -70,6 +70,7 @@ libc = "0.2" log = "0.4" malloc_size_of = { path = "../malloc_size_of" } malloc_size_of_derive = "0.1" +media = {path = "../media"} metrics = {path = "../metrics"} mitochondria = "1.1.2" mime = "0.3.13" diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index abd6613f340..6b1191d6373 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -58,7 +58,6 @@ use crate::microtask::{Microtask, MicrotaskRunnable}; use crate::network_listener::{self, NetworkListener, PreInvoke, ResourceTimingListener}; use crate::script_thread::ScriptThread; use crate::task_source::TaskSource; -use canvas_traits::media::*; use dom_struct::dom_struct; use euclid::Size2D; use headers::{ContentLength, ContentRange, HeaderMapExt}; @@ -66,6 +65,7 @@ use html5ever::{LocalName, Prefix}; use http::header::{self, HeaderMap, HeaderValue}; use ipc_channel::ipc; use ipc_channel::router::ROUTER; +use media::{glplayer_channel, GLPlayerMsg, GLPlayerMsgForward}; use net_traits::image::base::Image; use net_traits::image_cache::ImageResponse; use net_traits::request::{CredentialsMode, Destination, Referrer, RequestBuilder, RequestMode}; diff --git a/components/script/dom/window.rs b/components/script/dom/window.rs index b834ebc8d6e..c5866bc5e6b 100644 --- a/components/script/dom/window.rs +++ b/components/script/dom/window.rs @@ -68,7 +68,6 @@ use crate::webdriver_handlers::jsval_to_webdriver; use app_units::Au; use base64; use bluetooth_traits::BluetoothRequest; -use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLChan; use crossbeam_channel::{unbounded, Sender, TryRecvError}; use cssparser::{Parser, ParserInput, SourceLocation}; @@ -86,6 +85,7 @@ use js::jsval::JSVal; use js::jsval::UndefinedValue; use js::rust::wrappers::JS_DefineProperty; use js::rust::HandleValue; +use media::WindowGLContext; use msg::constellation_msg::PipelineId; use net_traits::image_cache::{ImageCache, ImageResponder, ImageResponse}; use net_traits::image_cache::{PendingImageId, PendingImageResponse}; diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index 6e72e8f0e9e..fc9498a8ebb 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -86,7 +86,6 @@ use crate::task_source::websocket::WebsocketTaskSource; use crate::task_source::TaskSourceName; use crate::webdriver_handlers; use bluetooth_traits::BluetoothRequest; -use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLPipeline; use crossbeam_channel::{unbounded, Receiver, Sender}; use devtools_traits::CSSError; @@ -104,6 +103,7 @@ use js::jsapi::{JSContext, JS_SetWrapObjectCallbacks}; use js::jsapi::{JSTracer, SetWindowProxyClass}; use js::jsval::UndefinedValue; use js::rust::ParentRuntime; +use media::WindowGLContext; use metrics::{PaintTimeMetrics, MAX_TASK_NS}; use mime::{self, Mime}; use msg::constellation_msg::{ diff --git a/components/script_traits/Cargo.toml b/components/script_traits/Cargo.toml index 20f58e6bb27..5cc35639ae0 100644 --- a/components/script_traits/Cargo.toml +++ b/components/script_traits/Cargo.toml @@ -27,6 +27,7 @@ keyboard-types = "0.4.3" libc = "0.2" malloc_size_of = { path = "../malloc_size_of" } malloc_size_of_derive = "0.1" +media = {path = "../media"} msg = {path = "../msg"} net_traits = {path = "../net_traits"} pixels = {path = "../pixels"} diff --git a/components/script_traits/lib.rs b/components/script_traits/lib.rs index 8a7dd06e025..4b87a104a4d 100644 --- a/components/script_traits/lib.rs +++ b/components/script_traits/lib.rs @@ -21,7 +21,6 @@ pub mod webdriver_msg; use crate::webdriver_msg::{LoadStatus, WebDriverScriptCommand}; use bluetooth_traits::BluetoothRequest; -use canvas_traits::media::WindowGLContext; use canvas_traits::webgl::WebGLPipeline; use crossbeam_channel::{Receiver, RecvTimeoutError, Sender}; use devtools_traits::{DevtoolScriptControlMsg, ScriptToDevtoolsControlMsg, WorkerId}; @@ -35,6 +34,7 @@ use ipc_channel::Error as IpcError; use keyboard_types::webdriver::Event as WebDriverInputEvent; use keyboard_types::{CompositionEvent, KeyboardEvent}; use libc::c_void; +use media::WindowGLContext; use msg::constellation_msg::BackgroundHangMonitorRegister; use msg::constellation_msg::{BrowsingContextId, HistoryStateId, PipelineId}; use msg::constellation_msg::{PipelineNamespaceId, TopLevelBrowsingContextId, TraversalDirection}; diff --git a/components/servo/Cargo.toml b/components/servo/Cargo.toml index d64ab06149a..10ad4319651 100644 --- a/components/servo/Cargo.toml +++ b/components/servo/Cargo.toml @@ -55,6 +55,7 @@ ipc-channel = "0.11" keyboard-types = "0.4" layout_thread = {path = "../layout_thread"} log = "0.4" +media = {path = "../media"} msg = {path = "../msg"} net = {path = "../net"} net_traits = {path = "../net_traits"} diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 9917c8bea9a..1496e1ce054 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -35,6 +35,7 @@ pub use euclid; pub use gfx; pub use ipc_channel; pub use layout_thread; +pub use media; pub use msg; pub use net; pub use net_traits; @@ -63,9 +64,7 @@ fn webdriver(_port: u16, _constellation: Sender) {} use bluetooth::BluetoothThreadFactory; use bluetooth_traits::BluetoothRequest; use canvas::gl_context::GLContextFactory; -use canvas::media_thread::GLPlayerThreads; use canvas::webgl_thread::WebGLThreads; -use canvas_traits::media::WindowGLContext; use compositing::compositor_thread::{ CompositorProxy, CompositorReceiver, InitialCompositorState, Msg, }; @@ -95,6 +94,7 @@ use gaol::sandbox::{ChildSandbox, ChildSandboxMethods}; use gfx::font_cache_thread::FontCacheThread; use ipc_channel::ipc::{self, IpcSender}; use log::{Log, Metadata, Record}; +use media::{GLPlayerThreads, WindowGLContext}; use msg::constellation_msg::{PipelineNamespace, PipelineNamespaceId}; use net::resource_thread::new_resource_threads; use net_traits::IpcSend; From ba9cf85fb3c1697998abab92488a0ab7ecdf431d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 25 Jun 2019 19:42:47 +0200 Subject: [PATCH 23/31] Webrender external image handler demux --- Cargo.lock | 12 +++ components/canvas/Cargo.toml | 1 + components/canvas/webgl_mode/inprocess.rs | 23 +++-- components/canvas/webgl_thread.rs | 47 ---------- components/media/Cargo.toml | 1 + components/media/lib.rs | 16 ++-- components/media/media_thread.rs | 58 ------------ components/servo/Cargo.toml | 1 + components/servo/lib.rs | 14 ++- components/webrender_traits/Cargo.toml | 17 ++++ components/webrender_traits/lib.rs | 108 ++++++++++++++++++++++ 11 files changed, 172 insertions(+), 126 deletions(-) create mode 100644 components/webrender_traits/Cargo.toml create mode 100644 components/webrender_traits/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 864ca86d567..c702bd07cd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -410,6 +410,7 @@ dependencies = [ "servo_config 0.0.1", "webrender 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", "webrender_api 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", + "webrender_traits 0.0.1", ] [[package]] @@ -2539,6 +2540,7 @@ dependencies = [ "webdriver_server 0.0.1", "webrender 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", "webrender_api 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", + "webrender_traits 0.0.1", "webvr 0.0.1", "webvr_traits 0.0.1", "webxr-api 0.0.1 (git+https://github.com/servo/webxr)", @@ -2725,6 +2727,7 @@ dependencies = [ "servo_config 0.0.1", "webrender 0.60.0 (git+https://github.com/servo/webrender)", "webrender_api 0.60.0 (git+https://github.com/servo/webrender)", + "webrender_traits 0.0.1", ] [[package]] @@ -5399,6 +5402,15 @@ dependencies = [ "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "webrender_traits" +version = "0.0.1" +dependencies = [ + "euclid 0.19.8 (registry+https://github.com/rust-lang/crates.io-index)", + "webrender 0.60.0 (git+https://github.com/servo/webrender)", + "webrender_api 0.60.0 (git+https://github.com/servo/webrender)", +] + [[package]] name = "webvr" version = "0.0.1" diff --git a/components/canvas/Cargo.toml b/components/canvas/Cargo.toml index 906fa84b434..8211619ca91 100644 --- a/components/canvas/Cargo.toml +++ b/components/canvas/Cargo.toml @@ -37,3 +37,4 @@ serde_bytes = "0.10" servo_config = {path = "../config"} webrender = {git = "https://github.com/servo/webrender"} webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]} +webrender_traits = {path = "../webrender_traits"} diff --git a/components/canvas/webgl_mode/inprocess.rs b/components/canvas/webgl_mode/inprocess.rs index b93b2fb0a19..6e53879c501 100644 --- a/components/canvas/webgl_mode/inprocess.rs +++ b/components/canvas/webgl_mode/inprocess.rs @@ -3,7 +3,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ use crate::gl_context::GLContextFactory; -use crate::webgl_thread::{WebGLExternalImageApi, WebGLExternalImageHandler, WebGLThread}; +use crate::webgl_thread::WebGLThread; use canvas_traits::webgl::webgl_channel; use canvas_traits::webgl::DOMToTextureCommand; use canvas_traits::webgl::{WebGLChan, WebGLContextId, WebGLMsg, WebGLPipeline, WebGLReceiver}; @@ -13,6 +13,7 @@ use fnv::FnvHashMap; use gleam::gl; use servo_config::pref; use std::rc::Rc; +use webrender_traits::WebrenderExternalImageApi; /// WebGL Threading API entry point that lives in the constellation. pub struct WebGLThreads(WebGLSender); @@ -26,7 +27,7 @@ impl WebGLThreads { webvr_compositor: Option>, ) -> ( WebGLThreads, - Box, + Box, Option>, ) { // This implementation creates a single `WebGLThread` for all the pipelines. @@ -43,8 +44,7 @@ impl WebGLThreads { } else { None }; - let external = - WebGLExternalImageHandler::new(WebGLExternalImages::new(webrender_gl, channel.clone())); + let external = WebGLExternalImages::new(webrender_gl, channel.clone()); ( WebGLThreads(channel), Box::new(external), @@ -87,12 +87,15 @@ impl WebGLExternalImages { } } -impl WebGLExternalImageApi for WebGLExternalImages { - fn lock(&mut self, ctx_id: WebGLContextId) -> (u32, Size2D) { +impl WebrenderExternalImageApi for WebGLExternalImages { + fn lock(&mut self, id: u64) -> (u32, Size2D) { // WebGL Thread has it's own GL command queue that we need to synchronize with the WR GL command queue. // The WebGLMsg::Lock message inserts a fence in the WebGL command queue. self.webgl_channel - .send(WebGLMsg::Lock(ctx_id, self.lock_channel.0.clone())) + .send(WebGLMsg::Lock( + WebGLContextId(id as usize), + self.lock_channel.0.clone(), + )) .unwrap(); let (image_id, size, gl_sync) = self.lock_channel.1.recv().unwrap(); // The next glWaitSync call is run on the WR thread and it's used to synchronize the two @@ -103,8 +106,10 @@ impl WebGLExternalImageApi for WebGLExternalImages { (image_id, size) } - fn unlock(&mut self, ctx_id: WebGLContextId) { - self.webgl_channel.send(WebGLMsg::Unlock(ctx_id)).unwrap(); + fn unlock(&mut self, id: u64) { + self.webgl_channel + .send(WebGLMsg::Unlock(WebGLContextId(id as usize))) + .unwrap(); } } diff --git a/components/canvas/webgl_thread.rs b/components/canvas/webgl_thread.rs index cec8cc929d6..229a7f3eca3 100644 --- a/components/canvas/webgl_thread.rs +++ b/components/canvas/webgl_thread.rs @@ -730,53 +730,6 @@ struct WebGLContextInfo { render_state: ContextRenderState, } -/// This trait is used as a bridge between the `WebGLThreads` implementation and -/// the WR ExternalImageHandler API implemented in the `WebGLExternalImageHandler` struct. -/// `WebGLExternalImageHandler` takes care of type conversions between WR and WebGL info (e.g keys, uvs). -/// It uses this trait to notify lock/unlock messages and get the required info that WR needs. -/// `WebGLThreads` receives lock/unlock message notifications and takes care of sending -/// the unlock/lock messages to the appropiate `WebGLThread`. -pub trait WebGLExternalImageApi { - fn lock(&mut self, ctx_id: WebGLContextId) -> (u32, Size2D); - fn unlock(&mut self, ctx_id: WebGLContextId); -} - -/// WebRender External Image Handler implementation -pub struct WebGLExternalImageHandler { - handler: T, -} - -impl WebGLExternalImageHandler { - pub fn new(handler: T) -> Self { - Self { handler: handler } - } -} - -impl webrender::ExternalImageHandler for WebGLExternalImageHandler { - /// Lock the external image. Then, WR could start to read the image content. - /// The WR client should not change the image content until the unlock() call. - fn lock( - &mut self, - key: webrender_api::ExternalImageId, - _channel_index: u8, - _rendering: webrender_api::ImageRendering, - ) -> webrender::ExternalImage { - let ctx_id = WebGLContextId(key.0 as _); - let (texture_id, size) = self.handler.lock(ctx_id); - - webrender::ExternalImage { - uv: webrender_api::TexelRect::new(0.0, size.height as f32, size.width as f32, 0.0), - source: webrender::ExternalImageSource::NativeTexture(texture_id), - } - } - /// Unlock the external image. The WR should not read the image content - /// after this call. - fn unlock(&mut self, key: webrender_api::ExternalImageId, _channel_index: u8) { - let ctx_id = WebGLContextId(key.0 as _); - self.handler.unlock(ctx_id); - } -} - /// Data about the linked DOM<->WebGLTexture elements. struct DOMToTextureData { context_id: WebGLContextId, diff --git a/components/media/Cargo.toml b/components/media/Cargo.toml index 53f531a26cf..254b4246f7e 100644 --- a/components/media/Cargo.toml +++ b/components/media/Cargo.toml @@ -21,3 +21,4 @@ servo_config = {path = "../config"} servo-media = {git = "https://github.com/servo/media"} webrender = {git = "https://github.com/servo/webrender"} webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]} +webrender_traits = {path = "../webrender_traits"} diff --git a/components/media/lib.rs b/components/media/lib.rs index c98d97df597..69697c3cecb 100644 --- a/components/media/lib.rs +++ b/components/media/lib.rs @@ -13,16 +13,15 @@ extern crate log; #[macro_use] extern crate serde; -use euclid::Size2D; -use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; - mod media_channel; mod media_thread; pub use crate::media_channel::glplayer_channel; - use crate::media_channel::{GLPlayerChan, GLPlayerPipeline, GLPlayerReceiver, GLPlayerSender}; -use crate::media_thread::{GLPlayerExternalImageApi, GLPlayerExternalImageHandler, GLPlayerThread}; +use crate::media_thread::GLPlayerThread; +use euclid::Size2D; +use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; +use webrender_traits::WebrenderExternalImageApi; /// These are the messages that the GLPlayer thread will forward to /// the video player which lives in htmlmediaelement @@ -100,10 +99,9 @@ impl PlayerGLContext for WindowGLContext { pub struct GLPlayerThreads(GLPlayerSender); impl GLPlayerThreads { - pub fn new() -> (GLPlayerThreads, Box) { + pub fn new() -> (GLPlayerThreads, Box) { let channel = GLPlayerThread::start(); - let external = - GLPlayerExternalImageHandler::new(GLPlayerExternalImages::new(channel.clone())); + let external = GLPlayerExternalImages::new(channel.clone()); (GLPlayerThreads(channel), Box::new(external)) } @@ -146,7 +144,7 @@ impl GLPlayerExternalImages { } } -impl GLPlayerExternalImageApi for GLPlayerExternalImages { +impl WebrenderExternalImageApi for GLPlayerExternalImages { fn lock(&mut self, id: u64) -> (u32, Size2D) { // The GLPlayerMsgForward::Lock message inserts a fence in the // GLPlayer command queue. diff --git a/components/media/media_thread.rs b/components/media/media_thread.rs index e47e7c799f9..09d5a00b69b 100644 --- a/components/media/media_thread.rs +++ b/components/media/media_thread.rs @@ -6,7 +6,6 @@ use crate::media_channel::{glplayer_channel, GLPlayerSender}; /// GL player threading API entry point that lives in the /// constellation. use crate::{GLPlayerMsg, GLPlayerMsgForward}; -use euclid::Size2D; use fnv::FnvHashMap; use std::thread; @@ -78,60 +77,3 @@ impl GLPlayerThread { false } } - -/// This trait is used as a bridge between the `GLPlayerThreads` -/// implementation and the WR ExternalImageHandler API implemented in -/// the `GLPlayerExternalImageHandler` struct. -// -/// `GLPlayerExternalImageHandler` takes care of type conversions -/// between WR and GLPlayer info (e.g keys, uvs). -// -/// It uses this trait to notify lock/unlock messages and get the -/// required info that WR needs. -// -/// `GLPlayerThreads` receives lock/unlock message notifications and -/// takes care of sending the unlock/lock messages to the appropiate -/// `GLPlayerThread`. -pub trait GLPlayerExternalImageApi { - fn lock(&mut self, id: u64) -> (u32, Size2D); - fn unlock(&mut self, id: u64); -} - -/// WebRender External Image Handler implementation -pub struct GLPlayerExternalImageHandler { - handler: T, -} - -impl GLPlayerExternalImageHandler { - pub fn new(handler: T) -> Self { - Self { handler: handler } - } -} - -impl webrender::ExternalImageHandler - for GLPlayerExternalImageHandler -{ - /// Lock the external image. Then, WR could start to read the - /// image content. - /// The WR client should not change the image content until the - /// unlock() call. - fn lock( - &mut self, - key: webrender_api::ExternalImageId, - _channel_index: u8, - _rendering: webrender_api::ImageRendering, - ) -> webrender::ExternalImage { - let (texture_id, size) = self.handler.lock(key.0); - - webrender::ExternalImage { - uv: webrender_api::TexelRect::new(0.0, 0.0, size.width as f32, size.height as f32), - source: webrender::ExternalImageSource::NativeTexture(texture_id), - } - } - - /// Unlock the external image. The WR should not read the image - /// content after this call. - fn unlock(&mut self, key: webrender_api::ExternalImageId, _channel_index: u8) { - self.handler.unlock(key.0); - } -} diff --git a/components/servo/Cargo.toml b/components/servo/Cargo.toml index 10ad4319651..5b0b8a651a9 100644 --- a/components/servo/Cargo.toml +++ b/components/servo/Cargo.toml @@ -72,6 +72,7 @@ style = {path = "../style", features = ["servo"]} style_traits = {path = "../style_traits", features = ["servo"]} webrender = {git = "https://github.com/servo/webrender"} webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]} +webrender_traits = {path = "../webrender_traits"} webdriver_server = {path = "../webdriver_server", optional = true} webvr = {path = "../webvr"} webvr_traits = {path = "../webvr_traits"} diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 1496e1ce054..5321e4720b8 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -50,6 +50,7 @@ pub use servo_url; pub use style; pub use style_traits; pub use webrender_api; +pub use webrender_traits; pub use webvr; pub use webvr_traits; @@ -112,6 +113,7 @@ use std::cmp::max; use std::path::PathBuf; use std::rc::Rc; use webrender::{RendererKind, ShaderPrecacheFlags}; +use webrender_traits::{WebrenderExternalImageHandler, WebrenderImageHandlerType}; use webvr::{VRServiceManager, WebVRCompositorHandler, WebVRThread}; pub use gleam::gl; @@ -688,9 +690,11 @@ fn create_constellation( GLContextFactory::current_native_handle(&compositor_proxy) }; + let mut webrender_external_image_handler = Box::new(WebrenderExternalImageHandler::new()); + // Initialize WebGL Thread entry point. let webgl_threads = gl_factory.map(|factory| { - let (webgl_threads, _image_handler, output_handler) = WebGLThreads::new( + let (webgl_threads, image_handler, output_handler) = WebGLThreads::new( factory, window_gl, webrender_api_sender.clone(), @@ -698,7 +702,8 @@ fn create_constellation( ); // Set webrender external image handler for WebGL textures - //webrender.set_external_image_handler(image_handler); + webrender_external_image_handler + .set_handler(image_handler, WebrenderImageHandlerType::WebGL); // Set DOM to texture handler, if enabled. if let Some(output_handler) = output_handler { @@ -712,11 +717,14 @@ fn create_constellation( GlContext::Unknown => None, _ => { let (glplayer_threads, image_handler) = GLPlayerThreads::new(); - webrender.set_external_image_handler(image_handler); + webrender_external_image_handler + .set_handler(image_handler, WebrenderImageHandlerType::Media); Some(glplayer_threads) }, }; + webrender.set_external_image_handler(webrender_external_image_handler); + let player_context = WindowGLContext { glplayer_chan: glplayer_threads.as_ref().map(|threads| threads.pipeline()), ..player_context diff --git a/components/webrender_traits/Cargo.toml b/components/webrender_traits/Cargo.toml new file mode 100644 index 00000000000..01e2ca2f2e3 --- /dev/null +++ b/components/webrender_traits/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "webrender_traits" +version = "0.0.1" +authors = ["The Servo Project Developers"] +license = "MPL-2.0" +edition = "2018" +publish = false + +[lib] +name = "webrender_traits" +path = "lib.rs" + +[dependencies] +euclid = "0.19" +webrender = {git = "https://github.com/servo/webrender"} +webrender_api = {git = "https://github.com/servo/webrender", features = ["ipc"]} + diff --git a/components/webrender_traits/lib.rs b/components/webrender_traits/lib.rs new file mode 100644 index 00000000000..7a2f3b4b69b --- /dev/null +++ b/components/webrender_traits/lib.rs @@ -0,0 +1,108 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +#![crate_name = "webrender_traits"] +#![crate_type = "rlib"] +#![deny(unsafe_code)] + +use euclid::Size2D; +use std::collections::HashMap; + +/// This trait is used as a bridge between the different GL clients +/// in Servo that handles WebRender ExternalImages and the WebRender +/// ExternalImageHandler API. +// +/// This trait is used to notify lock/unlock messages and get the +/// required info that WR needs. +pub trait WebrenderExternalImageApi { + fn lock(&mut self, id: u64) -> (u32, Size2D); + fn unlock(&mut self, id: u64); +} + +/// Type of Webrender External Image Handler. +pub enum WebrenderImageHandlerType { + WebGL, + Media, +} + +/// WebRender External Image Handler implementation. +pub struct WebrenderExternalImageHandler { + webgl_handler: Option>, + media_handler: Option>, + //XXX(ferjm) register external images. + external_images: HashMap, +} + +impl WebrenderExternalImageHandler { + pub fn new() -> Self { + Self { + webgl_handler: None, + media_handler: None, + external_images: HashMap::new(), + } + } + + pub fn set_handler( + &mut self, + handler: Box, + handler_type: WebrenderImageHandlerType, + ) { + match handler_type { + WebrenderImageHandlerType::WebGL => self.webgl_handler = Some(handler), + WebrenderImageHandlerType::Media => self.media_handler = Some(handler), + } + } +} + +impl webrender::ExternalImageHandler for WebrenderExternalImageHandler { + /// Lock the external image. Then, WR could start to read the + /// image content. + /// The WR client should not change the image content until the + /// unlock() call. + fn lock( + &mut self, + key: webrender_api::ExternalImageId, + _channel_index: u8, + _rendering: webrender_api::ImageRendering, + ) -> webrender::ExternalImage { + if let Some(handler_type) = self.external_images.get(&key) { + // It is safe to unwrap the handlers here because we forbid registration + // for specific types that has no handler set. + // XXX(ferjm) make this ^ true. + let (texture_id, size) = match handler_type { + WebrenderImageHandlerType::WebGL => { + self.webgl_handler.as_mut().unwrap().lock(key.0) + }, + WebrenderImageHandlerType::Media => { + self.media_handler.as_mut().unwrap().lock(key.0) + }, + }; + webrender::ExternalImage { + uv: webrender_api::TexelRect::new(0.0, 0.0, size.width as f32, size.height as f32), + source: webrender::ExternalImageSource::NativeTexture(texture_id), + } + } else { + unreachable!() + } + } + + /// Unlock the external image. The WR should not read the image + /// content after this call. + fn unlock(&mut self, key: webrender_api::ExternalImageId, _channel_index: u8) { + if let Some(handler_type) = self.external_images.get(&key) { + // It is safe to unwrap the handlers here because we forbid registration + // for specific types that has no handler set. + match handler_type { + WebrenderImageHandlerType::WebGL => { + self.webgl_handler.as_mut().unwrap().unlock(key.0) + }, + WebrenderImageHandlerType::Media => { + self.media_handler.as_mut().unwrap().unlock(key.0) + }, + }; + } else { + unreachable!(); + } + } +} From 0da87ad169ca33c8d2bce7a49de77d8821d5cac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Fri, 28 Jun 2019 18:08:08 +0200 Subject: [PATCH 24/31] Introduce WebrenderExternalImageRegistry --- components/canvas/webgl_mode/inprocess.rs | 5 +- components/canvas/webgl_thread.rs | 28 ++++++--- components/media/lib.rs | 9 ++- components/media/media_thread.rs | 29 ++++++--- components/servo/lib.rs | 16 ++--- components/webrender_traits/lib.rs | 71 ++++++++++++++++++----- 6 files changed, 115 insertions(+), 43 deletions(-) diff --git a/components/canvas/webgl_mode/inprocess.rs b/components/canvas/webgl_mode/inprocess.rs index 6e53879c501..d0203f7498b 100644 --- a/components/canvas/webgl_mode/inprocess.rs +++ b/components/canvas/webgl_mode/inprocess.rs @@ -13,7 +13,8 @@ use fnv::FnvHashMap; use gleam::gl; use servo_config::pref; use std::rc::Rc; -use webrender_traits::WebrenderExternalImageApi; +use std::sync::{Arc, Mutex}; +use webrender_traits::{WebrenderExternalImageApi, WebrenderExternalImageRegistry}; /// WebGL Threading API entry point that lives in the constellation. pub struct WebGLThreads(WebGLSender); @@ -25,6 +26,7 @@ impl WebGLThreads { webrender_gl: Rc, webrender_api_sender: webrender_api::RenderApiSender, webvr_compositor: Option>, + external_images: Arc>, ) -> ( WebGLThreads, Box, @@ -35,6 +37,7 @@ impl WebGLThreads { gl_factory, webrender_api_sender, webvr_compositor.map(|c| WebVRRenderWrapper(c)), + external_images, ); let output_handler = if pref!(dom.webgl.dom_to_texture.enabled) { Some(Box::new(OutputHandler::new( diff --git a/components/canvas/webgl_thread.rs b/components/canvas/webgl_thread.rs index 229a7f3eca3..4b2e59dd216 100644 --- a/components/canvas/webgl_thread.rs +++ b/components/canvas/webgl_thread.rs @@ -12,7 +12,9 @@ use half::f16; use offscreen_gl_context::{DrawBuffer, GLContext, NativeGLContextMethods}; use pixels::{self, PixelFormat}; use std::borrow::Cow; +use std::sync::{Arc, Mutex}; use std::thread; +use webrender_traits::{WebrenderExternalImageRegistry, WebrenderImageHandlerType}; /// WebGL Threading API entry point that lives in the constellation. /// It allows to get a WebGLThread handle for each script pipeline. @@ -58,12 +60,13 @@ pub struct WebGLThread { cached_context_info: FnvHashMap, /// Current bound context. bound_context_id: Option, - /// Id generator for new WebGLContexts. - next_webgl_id: usize, /// Handler user to send WebVR commands. webvr_compositor: Option, /// Texture ids and sizes used in DOM to texture outputs. dom_outputs: FnvHashMap, + /// List of registered webrender external images. + /// We use it to get an unique ID for new WebGLContexts. + external_images: Arc>, } impl WebGLThread { @@ -71,6 +74,7 @@ impl WebGLThread { gl_factory: GLContextFactory, webrender_api_sender: webrender_api::RenderApiSender, webvr_compositor: Option, + external_images: Arc>, ) -> Self { WebGLThread { gl_factory, @@ -78,9 +82,9 @@ impl WebGLThread { contexts: Default::default(), cached_context_info: Default::default(), bound_context_id: None, - next_webgl_id: 0, webvr_compositor, dom_outputs: Default::default(), + external_images, } } @@ -90,14 +94,19 @@ impl WebGLThread { gl_factory: GLContextFactory, webrender_api_sender: webrender_api::RenderApiSender, webvr_compositor: Option, + external_images: Arc>, ) -> WebGLSender { let (sender, receiver) = webgl_channel::().unwrap(); let result = sender.clone(); thread::Builder::new() .name("WebGLThread".to_owned()) .spawn(move || { - let mut renderer = - WebGLThread::new(gl_factory, webrender_api_sender, webvr_compositor); + let mut renderer = WebGLThread::new( + gl_factory, + webrender_api_sender, + webvr_compositor, + external_images, + ); let webgl_chan = WebGLChan(sender); loop { let msg = receiver.recv().unwrap(); @@ -287,9 +296,14 @@ impl WebGLThread { }) .map_err(|msg: &str| msg.to_owned())?; - let id = WebGLContextId(self.next_webgl_id); + let id = WebGLContextId( + self.external_images + .lock() + .unwrap() + .next_id(WebrenderImageHandlerType::WebGL) + .0 as usize, + ); let (size, texture_id, limits) = ctx.get_info(); - self.next_webgl_id += 1; self.contexts.insert( id, GLContextData { diff --git a/components/media/lib.rs b/components/media/lib.rs index 69697c3cecb..0f937125e7a 100644 --- a/components/media/lib.rs +++ b/components/media/lib.rs @@ -21,7 +21,8 @@ use crate::media_channel::{GLPlayerChan, GLPlayerPipeline, GLPlayerReceiver, GLP use crate::media_thread::GLPlayerThread; use euclid::Size2D; use servo_media::player::context::{GlApi, GlContext, NativeDisplay, PlayerGLContext}; -use webrender_traits::WebrenderExternalImageApi; +use std::sync::{Arc, Mutex}; +use webrender_traits::{WebrenderExternalImageApi, WebrenderExternalImageRegistry}; /// These are the messages that the GLPlayer thread will forward to /// the video player which lives in htmlmediaelement @@ -99,8 +100,10 @@ impl PlayerGLContext for WindowGLContext { pub struct GLPlayerThreads(GLPlayerSender); impl GLPlayerThreads { - pub fn new() -> (GLPlayerThreads, Box) { - let channel = GLPlayerThread::start(); + pub fn new( + external_images: Arc>, + ) -> (GLPlayerThreads, Box) { + let channel = GLPlayerThread::start(external_images); let external = GLPlayerExternalImages::new(channel.clone()); (GLPlayerThreads(channel), Box::new(external)) } diff --git a/components/media/media_thread.rs b/components/media/media_thread.rs index 09d5a00b69b..2d6e85573eb 100644 --- a/components/media/media_thread.rs +++ b/components/media/media_thread.rs @@ -7,31 +7,36 @@ use crate::media_channel::{glplayer_channel, GLPlayerSender}; /// constellation. use crate::{GLPlayerMsg, GLPlayerMsgForward}; use fnv::FnvHashMap; +use std::sync::{Arc, Mutex}; use std::thread; +use webrender_traits::{WebrenderExternalImageRegistry, WebrenderImageHandlerType}; /// A GLPlayerThrx1ead manages the life cycle and message multiplexign of /// a set of video players with GL render. pub struct GLPlayerThread { // Map of live players. players: FnvHashMap>, - /// Id generator for new WebGLContexts. - next_player_id: u64, + /// List of registered webrender external images. + /// We use it to get an unique ID for new players. + external_images: Arc>, } impl GLPlayerThread { - pub fn new() -> Self { + pub fn new(external_images: Arc>) -> Self { GLPlayerThread { players: Default::default(), - next_player_id: 1, + external_images, } } - pub fn start() -> GLPlayerSender { + pub fn start( + external_images: Arc>, + ) -> GLPlayerSender { let (sender, receiver) = glplayer_channel::().unwrap(); thread::Builder::new() .name("GLPlayerThread".to_owned()) .spawn(move || { - let mut renderer = GLPlayerThread::new(); + let mut renderer = GLPlayerThread::new(external_images); loop { let msg = receiver.recv().unwrap(); let exit = renderer.handle_msg(msg); @@ -51,12 +56,20 @@ impl GLPlayerThread { trace!("processing {:?}", msg); match msg { GLPlayerMsg::RegisterPlayer(sender) => { - let id = self.next_player_id; + let id = self + .external_images + .lock() + .unwrap() + .next_id(WebrenderImageHandlerType::Media) + .0; self.players.insert(id, sender.clone()); sender.send(GLPlayerMsgForward::PlayerId(id)).unwrap(); - self.next_player_id += 1; }, GLPlayerMsg::UnregisterPlayer(id) => { + self.external_images + .lock() + .unwrap() + .remove(&webrender_api::ExternalImageId(id)); if self.players.remove(&id).is_none() { warn!("Tried to remove an unknown player"); } diff --git a/components/servo/lib.rs b/components/servo/lib.rs index 5321e4720b8..0ba8fa478ef 100644 --- a/components/servo/lib.rs +++ b/components/servo/lib.rs @@ -113,7 +113,7 @@ use std::cmp::max; use std::path::PathBuf; use std::rc::Rc; use webrender::{RendererKind, ShaderPrecacheFlags}; -use webrender_traits::{WebrenderExternalImageHandler, WebrenderImageHandlerType}; +use webrender_traits::{WebrenderExternalImageHandlers, WebrenderImageHandlerType}; use webvr::{VRServiceManager, WebVRCompositorHandler, WebVRThread}; pub use gleam::gl; @@ -690,7 +690,8 @@ fn create_constellation( GLContextFactory::current_native_handle(&compositor_proxy) }; - let mut webrender_external_image_handler = Box::new(WebrenderExternalImageHandler::new()); + let (external_image_handlers, external_images) = WebrenderExternalImageHandlers::new(); + let mut external_image_handlers = Box::new(external_image_handlers); // Initialize WebGL Thread entry point. let webgl_threads = gl_factory.map(|factory| { @@ -699,11 +700,11 @@ fn create_constellation( window_gl, webrender_api_sender.clone(), webvr_compositor.map(|c| c as Box<_>), + external_images.clone(), ); // Set webrender external image handler for WebGL textures - webrender_external_image_handler - .set_handler(image_handler, WebrenderImageHandlerType::WebGL); + external_image_handlers.set_handler(image_handler, WebrenderImageHandlerType::WebGL); // Set DOM to texture handler, if enabled. if let Some(output_handler) = output_handler { @@ -716,14 +717,13 @@ fn create_constellation( let glplayer_threads = match player_context.gl_context { GlContext::Unknown => None, _ => { - let (glplayer_threads, image_handler) = GLPlayerThreads::new(); - webrender_external_image_handler - .set_handler(image_handler, WebrenderImageHandlerType::Media); + let (glplayer_threads, image_handler) = GLPlayerThreads::new(external_images); + external_image_handlers.set_handler(image_handler, WebrenderImageHandlerType::Media); Some(glplayer_threads) }, }; - webrender.set_external_image_handler(webrender_external_image_handler); + webrender.set_external_image_handler(external_image_handlers); let player_context = WindowGLContext { glplayer_chan: glplayer_threads.as_ref().map(|threads| threads.pipeline()), diff --git a/components/webrender_traits/lib.rs b/components/webrender_traits/lib.rs index 7a2f3b4b69b..8add48b2985 100644 --- a/components/webrender_traits/lib.rs +++ b/components/webrender_traits/lib.rs @@ -8,6 +8,7 @@ use euclid::Size2D; use std::collections::HashMap; +use std::sync::{Arc, Mutex}; /// This trait is used as a bridge between the different GL clients /// in Servo that handles WebRender ExternalImages and the WebRender @@ -26,23 +27,66 @@ pub enum WebrenderImageHandlerType { Media, } -/// WebRender External Image Handler implementation. -pub struct WebrenderExternalImageHandler { - webgl_handler: Option>, - media_handler: Option>, - //XXX(ferjm) register external images. +/// List of Webrender external images to be shared among all external image +/// consumers (WebGL, Media). +/// It ensures that external image identifiers are unique. +pub struct WebrenderExternalImageRegistry { + /// Map of all generated external images. external_images: HashMap, + /// Id generator for the next external image identifier. + next_image_id: u64, } -impl WebrenderExternalImageHandler { +impl WebrenderExternalImageRegistry { pub fn new() -> Self { Self { - webgl_handler: None, - media_handler: None, external_images: HashMap::new(), + next_image_id: 0, } } + pub fn next_id( + &mut self, + handler_type: WebrenderImageHandlerType, + ) -> webrender_api::ExternalImageId { + self.next_image_id += 1; + let key = webrender_api::ExternalImageId(self.next_image_id); + self.external_images.insert(key, handler_type); + key + } + + pub fn remove(&mut self, key: &webrender_api::ExternalImageId) { + self.external_images.remove(key); + } + + pub fn get(&self, key: &webrender_api::ExternalImageId) -> Option<&WebrenderImageHandlerType> { + self.external_images.get(key) + } +} + +/// WebRender External Image Handler implementation. +pub struct WebrenderExternalImageHandlers { + /// WebGL handler. + webgl_handler: Option>, + /// Media player handler. + media_handler: Option>, + /// Webrender external images. + external_images: Arc>, +} + +impl WebrenderExternalImageHandlers { + pub fn new() -> (Self, Arc>) { + let external_images = Arc::new(Mutex::new(WebrenderExternalImageRegistry::new())); + ( + Self { + webgl_handler: None, + media_handler: None, + external_images: external_images.clone(), + }, + external_images, + ) + } + pub fn set_handler( &mut self, handler: Box, @@ -55,7 +99,7 @@ impl WebrenderExternalImageHandler { } } -impl webrender::ExternalImageHandler for WebrenderExternalImageHandler { +impl webrender::ExternalImageHandler for WebrenderExternalImageHandlers { /// Lock the external image. Then, WR could start to read the /// image content. /// The WR client should not change the image content until the @@ -66,10 +110,7 @@ impl webrender::ExternalImageHandler for WebrenderExternalImageHandler { _channel_index: u8, _rendering: webrender_api::ImageRendering, ) -> webrender::ExternalImage { - if let Some(handler_type) = self.external_images.get(&key) { - // It is safe to unwrap the handlers here because we forbid registration - // for specific types that has no handler set. - // XXX(ferjm) make this ^ true. + if let Some(handler_type) = self.external_images.lock().unwrap().get(&key) { let (texture_id, size) = match handler_type { WebrenderImageHandlerType::WebGL => { self.webgl_handler.as_mut().unwrap().lock(key.0) @@ -90,9 +131,7 @@ impl webrender::ExternalImageHandler for WebrenderExternalImageHandler { /// Unlock the external image. The WR should not read the image /// content after this call. fn unlock(&mut self, key: webrender_api::ExternalImageId, _channel_index: u8) { - if let Some(handler_type) = self.external_images.get(&key) { - // It is safe to unwrap the handlers here because we forbid registration - // for specific types that has no handler set. + if let Some(handler_type) = self.external_images.lock().unwrap().get(&key) { match handler_type { WebrenderImageHandlerType::WebGL => { self.webgl_handler.as_mut().unwrap().unlock(key.0) From cd17b6ca666b22248f3ce44d2c071dc6c84a2499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Mon, 1 Jul 2019 16:48:18 +0200 Subject: [PATCH 25/31] Fix typos, warnings and other nits --- components/media/lib.rs | 4 +- components/media/media_thread.rs | 6 +- components/script/dom/bindings/trace.rs | 2 + components/script/dom/htmlmediaelement.rs | 68 ++++++++--------------- components/script/dom/window.rs | 2 +- components/script/script_thread.rs | 4 +- ports/glutin/context.rs | 2 + 7 files changed, 35 insertions(+), 53 deletions(-) diff --git a/components/media/lib.rs b/components/media/lib.rs index 0f937125e7a..fa07399af86 100644 --- a/components/media/lib.rs +++ b/components/media/lib.rs @@ -35,8 +35,8 @@ pub enum GLPlayerMsgForward { /// GLPlayer thread Message API /// -/// These are the message that the thread will receive from the -/// constellation, the webrender::ExternalImageHandle multiplexor +/// These are the messages that the thread will receive from the +/// constellation, the webrender::ExternalImageHandle demultiplexor /// implementation, or a htmlmediaelement #[derive(Debug, Deserialize, Serialize)] pub enum GLPlayerMsg { diff --git a/components/media/media_thread.rs b/components/media/media_thread.rs index 2d6e85573eb..9dc3bf625f1 100644 --- a/components/media/media_thread.rs +++ b/components/media/media_thread.rs @@ -11,10 +11,10 @@ use std::sync::{Arc, Mutex}; use std::thread; use webrender_traits::{WebrenderExternalImageRegistry, WebrenderImageHandlerType}; -/// A GLPlayerThrx1ead manages the life cycle and message multiplexign of +/// A GLPlayerThread manages the life cycle and message demultiplexing of /// a set of video players with GL render. pub struct GLPlayerThread { - // Map of live players. + /// Map of live players. players: FnvHashMap>, /// List of registered webrender external images. /// We use it to get an unique ID for new players. @@ -50,7 +50,7 @@ impl GLPlayerThread { sender } - /// Handles a generic WebGLMsg message + /// Handles a generic GLPlayerMsg message #[inline] fn handle_msg(&mut self, msg: GLPlayerMsg) -> bool { trace!("processing {:?}", msg); diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 7d5d0b0b093..d9e0cf893b6 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -73,6 +73,7 @@ use js::jsval::JSVal; use js::rust::{GCMethods, Handle, Runtime}; use js::typedarray::TypedArray; use js::typedarray::TypedArrayElement; +use media::WindowGLContext; use metrics::{InteractiveMetrics, InteractiveWindow}; use mime::Mime; use msg::constellation_msg::{ @@ -507,6 +508,7 @@ unsafe_no_jsmanaged_fields!(Rotation3D, Transform2D, Transform3D) unsafe_no_jsmanaged_fields!(Point2D, Vector2D, Rect); unsafe_no_jsmanaged_fields!(Rect, RigidTransform3D); unsafe_no_jsmanaged_fields!(CascadeData); +unsafe_no_jsmanaged_fields!(WindowGLContext); unsafe impl<'a> JSTraceable for &'a str { #[inline] diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 6b1191d6373..1d7dfebd832 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -133,7 +133,7 @@ impl FrameHolder { } pub struct MediaFrameRenderer { - id: u64, + player_id: Option, api: RenderApi, current_frame: Option<(ImageKey, i32, i32)>, old_frame: Option, @@ -144,7 +144,7 @@ pub struct MediaFrameRenderer { impl MediaFrameRenderer { fn new(render_api_sender: RenderApiSender) -> Self { Self { - id: 0, + player_id: None, api: render_api_sender.create_api(), current_frame: None, old_frame: None, @@ -187,7 +187,7 @@ impl FrameRenderer for MediaFrameRenderer { ImageData::Raw(frame.get_data()), &webrender_api::DirtyRect::All, ); - } else if self.id != 0 { + } else if self.player_id.is_some() { self.current_frame_holder .get_or_insert_with(|| FrameHolder::new(frame.clone())) .set(frame); @@ -207,55 +207,35 @@ impl FrameRenderer for MediaFrameRenderer { *width = frame.get_width(); *height = frame.get_height(); - if !frame.is_gl_texture() { - txn.add_image( - new_image_key, - descriptor, - ImageData::Raw(frame.get_data()), - None, - ); - } else if self.id != 0 { - txn.add_image( - new_image_key, - descriptor, - ImageData::External(ExternalImageData { - id: ExternalImageId(self.id), - channel_index: 0, - image_type: ExternalImageType::TextureHandle(TextureTarget::Default), - }), - None, - ); - + let image_data = if let Some(player_id) = self.player_id { self.current_frame_holder .get_or_insert_with(|| FrameHolder::new(frame.clone())) .set(frame); - } + ImageData::External(ExternalImageData { + id: ExternalImageId(player_id), + channel_index: 0, + image_type: ExternalImageType::TextureHandle(TextureTarget::Default), + }) + } else { + ImageData::Raw(frame.get_data()) + }; + txn.add_image(new_image_key, descriptor, image_data, None); }, None => { let image_key = self.api.generate_image_key(); self.current_frame = Some((image_key, frame.get_width(), frame.get_height())); - if !frame.is_gl_texture() { - txn.add_image( - image_key, - descriptor, - ImageData::Raw(frame.get_data()), - None, - ); - } else if self.id != 0 { - txn.add_image( - image_key, - descriptor, - ImageData::External(ExternalImageData { - id: ExternalImageId(self.id), - channel_index: 0, - image_type: ExternalImageType::TextureHandle(TextureTarget::Default), - }), - None, - ); - + let image_data = if let Some(player_id) = self.player_id { self.current_frame_holder = Some(FrameHolder::new(frame)); - } + ImageData::External(ExternalImageData { + id: ExternalImageId(player_id), + channel_index: 0, + image_type: ExternalImageType::TextureHandle(TextureTarget::Default), + }) + } else { + ImageData::Raw(frame.get_data()) + }; + txn.add_image(image_key, descriptor, image_data, None); }, } self.api.update_resources(txn.resource_updates); @@ -1371,7 +1351,7 @@ impl HTMLMediaElement { .unwrap_or((0, None)); self.id.set(player_id); - self.frame_renderer.lock().unwrap().id = player_id; + self.frame_renderer.lock().unwrap().player_id = Some(player_id); if let Some(image_receiver) = image_receiver { let trusted_node = Trusted::new(self); diff --git a/components/script/dom/window.rs b/components/script/dom/window.rs index c5866bc5e6b..8ca1e324511 100644 --- a/components/script/dom/window.rs +++ b/components/script/dom/window.rs @@ -319,7 +319,7 @@ pub struct Window { /// Replace unpaired surrogates in DOM strings with U+FFFD. /// See replace_surrogates: bool, - + /// Window's GL context from application #[ignore_malloc_size_of = "defined in script_thread"] player_context: WindowGLContext, diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index fc9498a8ebb..e6d91a1bc2e 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -513,8 +513,6 @@ unsafe_no_jsmanaged_fields!(TaskQueue); unsafe_no_jsmanaged_fields!(dyn BackgroundHangMonitorRegister); unsafe_no_jsmanaged_fields!(dyn BackgroundHangMonitor); -unsafe_no_jsmanaged_fields!(WindowGLContext); - #[derive(JSTraceable)] // ScriptThread instances are rooted on creation, so this is okay #[allow(unrooted_must_root)] @@ -681,7 +679,7 @@ pub struct ScriptThread { /// An optional string allowing the user agent to be set for testing. user_agent: Cow<'static, str>, - + /// Application window's GL Context for Media player player_context: WindowGLContext, } diff --git a/ports/glutin/context.rs b/ports/glutin/context.rs index 5e62c2b3e92..39749f82b90 100644 --- a/ports/glutin/context.rs +++ b/ports/glutin/context.rs @@ -74,6 +74,7 @@ impl GlContext { GlContext::None => unreachable!(), }; } + #[allow(unreachable_code, unused_variables)] pub fn raw_context(&self) -> RawContext { match self { GlContext::Current(c) => { @@ -135,6 +136,7 @@ impl GlContext { } } + #[allow(dead_code)] pub fn egl_display(&self) -> Option<*const raw::c_void> { match self { GlContext::Current(c) => unsafe { c.get_egl_display() }, From 89dc0119f00c4dff5a14f0096f4dcc9c19027b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 2 Jul 2019 13:19:02 +0200 Subject: [PATCH 26/31] Do not use WR external images if frames are not textures --- components/script/dom/htmlmediaelement.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 1d7dfebd832..690692a2481 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -207,12 +207,12 @@ impl FrameRenderer for MediaFrameRenderer { *width = frame.get_width(); *height = frame.get_height(); - let image_data = if let Some(player_id) = self.player_id { + let image_data = if frame.is_gl_texture() && self.player_id.is_some() { self.current_frame_holder .get_or_insert_with(|| FrameHolder::new(frame.clone())) .set(frame); ImageData::External(ExternalImageData { - id: ExternalImageId(player_id), + id: ExternalImageId(self.player_id.unwrap()), channel_index: 0, image_type: ExternalImageType::TextureHandle(TextureTarget::Default), }) @@ -225,10 +225,10 @@ impl FrameRenderer for MediaFrameRenderer { let image_key = self.api.generate_image_key(); self.current_frame = Some((image_key, frame.get_width(), frame.get_height())); - let image_data = if let Some(player_id) = self.player_id { + let image_data = if frame.is_gl_texture() && self.player_id.is_some() { self.current_frame_holder = Some(FrameHolder::new(frame)); ImageData::External(ExternalImageData { - id: ExternalImageId(player_id), + id: ExternalImageId(self.player_id.unwrap()), channel_index: 0, image_type: ExternalImageType::TextureHandle(TextureTarget::Default), }) From 63920da347ab538d669bcb77972a22abcdafa806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Tue, 2 Jul 2019 15:27:58 +0200 Subject: [PATCH 27/31] Differentiate texel space coordinates for webgl and media --- components/webrender_traits/lib.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/components/webrender_traits/lib.rs b/components/webrender_traits/lib.rs index 8add48b2985..1382c2ec66f 100644 --- a/components/webrender_traits/lib.rs +++ b/components/webrender_traits/lib.rs @@ -111,16 +111,34 @@ impl webrender::ExternalImageHandler for WebrenderExternalImageHandlers { _rendering: webrender_api::ImageRendering, ) -> webrender::ExternalImage { if let Some(handler_type) = self.external_images.lock().unwrap().get(&key) { - let (texture_id, size) = match handler_type { + let (texture_id, uv) = match handler_type { WebrenderImageHandlerType::WebGL => { - self.webgl_handler.as_mut().unwrap().lock(key.0) + let (texture_id, size) = self.webgl_handler.as_mut().unwrap().lock(key.0); + ( + texture_id, + webrender_api::TexelRect::new( + 0.0, + size.height as f32, + size.width as f32, + 0.0, + ), + ) }, WebrenderImageHandlerType::Media => { - self.media_handler.as_mut().unwrap().lock(key.0) + let (texture_id, size) = self.media_handler.as_mut().unwrap().lock(key.0); + ( + texture_id, + webrender_api::TexelRect::new( + 0.0, + 0.0, + size.width as f32, + size.height as f32, + ), + ) }, }; webrender::ExternalImage { - uv: webrender_api::TexelRect::new(0.0, 0.0, size.width as f32, size.height as f32), + uv, source: webrender::ExternalImageSource::NativeTexture(texture_id), } } else { From 208473cdbce1e125338b461a3f0d55fb9d0bdd7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 4 Jul 2019 10:22:57 +0200 Subject: [PATCH 28/31] Final tweaks: use expect and remove unnecessary crate attributes --- Cargo.lock | 8 +-- components/constellation/constellation.rs | 2 +- components/constellation/pipeline.rs | 2 +- components/media/lib.rs | 2 - components/webrender_traits/lib.rs | 78 +++++++++-------------- 5 files changed, 37 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c702bd07cd0..22670c118f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2725,8 +2725,8 @@ dependencies = [ "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "servo-media 0.1.0 (git+https://github.com/servo/media)", "servo_config 0.0.1", - "webrender 0.60.0 (git+https://github.com/servo/webrender)", - "webrender_api 0.60.0 (git+https://github.com/servo/webrender)", + "webrender 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", + "webrender_api 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", "webrender_traits 0.0.1", ] @@ -5407,8 +5407,8 @@ name = "webrender_traits" version = "0.0.1" dependencies = [ "euclid 0.19.8 (registry+https://github.com/rust-lang/crates.io-index)", - "webrender 0.60.0 (git+https://github.com/servo/webrender)", - "webrender_api 0.60.0 (git+https://github.com/servo/webrender)", + "webrender 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", + "webrender_api 0.60.0 (git+https://github.com/jdm/webrender?branch=servo-hl)", ] [[package]] diff --git a/components/constellation/constellation.rs b/components/constellation/constellation.rs index 1d6bef75b4e..77fd77ab307 100644 --- a/components/constellation/constellation.rs +++ b/components/constellation/constellation.rs @@ -464,7 +464,7 @@ pub struct InitialConstellationState { /// The XR device registry pub webxr_registry: webxr_api::Registry, - + pub glplayer_threads: Option, /// Application window's GL Context for Media player diff --git a/components/constellation/pipeline.rs b/components/constellation/pipeline.rs index 07df7a4b2c9..f3acb29ad14 100644 --- a/components/constellation/pipeline.rs +++ b/components/constellation/pipeline.rs @@ -192,7 +192,7 @@ pub struct InitialPipelineState { /// The XR device registry pub webxr_registry: webxr_api::Registry, - + /// Application window's GL Context for Media player pub player_context: WindowGLContext, } diff --git a/components/media/lib.rs b/components/media/lib.rs index fa07399af86..7c339693da2 100644 --- a/components/media/lib.rs +++ b/components/media/lib.rs @@ -2,8 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#![crate_name = "media"] -#![crate_type = "rlib"] #![deny(unsafe_code)] #[macro_use] diff --git a/components/webrender_traits/lib.rs b/components/webrender_traits/lib.rs index 1382c2ec66f..40d9f0956c3 100644 --- a/components/webrender_traits/lib.rs +++ b/components/webrender_traits/lib.rs @@ -2,8 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -#![crate_name = "webrender_traits"] -#![crate_type = "rlib"] #![deny(unsafe_code)] use euclid::Size2D; @@ -110,56 +108,42 @@ impl webrender::ExternalImageHandler for WebrenderExternalImageHandlers { _channel_index: u8, _rendering: webrender_api::ImageRendering, ) -> webrender::ExternalImage { - if let Some(handler_type) = self.external_images.lock().unwrap().get(&key) { - let (texture_id, uv) = match handler_type { - WebrenderImageHandlerType::WebGL => { - let (texture_id, size) = self.webgl_handler.as_mut().unwrap().lock(key.0); - ( - texture_id, - webrender_api::TexelRect::new( - 0.0, - size.height as f32, - size.width as f32, - 0.0, - ), - ) - }, - WebrenderImageHandlerType::Media => { - let (texture_id, size) = self.media_handler.as_mut().unwrap().lock(key.0); - ( - texture_id, - webrender_api::TexelRect::new( - 0.0, - 0.0, - size.width as f32, - size.height as f32, - ), - ) - }, - }; - webrender::ExternalImage { - uv, - source: webrender::ExternalImageSource::NativeTexture(texture_id), - } - } else { - unreachable!() + let external_images = self.external_images.lock().unwrap(); + let handler_type = external_images + .get(&key) + .expect("Tried to get unknown external image"); + let (texture_id, uv) = match handler_type { + WebrenderImageHandlerType::WebGL => { + let (texture_id, size) = self.webgl_handler.as_mut().unwrap().lock(key.0); + ( + texture_id, + webrender_api::TexelRect::new(0.0, size.height as f32, size.width as f32, 0.0), + ) + }, + WebrenderImageHandlerType::Media => { + let (texture_id, size) = self.media_handler.as_mut().unwrap().lock(key.0); + ( + texture_id, + webrender_api::TexelRect::new(0.0, 0.0, size.width as f32, size.height as f32), + ) + }, + }; + webrender::ExternalImage { + uv, + source: webrender::ExternalImageSource::NativeTexture(texture_id), } } /// Unlock the external image. The WR should not read the image /// content after this call. fn unlock(&mut self, key: webrender_api::ExternalImageId, _channel_index: u8) { - if let Some(handler_type) = self.external_images.lock().unwrap().get(&key) { - match handler_type { - WebrenderImageHandlerType::WebGL => { - self.webgl_handler.as_mut().unwrap().unlock(key.0) - }, - WebrenderImageHandlerType::Media => { - self.media_handler.as_mut().unwrap().unlock(key.0) - }, - }; - } else { - unreachable!(); - } + let external_images = self.external_images.lock().unwrap(); + let handler_type = external_images + .get(&key) + .expect("Tried to get unknown external image"); + match handler_type { + WebrenderImageHandlerType::WebGL => self.webgl_handler.as_mut().unwrap().unlock(key.0), + WebrenderImageHandlerType::Media => self.media_handler.as_mut().unwrap().unlock(key.0), + }; } } From 728fdff72146663c3fb7da96674f0f9ad85c06ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 4 Jul 2019 16:30:21 +0200 Subject: [PATCH 29/31] Fix libsimpleservo build --- ports/libsimpleservo/api/src/lib.rs | 4 ++-- ports/libsimpleservo/capi/src/lib.rs | 23 ++++++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ports/libsimpleservo/api/src/lib.rs b/ports/libsimpleservo/api/src/lib.rs index 50284dc1111..2935852ad86 100644 --- a/ports/libsimpleservo/api/src/lib.rs +++ b/ports/libsimpleservo/api/src/lib.rs @@ -587,8 +587,8 @@ struct ServoWindowCallbacks { host_callbacks: Box, coordinates: RefCell, density: f32, - gl_context_pointer: Option<*const libc::c_void>, - native_display_pointer: Option<*const libc::c_void>, + gl_context_pointer: Option<*const c_void>, + native_display_pointer: Option<*const c_void>, } impl EmbedderMethods for ServoEmbedderCallbacks { diff --git a/ports/libsimpleservo/capi/src/lib.rs b/ports/libsimpleservo/capi/src/lib.rs index 89d621e3ffe..e415550e5f6 100644 --- a/ports/libsimpleservo/capi/src/lib.rs +++ b/ports/libsimpleservo/capi/src/lib.rs @@ -93,16 +93,18 @@ fn init_logger() { crate::env_logger::init(); } -fn init( +unsafe fn init( opts: CInitOptions, gl: gl_glue::ServoGl, + gl_context: Option<*const c_void>, + display: Option<*const c_void>, wakeup: extern "C" fn(), callbacks: CHostCallbacks, ) { init_logger(); let args = if !opts.args.is_null() { - let args = unsafe { CStr::from_ptr(opts.args) }; + let args = CStr::from_ptr(opts.args); args.to_str() .unwrap_or("") .split(' ') @@ -112,7 +114,7 @@ fn init( vec![] }; - let url = unsafe { CStr::from_ptr(opts.url) }; + let url = CStr::from_ptr(opts.url); let url = url.to_str().map(|s| s.to_string()).ok(); let coordinates = Coordinates::new(0, 0, opts.width, opts.height, opts.width, opts.height); @@ -128,6 +130,8 @@ fn init( VRInitOptions::VRExternal(opts.vr_pointer) }, enable_subpixel_text_antialiasing: opts.enable_subpixel_text_antialiasing, + gl_context_pointer: gl_context, + native_display_pointer: display, }; let wakeup = Box::new(WakeupCallback::new(wakeup)); @@ -145,7 +149,16 @@ pub extern "C" fn init_with_egl( ) { init_logger(); let gl = gl_glue::egl::init().unwrap(); - init(opts, gl, wakeup, callbacks) + unsafe { + init( + opts, + gl.gl_wrapper, + Some(gl.gl_context), + Some(gl.display), + wakeup, + callbacks, + ) + } } #[cfg(any(target_os = "linux", target_os = "windows", target_os = "macos"))] @@ -157,7 +170,7 @@ pub extern "C" fn init_with_gl( ) { init_logger(); let gl = gl_glue::gl::init().unwrap(); - init(opts, gl, wakeup, callbacks) + unsafe { init(opts, gl, None, None, wakeup, callbacks) } } #[no_mangle] From f3e237e5584f4f1399d04abc9d0f4765392bca33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Jim=C3=A9nez=20Moreno?= Date: Thu, 4 Jul 2019 16:47:08 +0200 Subject: [PATCH 30/31] Fix Windows build --- ports/libsimpleservo/api/src/gl_glue.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ports/libsimpleservo/api/src/gl_glue.rs b/ports/libsimpleservo/api/src/gl_glue.rs index 3f69ef15d46..bc982959930 100644 --- a/ports/libsimpleservo/api/src/gl_glue.rs +++ b/ports/libsimpleservo/api/src/gl_glue.rs @@ -33,7 +33,6 @@ pub mod egl { pub display: EGLNativeDisplayType, } - #[cfg(target_os = "android")] pub fn init() -> Result { info!("Loading EGL..."); unsafe { From 4fe129109bbc40a32e491825d77a67fa8e5e9662 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Thu, 4 Jul 2019 18:06:34 -0400 Subject: [PATCH 31/31] Fix magic leap build. --- ports/libmlservo/src/lib.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ports/libmlservo/src/lib.rs b/ports/libmlservo/src/lib.rs index b71900c17ea..56739a54f23 100644 --- a/ports/libmlservo/src/lib.rs +++ b/ports/libmlservo/src/lib.rs @@ -144,8 +144,8 @@ pub unsafe extern "C" fn init_servo( VRInitOptions::None } else { let name = String::from("Magic Leap VR Display"); - let (service, heartbeat) = - MagicLeapVRService::new(name, ctxt, gl.clone()).expect("Failed to create VR service"); + let (service, heartbeat) = MagicLeapVRService::new(name, ctxt, gl.gl_wrapper.clone()) + .expect("Failed to create VR service"); let service = Box::new(service); let heartbeat = Box::new(heartbeat); VRInitOptions::VRService(service, heartbeat) @@ -157,6 +157,8 @@ pub unsafe extern "C" fn init_servo( enable_subpixel_text_antialiasing: false, vr_init, coordinates, + gl_context_pointer: Some(gl.gl_context), + native_display_pointer: Some(gl.display), }; let wakeup = Box::new(EventLoopWakerInstance); let shut_down_complete = Rc::new(Cell::new(false)); @@ -172,7 +174,7 @@ pub unsafe extern "C" fn init_servo( keyboard, }); info!("Starting servo"); - simpleservo::init(opts, gl, wakeup, callbacks).expect("error initializing Servo"); + simpleservo::init(opts, gl.gl_wrapper, wakeup, callbacks).expect("error initializing Servo"); let result = Box::new(ServoInstance { scroll_state: ScrollState::TriggerUp,