Auto merge of #23483 - ceyusa:player-context, r=jdm

Media player rendering with GL textures

These patches pass the application's OpenGL raw context  and the its native display address to the media player, in order to create an internal wrapped context, thus it will generate video frames as textures.

For now only EGL from glutin-based app and android are in place, though tested only in Linux glutin app.

This PR also renders the generated frame textures by Servo/Media and renders them by using a thread that connects Webrenderer's ExternalImageHandler and each instantiated player. **By now, these patches, disable the WebGL rendering**. We need to provide a ExternalImageHandler demuxer.

This PR depends on https://github.com/servo/media/pull/270

- [X]  `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- This PR fixes #22300 and fixes #22920

In order to test it you must launch servo as

`./mach run -- --pref media.glvideo.enabled [...]`

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/23483)
<!-- Reviewable:end -->

ports/glutin/Cargo.toml

@@ -46,6 +46,7 @@ webrender_debugger = ["libservo/webrender_debugger"]
 [target.'cfg(not(target_os = "android"))'.dependencies]
 backtrace = "0.3"
 bitflags = "1.0"
+clipboard = "0.5"
 crossbeam-channel = "0.3"
 euclid = "0.19"
 gleam = "0.6"
@@ -56,9 +57,9 @@ libservo = {path = "../../components/servo"}
 libc = "0.2"
 log = "0.4"
 rust-webvr = { version = "0.13", features = ["glwindow"] }
-webxr =  { git = "https://github.com/servo/webxr", features = ["glwindow"] }
+servo-media = {git = "https://github.com/servo/media"}
 tinyfiledialogs = "3.0"
-clipboard = "0.5"
+webxr =  { git = "https://github.com/servo/webxr", features = ["glwindow"] }
 
 [target.'cfg(any(target_os = "linux", target_os = "windows"))'.dependencies]
 image = "0.21"

ports/glutin/context.rs

@@ -2,7 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-use glutin::{WindowedContext, NotCurrent, PossiblyCurrent};
+use glutin::os::ContextTraitExt;
+use glutin::{NotCurrent, PossiblyCurrent, WindowedContext};
+use servo_media::player::context::GlContext as RawContext;
+use std::os::raw;
 
 pub enum GlContext {
     Current(WindowedContext<PossiblyCurrent>),
@@ -71,4 +74,85 @@ impl GlContext {
             GlContext::None => unreachable!(),
         };
     }
+    #[allow(unreachable_code, unused_variables)]
+    pub fn raw_context(&self) -> RawContext {
+        match self {
+            GlContext::Current(c) => {
+                #[cfg(any(
+                    target_os = "linux",
+                    target_os = "dragonfly",
+                    target_os = "freebsd",
+                    target_os = "netbsd",
+                    target_os = "openbsd",
+                ))]
+                {
+                    use glutin::os::unix::RawHandle;
+
+                    let raw_handle = unsafe { c.raw_handle() };
+                    return match raw_handle {
+                        RawHandle::Egl(handle) => RawContext::Egl(handle as usize),
+                        RawHandle::Glx(handle) => RawContext::Glx(handle as usize),
+                    };
+                }
+
+                #[cfg(target_os = "windows")]
+                {
+                    use glutin::os::windows::RawHandle;
+
+                    let raw_handle = unsafe { c.raw_handle() };
+                    return match raw_handle {
+                        RawHandle::Egl(handle) => RawContext::Egl(handle as usize),
+                        // @TODO(victor): RawContext::Wgl in servo-media
+                        RawHandle::Wgl(_) => unimplemented!(),
+                    }
+                }
+
+                #[cfg(target_os = "android")]
+                {
+                    let raw_handle = unsafe { c.raw_handle() };
+                    return RawContext::Egl(raw_handle as usize);
+                }
+
+                #[cfg(target_os = "macos")]
+                return unimplemented!(); // @TODO(victor): RawContext::Cocoa in servo-media
+
+                #[cfg(not(any(
+                    target_os = "linux",
+                    target_os = "dragonfly",
+                    target_os = "freebsd",
+                    target_os = "netbsd",
+                    target_os = "openbsd",
+                    target_os = "windows",
+                    target_os = "android",
+                    target_os = "macos",
+                )))]
+                unimplemented!()
+            }
+            GlContext::NotCurrent(_) => {
+                error!("Context is not current.");
+                RawContext::Unknown
+            }
+            GlContext::None => unreachable!(),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn egl_display(&self) -> Option<*const raw::c_void> {
+        match self {
+            GlContext::Current(c) => unsafe { c.get_egl_display() },
+            GlContext::NotCurrent(_) => {
+                error!("Context is not current.");
+                None
+            },
+            GlContext::None => unreachable!(),
+        }
+    }
+
+    pub fn get_api(&self) -> glutin::Api {
+        match self {
+            GlContext::Current(c) => c.get_api(),
+            GlContext::NotCurrent(c) => c.get_api(),
+            GlContext::None => unreachable!(),
+        }
+    }
 }

ports/glutin/headed_window.rs

@@ -13,9 +13,9 @@ use gleam::gl;
 use glutin::dpi::{LogicalPosition, LogicalSize, PhysicalSize};
 #[cfg(target_os = "macos")]
 use glutin::os::macos::{ActivationPolicy, WindowBuilderExt};
+use glutin::Api;
 #[cfg(any(target_os = "linux", target_os = "windows"))]
 use glutin::Icon;
-use glutin::Api;
 use glutin::{ElementState, KeyboardInput, MouseButton, MouseScrollDelta, TouchPhase};
 #[cfg(any(target_os = "linux", target_os = "windows"))]
 use image;
@@ -24,12 +24,13 @@ use servo::compositing::windowing::{AnimationState, MouseWindowEvent, WindowEven
 use servo::compositing::windowing::{EmbedderCoordinates, WindowMethods};
 use servo::embedder_traits::Cursor;
 use servo::script_traits::{TouchEventType, WheelMode, WheelDelta};
-use servo::servo_config::opts;
+use servo::servo_config::{opts, pref};
 use servo::servo_geometry::DeviceIndependentPixel;
 use servo::style_traits::DevicePixel;
 use servo::webrender_api::{
     DeviceIntPoint, DeviceIntRect, DeviceIntSize, FramebufferIntSize, ScrollLocation,
 };
+use servo_media::player::context::{GlApi, GlContext as PlayerGLContext, NativeDisplay};
 use std::cell::{Cell, RefCell};
 use std::mem;
 use std::rc::Rc;
@@ -524,6 +525,102 @@ impl WindowMethods for Window {
     fn prepare_for_composite(&self) {
         self.gl_context.borrow_mut().make_current();
     }
+
+    fn get_gl_context(&self) -> PlayerGLContext {
+        if pref!(media.glvideo.enabled) {
+            self.gl_context.borrow().raw_context()
+        } else {
+            PlayerGLContext::Unknown
+        }
+    }
+
+    fn get_native_display(&self) -> NativeDisplay {
+        if !pref!(media.glvideo.enabled) {
+            return NativeDisplay::Unknown;
+        }
+
+        #[cfg(any(
+            target_os = "linux",
+            target_os = "dragonfly",
+            target_os = "freebsd",
+            target_os = "netbsd",
+            target_os = "openbsd",
+            target_os = "windows",
+            target_os = "android",
+        ))]
+        let native_display = {
+            if let Some(display) = self.gl_context.borrow().egl_display() {
+                NativeDisplay::Egl(display as usize)
+            } else {
+                #[cfg(any(
+                    target_os = "linux",
+                    target_os = "dragonfly",
+                    target_os = "freebsd",
+                    target_os = "netbsd",
+                    target_os = "openbsd",
+                ))]
+                {
+                    use glutin::os::unix::WindowExt;
+
+                    if let Some(display) = self.gl_context.borrow().window().get_wayland_display() {
+                        NativeDisplay::Wayland(display as usize)
+                    } else if let Some(display) =
+                        self.gl_context.borrow().window().get_xlib_display()
+                    {
+                        NativeDisplay::X11(display as usize)
+                    } else {
+                        NativeDisplay::Unknown
+                    }
+                }
+
+                #[cfg(not(any(
+                    target_os = "linux",
+                    target_os = "dragonfly",
+                    target_os = "freebsd",
+                    target_os = "netbsd",
+                    target_os = "openbsd",
+                )))]
+                NativeDisplay::Unknown
+            }
+        };
+
+        #[cfg(not(any(
+            target_os = "linux",
+            target_os = "dragonfly",
+            target_os = "freebsd",
+            target_os = "netbsd",
+            target_os = "openbsd",
+            target_os = "windows",
+            target_os = "android",
+        )))]
+        let native_display = NativeDisplay::Unknown;
+
+        native_display
+    }
+
+    fn get_gl_api(&self) -> GlApi {
+        let api = self.gl_context.borrow().get_api();
+
+        let version = self.gl.get_string(gl::VERSION);
+        let version = version.trim_start_matches("OpenGL ES ");
+        let mut values = version.split(&['.', ' '][..]);
+        let major = values
+            .next()
+            .and_then(|v| v.parse::<u32>().ok())
+            .unwrap_or(1);
+        let minor = values
+            .next()
+            .and_then(|v| v.parse::<u32>().ok())
+            .unwrap_or(20);
+
+        match api {
+            glutin::Api::OpenGl if major >= 3 && minor >= 2 => GlApi::OpenGL3,
+            glutin::Api::OpenGl => GlApi::OpenGL,
+            glutin::Api::OpenGlEs if major > 1 => GlApi::Gles2,
+            glutin::Api::OpenGlEs => GlApi::Gles1,
+            _ => GlApi::None,
+        }
+    }
 }
 
 fn winit_phase_to_touch_event_type(phase: TouchPhase) -> TouchEventType {

ports/glutin/headless_window.rs

@@ -14,6 +14,7 @@ use servo::servo_config::opts;
 use servo::servo_geometry::DeviceIndependentPixel;
 use servo::style_traits::DevicePixel;
 use servo::webrender_api::{DeviceIntRect, FramebufferIntSize};
+use servo_media::player::context as MediaPlayerCtxt;
 use std::cell::Cell;
 #[cfg(any(target_os = "linux", target_os = "macos"))]
 use std::ffi::CString;
@@ -195,4 +196,16 @@ impl WindowMethods for Window {
     }
 
     fn prepare_for_composite(&self) { }
+
+    fn get_gl_context(&self) -> MediaPlayerCtxt::GlContext {
+        MediaPlayerCtxt::GlContext::Unknown
+    }
+
+    fn get_native_display(&self) -> MediaPlayerCtxt::NativeDisplay {
+        MediaPlayerCtxt::NativeDisplay::Unknown
+    }
+
+    fn get_gl_api(&self) -> MediaPlayerCtxt::GlApi {
+        MediaPlayerCtxt::GlApi::None
+    }
 }

ports/libmlservo/src/lib.rs

@@ -144,8 +144,8 @@ pub unsafe extern "C" fn init_servo(
         VRInitOptions::None
     } else {
         let name = String::from("Magic Leap VR Display");
-        let (service, heartbeat) =
-            MagicLeapVRService::new(name, ctxt, gl.clone()).expect("Failed to create VR service");
+        let (service, heartbeat) = MagicLeapVRService::new(name, ctxt, gl.gl_wrapper.clone())
+            .expect("Failed to create VR service");
         let service = Box::new(service);
         let heartbeat = Box::new(heartbeat);
         VRInitOptions::VRService(service, heartbeat)
@@ -157,6 +157,8 @@ pub unsafe extern "C" fn init_servo(
         enable_subpixel_text_antialiasing: false,
         vr_init,
         coordinates,
+        gl_context_pointer: Some(gl.gl_context),
+        native_display_pointer: Some(gl.display),
     };
     let wakeup = Box::new(EventLoopWakerInstance);
     let shut_down_complete = Rc::new(Cell::new(false));
@@ -172,7 +174,7 @@ pub unsafe extern "C" fn init_servo(
         keyboard,
     });
     info!("Starting servo");
-    simpleservo::init(opts, gl, wakeup, callbacks).expect("error initializing Servo");
+    simpleservo::init(opts, gl.gl_wrapper, wakeup, callbacks).expect("error initializing Servo");
 
     let result = Box::new(ServoInstance {
         scroll_state: ScrollState::TriggerUp,

ports/libsimpleservo/api/Cargo.toml

@@ -9,6 +9,7 @@ publish = false
 [dependencies]
 libservo = { path = "../../../components/servo" }
 log = "0.4"
+servo-media = { git = "https://github.com/servo/media" }
 
 [target.'cfg(not(target_os = "macos"))'.dependencies]
 libc = "0.2"

ports/libsimpleservo/api/src/gl_glue.rs

@@ -18,6 +18,7 @@ pub mod egl {
     pub type khronos_uint64_t = libc::uint64_t;
     pub type khronos_ssize_t = libc::c_long;
     pub type EGLint = libc::int32_t;
+    pub type EGLContext = *const libc::c_void;
     pub type EGLNativeDisplayType = *const libc::c_void;
     pub type EGLNativePixmapType = *const libc::c_void;
     pub type NativeDisplayType = EGLNativeDisplayType;
@@ -26,12 +27,18 @@ pub mod egl {
 
     include!(concat!(env!("OUT_DIR"), "/egl_bindings.rs"));
 
-    pub fn init() -> Result<crate::gl_glue::ServoGl, &'static str> {
+    pub struct EGLInitResult {
+        pub gl_wrapper: crate::gl_glue::ServoGl,
+        pub gl_context: EGLContext,
+        pub display: EGLNativeDisplayType,
+    }
+
+    pub fn init() -> Result<EGLInitResult, &'static str> {
         info!("Loading EGL...");
         unsafe {
             let egl = Egl;
-            let d = egl.GetCurrentDisplay();
-            egl.SwapInterval(d, 1);
+            let display = egl.GetCurrentDisplay();
+            egl.SwapInterval(display, 1);
             let egl = GlesFns::load_with(|addr| {
                 let addr = CString::new(addr.as_bytes()).unwrap();
                 let addr = addr.as_ptr();
@@ -39,7 +46,11 @@ pub mod egl {
                 egl.GetProcAddress(addr) as *const c_void
             });
             info!("EGL loaded");
-            Ok(egl)
+            Ok(EGLInitResult {
+                gl_wrapper: egl,
+                gl_context: Egl.GetCurrentContext(),
+                display,
+            })
         }
     }
 }

ports/libsimpleservo/api/src/lib.rs

@@ -25,7 +25,7 @@ use servo::servo_url::ServoUrl;
 use servo::webrender_api::{DevicePixel, FramebufferPixel, ScrollLocation};
 use servo::webvr::{VRExternalShmemPtr, VRMainThreadHeartbeat, VRService, VRServiceManager};
 use servo::{self, gl, BrowserId, Servo};
-
+use servo_media::player::context as MediaPlayerContext;
 use std::cell::RefCell;
 use std::mem;
 use std::os::raw::c_void;
@@ -48,6 +48,8 @@ pub struct InitOptions {
     pub density: f32,
     pub vr_init: VRInitOptions,
     pub enable_subpixel_text_antialiasing: bool,
+    pub gl_context_pointer: Option<*const c_void>,
+    pub native_display_pointer: Option<*const c_void>,
 }
 
 pub enum VRInitOptions {
@@ -187,6 +189,8 @@ pub fn init(
         host_callbacks: callbacks,
         coordinates: RefCell::new(init_opts.coordinates),
         density: init_opts.density,
+        gl_context_pointer: init_opts.gl_context_pointer,
+        native_display_pointer: init_opts.native_display_pointer,
     });
 
     let embedder_callbacks = Box::new(ServoEmbedderCallbacks {
@@ -583,6 +587,8 @@ struct ServoWindowCallbacks {
     host_callbacks: Box<dyn HostTrait>,
     coordinates: RefCell<Coordinates>,
     density: f32,
+    gl_context_pointer: Option<*const c_void>,
+    native_display_pointer: Option<*const c_void>,
 }
 
 impl EmbedderMethods for ServoEmbedderCallbacks {
@@ -643,6 +649,24 @@ impl WindowMethods for ServoWindowCallbacks {
             hidpi_factor: TypedScale::new(self.density),
         }
     }
+
+    fn get_gl_context(&self) -> MediaPlayerContext::GlContext {
+        match self.gl_context_pointer {
+            Some(context) => MediaPlayerContext::GlContext::Egl(context as usize),
+            None => MediaPlayerContext::GlContext::Unknown,
+        }
+    }
+
+    fn get_native_display(&self) -> MediaPlayerContext::NativeDisplay {
+        match self.native_display_pointer {
+            Some(display) => MediaPlayerContext::NativeDisplay::Egl(display as usize),
+            None => MediaPlayerContext::NativeDisplay::Unknown,
+        }
+    }
+
+    fn get_gl_api(&self) -> MediaPlayerContext::GlApi {
+        MediaPlayerContext::GlApi::Gles2
+    }
 }
 
 struct ResourceReaderInstance;

ports/libsimpleservo/capi/src/lib.rs

@@ -93,16 +93,18 @@ fn init_logger() {
     crate::env_logger::init();
 }
 
-fn init(
+unsafe fn init(
     opts: CInitOptions,
     gl: gl_glue::ServoGl,
+    gl_context: Option<*const c_void>,
+    display: Option<*const c_void>,
     wakeup: extern "C" fn(),
     callbacks: CHostCallbacks,
 ) {
     init_logger();
 
     let args = if !opts.args.is_null() {
-        let args = unsafe { CStr::from_ptr(opts.args) };
+        let args = CStr::from_ptr(opts.args);
         args.to_str()
             .unwrap_or("")
             .split(' ')
@@ -112,7 +114,7 @@ fn init(
         vec![]
     };
 
-    let url = unsafe { CStr::from_ptr(opts.url) };
+    let url = CStr::from_ptr(opts.url);
     let url = url.to_str().map(|s| s.to_string()).ok();
 
     let coordinates = Coordinates::new(0, 0, opts.width, opts.height, opts.width, opts.height);
@@ -128,6 +130,8 @@ fn init(
             VRInitOptions::VRExternal(opts.vr_pointer)
         },
         enable_subpixel_text_antialiasing: opts.enable_subpixel_text_antialiasing,
+        gl_context_pointer: gl_context,
+        native_display_pointer: display,
     };
 
     let wakeup = Box::new(WakeupCallback::new(wakeup));
@@ -145,7 +149,16 @@ pub extern "C" fn init_with_egl(
 ) {
     init_logger();
     let gl = gl_glue::egl::init().unwrap();
-    init(opts, gl, wakeup, callbacks)
+    unsafe {
+        init(
+            opts,
+            gl.gl_wrapper,
+            Some(gl.gl_context),
+            Some(gl.display),
+            wakeup,
+            callbacks,
+        )
+    }
 }
 
 #[cfg(any(target_os = "linux", target_os = "windows", target_os = "macos"))]
@@ -157,7 +170,7 @@ pub extern "C" fn init_with_gl(
 ) {
     init_logger();
     let gl = gl_glue::gl::init().unwrap();
-    init(opts, gl, wakeup, callbacks)
+    unsafe { init(opts, gl, None, None, wakeup, callbacks) }
 }
 
 #[no_mangle]

ports/libsimpleservo/jniapi/src/lib.rs

@@ -52,7 +52,7 @@ pub fn Java_org_mozilla_servoview_JNIServo_init(
     opts: JObject,
     callbacks_obj: JObject,
 ) {
-    let (opts, log, log_str) = match get_options(&env, opts) {
+    let (mut opts, log, log_str) = match get_options(&env, opts) {
         Ok((opts, log, log_str)) => (opts, log, log_str),
         Err(err) => {
             throw(&env, &err);
@@ -104,9 +104,11 @@ pub fn Java_org_mozilla_servoview_JNIServo_init(
     let wakeup = Box::new(WakeupCallback::new(callbacks_ref.clone(), &env));
     let callbacks = Box::new(HostCallbacks::new(callbacks_ref, &env));
 
-    if let Err(err) =
-        gl_glue::egl::init().and_then(|gl| simpleservo::init(opts, gl, wakeup, callbacks))
-    {
+    if let Err(err) = gl_glue::egl::init().and_then(|egl_init| {
+        opts.gl_context_pointer = Some(egl_init.gl_context);
+        opts.native_display_pointer = Some(egl_init.display);
+        simpleservo::init(opts, egl_init.gl_wrapper, wakeup, callbacks)
+    }) {
         throw(&env, err)
     };
 }
@@ -726,6 +728,8 @@ fn get_options(env: &JNIEnv, opts: JObject) -> Result<(InitOptions, bool, Option
         } else {
             VRInitOptions::VRExternal(vr_pointer)
         },
+        gl_context_pointer: None,
+        native_display_pointer: None,
     };
     Ok((opts, log, log_str))
 }