diff --git a/Cargo.lock b/Cargo.lock index 271edc32627..b20275dddf3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5072,6 +5072,7 @@ dependencies = [ "surfman", "surfman-chains", "surfman-chains-api", + "webxr", ] [[package]] @@ -6779,7 +6780,7 @@ dependencies = [ [[package]] name = "webxr" version = "0.0.1" -source = "git+https://github.com/servo/webxr#d9ec2263b017d9c415ce35e0f4417a454564bcf2" +source = "git+https://github.com/servo/webxr#3e731eac4180f739e85f8b6b9dacbc5a2015aec8" dependencies = [ "android_injected_glue", "bindgen", @@ -6802,7 +6803,7 @@ dependencies = [ [[package]] name = "webxr-api" version = "0.0.1" -source = "git+https://github.com/servo/webxr#d9ec2263b017d9c415ce35e0f4417a454564bcf2" +source = "git+https://github.com/servo/webxr#3e731eac4180f739e85f8b6b9dacbc5a2015aec8" dependencies = [ "euclid", "ipc-channel", diff --git a/components/compositing/compositor.rs b/components/compositing/compositor.rs index f1f0149ec6b..ba5002a83dd 100644 --- a/components/compositing/compositor.rs +++ b/components/compositing/compositor.rs @@ -1689,6 +1689,9 @@ impl IOCompositor { // Run the WebXR main thread self.webxr_main_thread.run_one_frame(); + // The WebXR thread may make a different context current + let _ = self.webrender_surfman.make_gl_context_current(); + if !self.pending_scroll_zoom_events.is_empty() && !self.waiting_for_results_of_scroll { self.process_pending_scroll_events() } diff --git a/ports/gstplugin/Cargo.toml b/ports/gstplugin/Cargo.toml index dd45c6225a8..05816b4e629 100644 --- a/ports/gstplugin/Cargo.toml +++ b/ports/gstplugin/Cargo.toml @@ -32,6 +32,7 @@ sparkle = "0.1" surfman = "0.3" surfman-chains = "0.4" surfman-chains-api = "0.2" +webxr = { git = "https://github.com/servo/webxr", features = ["glwindow"] } [build-dependencies] gst-plugin-version-helper = "0.2" diff --git a/ports/gstplugin/README.md b/ports/gstplugin/README.md index 0fdc714c1e8..987ec11fd61 100644 --- a/ports/gstplugin/README.md +++ b/ports/gstplugin/README.md @@ -59,6 +59,22 @@ GST_PLUGIN_PATH=target/gstplugins \ ! filesink location=test.ogg ``` +To stream webxr content and save to a file: +``` +GST_PLUGIN_PATH=target/gstplugins \ + gst-launch-1.0 -e servowebsrc url=... webxr=left-right \ + ! video/x-raw\(memory:GLMemory\),framerate=50/1,width=512,height=512,format=RGBA \ + ! glvideoflip video-direction=vert \ + ! glcolorconvert \ + ! gldownload \ + ! queue \ + ! x264enc \ + ! mp4mux \ + ! filesink location=test.mp4 +``` +This requires the webxr content to support the `sessionavailable` event for launching directly into immersive mode. +Values for `webxr` include `none`, `left-right`, `red-cyan`, `cubemap` and `spherical`. + *Note*: killing the gstreamer pipeline with control-C sometimes locks up macOS to the point of needing a power cycle. Killing the pipeline by closing the window seems to work. diff --git a/ports/gstplugin/servowebsrc.rs b/ports/gstplugin/servowebsrc.rs index 357dfca2bed..90a69b2967c 100644 --- a/ports/gstplugin/servowebsrc.rs +++ b/ports/gstplugin/servowebsrc.rs @@ -7,6 +7,8 @@ use crate::logging::CATEGORY; use crossbeam_channel::Receiver; use crossbeam_channel::Sender; +use euclid::default::Rotation3D; +use euclid::default::Vector3D; use euclid::Point2D; use euclid::Rect; use euclid::Scale; @@ -63,14 +65,17 @@ use gstreamer_video::VideoInfo; use log::debug; use log::error; use log::info; +use log::warn; use servo::compositing::windowing::AnimationState; use servo::compositing::windowing::EmbedderCoordinates; use servo::compositing::windowing::EmbedderMethods; use servo::compositing::windowing::WindowEvent; use servo::compositing::windowing::WindowMethods; +use servo::embedder_traits::EmbedderProxy; use servo::embedder_traits::EventLoopWaker; use servo::msg::constellation_msg::TopLevelBrowsingContextId; +use servo::servo_config::set_pref; use servo::servo_url::ServoUrl; use servo::webrender_api::units::DevicePixel; use servo::webrender_surfman::WebrenderSurfman; @@ -83,6 +88,7 @@ use sparkle::gl::Gl; use surfman::Connection; use surfman::Context; use surfman::Device; +use surfman::SurfaceAccess; use surfman::SurfaceType; use surfman_chains::SwapChain; use surfman_chains_api::SwapChainAPI; @@ -100,9 +106,15 @@ use std::thread; use std::time::Duration; use std::time::Instant; +use webxr::glwindow::GlWindow as WebXRWindow; +use webxr::glwindow::GlWindowDiscovery as WebXRDiscovery; +use webxr::glwindow::GlWindowMode as WebXRMode; +use webxr::glwindow::GlWindowRenderTarget as WebXRRenderTarget; + pub struct ServoWebSrc { sender: Sender, url: Mutex>, + webxr_mode: Mutex>, outcaps: Mutex>, info: Mutex>, buffer_pool: Mutex>, @@ -121,7 +133,7 @@ pub struct ServoWebSrc { struct ServoWebSrcGfx { device: Device, context: Context, - swap_chain: SwapChain, + swap_chain: Option>, gl: Rc, read_fbo: GLuint, draw_fbo: GLuint, @@ -146,10 +158,24 @@ impl std::fmt::Debug for ConnectionWhichImplementsDebug { } } +struct SwapChainWhichImplementsDebug(SwapChain); + +impl std::fmt::Debug for SwapChainWhichImplementsDebug { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + "SwapChain".fmt(fmt) + } +} + #[derive(Debug)] enum ServoWebSrcMsg { - Start(ConnectionWhichImplementsDebug, ServoUrl), + Start( + ConnectionWhichImplementsDebug, + ServoUrl, + Option, + Size2D, + ), GetSwapChain(Sender>), + SetSwapChain(SwapChainWhichImplementsDebug), Resize(Size2D), Heartbeat, Stop, @@ -164,24 +190,44 @@ const DEFAULT_FRAME_DURATION: Duration = Duration::from_micros(16_667); struct ServoThread { receiver: Receiver, servo: Servo, + swap_chain: Option>, } impl ServoThread { - fn new(receiver: Receiver) -> Self { - let (connection, url) = match receiver.recv() { - Ok(ServoWebSrcMsg::Start(connection, url)) => (connection.0, url), + fn new(sender: Sender, receiver: Receiver) -> Self { + let (connection, url, webxr_mode, size) = match receiver.recv() { + Ok(ServoWebSrcMsg::Start(connection, url, webxr_mode, size)) => { + (connection.0, url, webxr_mode, size) + }, e => panic!("Failed to start ({:?})", e), }; - info!("Created new servo thread for {}", url); - let embedder = Box::new(ServoWebSrcEmbedder); - let window = Rc::new(ServoWebSrcWindow::new(connection)); - + info!("Created new servo thread for {} ({:?})", url, webxr_mode); + let window = Rc::new(ServoWebSrcWindow::new(connection, webxr_mode, sender, size)); + let embedder = Box::new(ServoWebSrcEmbedder::new(&window)); + let webrender_swap_chain = window + .webrender_surfman + .swap_chain() + .expect("Failed to get webrender swap chain") + .clone(); let mut servo = Servo::new(embedder, window, None); let id = TopLevelBrowsingContextId::new(); servo.handle_events(vec![WindowEvent::NewBrowser(url, id)]); - Self { receiver, servo } + let swap_chain = match webxr_mode { + None => Some(webrender_swap_chain), + Some(..) => { + set_pref!(dom.webxr.sessionavailable, true); + servo.handle_events(vec![WindowEvent::ChangeBrowserVisibility(id, false)]); + None + }, + }; + + Self { + receiver, + servo, + swap_chain, + } } fn run(&mut self) { @@ -190,6 +236,7 @@ impl ServoThread { match msg { ServoWebSrcMsg::Start(..) => error!("Already started"), ServoWebSrcMsg::GetSwapChain(sender) => self.send_swap_chain(sender), + ServoWebSrcMsg::SetSwapChain(swap_chain) => self.swap_chain = Some(swap_chain.0), ServoWebSrcMsg::Resize(size) => self.resize(size), ServoWebSrcMsg::Heartbeat => self.servo.handle_events(vec![]), ServoWebSrcMsg::Stop => break, @@ -199,17 +246,14 @@ impl ServoThread { } fn send_swap_chain(&mut self, sender: Sender>) { - let swap_chain = self - .servo - .window() - .webrender_surfman - .swap_chain() - .expect("Failed to get swap chain") - .clone(); - sender.send(swap_chain).expect("Failed to send swap chain"); + if let Some(ref swap_chain) = self.swap_chain { + debug!("Sending swap chain"); + let _ = sender.send(swap_chain.clone()); + } } fn resize(&mut self, size: Size2D) { + debug!("Servo resized to {:?}", size); let _ = self .servo .window() @@ -219,17 +263,50 @@ impl ServoThread { } } -struct ServoWebSrcEmbedder; +struct ServoWebSrcEmbedder { + webrender_surfman: WebrenderSurfman, + webxr_mode: Option, + sender: Sender, +} -impl EmbedderMethods for ServoWebSrcEmbedder { - fn create_event_loop_waker(&mut self) -> Box { - Box::new(ServoWebSrcEmbedder) +impl ServoWebSrcEmbedder { + fn new(window: &ServoWebSrcWindow) -> ServoWebSrcEmbedder { + let webxr_mode = window.webxr_mode; + let sender = window.sender.clone(); + let webrender_surfman = window.webrender_surfman.clone(); + ServoWebSrcEmbedder { + webxr_mode, + webrender_surfman, + sender, + } } } -impl EventLoopWaker for ServoWebSrcEmbedder { +impl EmbedderMethods for ServoWebSrcEmbedder { + fn create_event_loop_waker(&mut self) -> Box { + Box::new(ServoWebSrcEventLoopWaker) + } + + fn register_webxr(&mut self, registry: &mut webxr::MainThreadRegistry, _: EmbedderProxy) { + let connection = self.webrender_surfman.connection(); + let adapter = self.webrender_surfman.adapter(); + let context_attributes = self.webrender_surfman.context_attributes(); + let sender = self.sender.clone(); + let webrender_surfman = self.webrender_surfman.clone(); + let webxr_mode = self.webxr_mode; + let factory = Box::new(move || { + ServoWebSrcWebXR::new(webrender_surfman.clone(), webxr_mode, sender.clone()) + }); + let discovery = WebXRDiscovery::new(connection, adapter, context_attributes, factory); + registry.register(discovery); + } +} + +struct ServoWebSrcEventLoopWaker; + +impl EventLoopWaker for ServoWebSrcEventLoopWaker { fn clone_box(&self) -> Box { - Box::new(ServoWebSrcEmbedder) + Box::new(ServoWebSrcEventLoopWaker) } fn wake(&self) {} @@ -237,19 +314,30 @@ impl EventLoopWaker for ServoWebSrcEmbedder { struct ServoWebSrcWindow { webrender_surfman: WebrenderSurfman, + webxr_mode: Option, + sender: Sender, } impl ServoWebSrcWindow { - fn new(connection: Connection) -> Self { + fn new( + connection: Connection, + webxr_mode: Option, + sender: Sender, + size: Size2D, + ) -> Self { let adapter = connection .create_adapter() .expect("Failed to create adapter"); - let size = Size2D::new(512, 512); + let size = size.to_untyped(); let surface_type = SurfaceType::Generic { size }; let webrender_surfman = WebrenderSurfman::create(&connection, &adapter, surface_type) .expect("Failed to create surfman"); - Self { webrender_surfman } + Self { + webrender_surfman, + webxr_mode, + sender, + } } } @@ -292,15 +380,88 @@ impl WindowMethods for ServoWebSrcWindow { } } -static PROPERTIES: [Property; 1] = [Property("url", |name| { - ParamSpec::string( - name, - "URL", - "Initial URL", - Some(DEFAULT_URL), - glib::ParamFlags::READWRITE, - ) -})]; +struct ServoWebSrcWebXR { + webrender_surfman: WebrenderSurfman, + webxr_mode: Option, + sender: Sender, +} + +impl ServoWebSrcWebXR { + fn new( + webrender_surfman: WebrenderSurfman, + webxr_mode: Option, + sender: Sender, + ) -> Result, ()> { + Ok(Box::new(ServoWebSrcWebXR { + webrender_surfman, + webxr_mode, + sender, + })) + } +} + +impl WebXRWindow for ServoWebSrcWebXR { + fn get_mode(&self) -> WebXRMode { + self.webxr_mode.unwrap() + } + + fn get_render_target(&self, device: &mut Device, context: &mut Context) -> WebXRRenderTarget { + log::debug!("Creating webxr render target"); + let size = self + .webrender_surfman + .context_surface_info() + .expect("Failed to get webrender size") + .expect("Failed to get webrender size") + .size; + let surface_access = SurfaceAccess::GPUOnly; + let surface_type = SurfaceType::Generic { size }; + let surface = device + .create_surface(context, surface_access, surface_type) + .expect("Failed to create target surface"); + device + .bind_surface_to_context(context, surface) + .expect("Failed to bind target surface"); + let webxr_swap_chain = SwapChain::create_attached(device, context, surface_access) + .expect("Failed to create target swap chain"); + + log::debug!("Created webxr render target {:?}", size); + let _ = self + .sender + .send(ServoWebSrcMsg::SetSwapChain(SwapChainWhichImplementsDebug( + webxr_swap_chain.clone(), + ))); + WebXRRenderTarget::SwapChain(webxr_swap_chain) + } + + fn get_rotation(&self) -> Rotation3D { + Rotation3D::identity() + } + + fn get_translation(&self) -> Vector3D { + Vector3D::zero() + } +} + +static PROPERTIES: [Property; 2] = [ + Property("url", |name| { + ParamSpec::string( + name, + "URL", + "Initial URL", + Some(DEFAULT_URL), + glib::ParamFlags::READWRITE, + ) + }), + Property("webxr", |name| { + ParamSpec::string( + name, + "WebXR", + "Stream immersive WebXR content", + None, + glib::ParamFlags::READWRITE, + ) + }), +]; const CAPS: &str = "video/x-raw(memory:GLMemory), format={RGBA,RGBx}, @@ -317,11 +478,13 @@ impl ObjectSubclass for ServoWebSrc { type Class = ClassStruct; fn new() -> Self { - let (sender, receiver) = crossbeam_channel::bounded(1); - thread::spawn(move || ServoThread::new(receiver).run()); + let (sender, receiver) = crossbeam_channel::unbounded(); + let sender_clone = sender.clone(); + thread::spawn(move || ServoThread::new(sender_clone, receiver).run()); let info = Mutex::new(None); let outcaps = Mutex::new(None); let url = Mutex::new(None); + let webxr_mode = Mutex::new(None); let buffer_pool = Mutex::new(None); let gl_context = Mutex::new(None); let connection = Mutex::new(None); @@ -333,6 +496,7 @@ impl ObjectSubclass for ServoWebSrc { info, outcaps, url, + webxr_mode, buffer_pool, gl_context, connection, @@ -379,6 +543,19 @@ impl ObjectImpl for ServoWebSrc { let url = value.get().expect("Failed to get url value"); *guard = url; }, + Property("webxr", ..) => { + let mut guard = self.webxr_mode.lock().expect("Failed to lock mutex"); + let webxr_mode = match value.get().expect("Failed to get url value") { + None | Some("none") => None, + Some("blit") => Some(WebXRMode::Blit), + Some("left-right") => Some(WebXRMode::StereoLeftRight), + Some("red-cyan") => Some(WebXRMode::StereoRedCyan), + Some("cubemap") => Some(WebXRMode::Cubemap), + Some("spherical") => Some(WebXRMode::Spherical), + Some(mode) => panic!("Unknown WebXR mode {}", mode), + }; + *guard = webxr_mode; + }, _ => unimplemented!(), } } @@ -390,6 +567,18 @@ impl ObjectImpl for ServoWebSrc { let guard = self.url.lock().expect("Failed to lock mutex"); Ok(Value::from(guard.as_ref())) }, + Property("webxr", ..) => { + let guard = self.webxr_mode.lock().expect("Failed to lock mutex"); + let webxr_mode = match guard.as_ref() { + Some(WebXRMode::Blit) => Some("blit"), + Some(WebXRMode::StereoLeftRight) => Some("left-right"), + Some(WebXRMode::StereoRedCyan) => Some("red-cyan"), + Some(WebXRMode::Cubemap) => Some("cubemap"), + Some(WebXRMode::Spherical) => Some("spherical"), + None => None, + }; + Ok(Value::from(webxr_mode)) + }, _ => unimplemented!(), } } @@ -501,7 +690,6 @@ impl BaseSrcImpl for ServoWebSrc { let data = &mut task as *mut FillOnGLThread as *mut c_void; unsafe { gst_gl_context_thread_add(gl_memory.mem.context, Some(fill_on_gl_thread), data) }; - task.result?; // Put down a GL sync point if needed if let Some(meta) = buffer.get_meta::() { @@ -510,7 +698,10 @@ impl BaseSrcImpl for ServoWebSrc { } // Wake up Servo + debug!("Sending heartbeat"); let _ = self.sender.send(ServoWebSrcMsg::Heartbeat); + + task.result?; Ok(buffer) } } @@ -569,9 +760,19 @@ impl ServoWebSrc { error!("Failed to parse url {} ({:?})", url_string, e); FlowError::Error })?; + let size = self + .info + .lock() + .expect("Poisoned mutex") + .as_ref() + .map(|info| Size2D::new(info.width() as i32, info.height() as i32)) + .unwrap_or(Size2D::new(512, 512)); + let webxr_mode = *self.webxr_mode.lock().expect("Poisoned mutex"); let _ = self.sender.send(ServoWebSrcMsg::Start( ConnectionWhichImplementsDebug(connection), url, + webxr_mode, + size, )); // Create a new buffer pool for GL memory @@ -702,6 +903,8 @@ impl ServoWebSrc { .expect("Failed to bootstrap surfman context") }; + let swap_chain = None; + debug!("Creating GL bindings"); let gl = Gl::gl_fns(gl::ffi_gl::Gl::load_with(|s| { gl_context.get_proc_address(s) as *const _ @@ -709,11 +912,6 @@ impl ServoWebSrc { let draw_fbo = gl.gen_framebuffers(1)[0]; let read_fbo = gl.gen_framebuffers(1)[0]; - debug!("Getting the swap chain"); - let (acks, ackr) = crossbeam_channel::bounded(1); - let _ = self.sender.send(ServoWebSrcMsg::GetSwapChain(acks)); - let swap_chain = ackr.recv().expect("Failed to get swap chain"); - ServoWebSrcGfx { device, context, @@ -724,6 +922,13 @@ impl ServoWebSrc { } }); + if gfx.swap_chain.is_none() { + debug!("Getting the swap chain"); + let (acks, ackr) = crossbeam_channel::bounded(1); + let _ = self.sender.send(ServoWebSrcMsg::GetSwapChain(acks)); + gfx.swap_chain = ackr.recv_timeout(Duration::from_millis(16)).ok(); + } + gfx.device .make_context_current(&gfx.context) .expect("Failed to make surfman context current"); @@ -753,7 +958,11 @@ impl ServoWebSrc { gfx.gl.clear(gl::COLOR_BUFFER_BIT); debug_assert_eq!(gfx.gl.get_error(), gl::NO_ERROR); - if let Some(surface) = gfx.swap_chain.take_surface() { + if let Some((swap_chain, surface)) = gfx.swap_chain.as_ref().and_then(|swap_chain| { + swap_chain + .take_surface() + .map(|surface| (swap_chain, surface)) + }) { debug!("Rendering surface"); let surface_size = Size2D::from_untyped(gfx.device.surface_info(&surface).size); if size != surface_size { @@ -764,7 +973,7 @@ impl ServoWebSrc { if size.width <= 0 || size.height <= 0 { info!("Surface is zero-sized"); - gfx.swap_chain.recycle_surface(surface); + swap_chain.recycle_surface(surface); return; } @@ -833,9 +1042,9 @@ impl ServoWebSrc { .device .destroy_surface_texture(&mut gfx.context, surface_texture) .unwrap(); - gfx.swap_chain.recycle_surface(surface); + swap_chain.recycle_surface(surface); } else { - debug!("Failed to get current surface"); + warn!("Failed to get current surface"); } // Restore the GL state diff --git a/ports/winit/headed_window.rs b/ports/winit/headed_window.rs index 55d58e569f8..04fbbd762bb 100644 --- a/ports/winit/headed_window.rs +++ b/ports/winit/headed_window.rs @@ -41,10 +41,10 @@ use surfman::platform::generic::multi::connection::NativeConnection; #[cfg(target_os = "linux")] use surfman::platform::generic::multi::context::NativeContext; use surfman::Connection; +use surfman::Context; use surfman::Device; use surfman::GLApi; use surfman::GLVersion; -use surfman::NativeWidget; use surfman::SurfaceType; #[cfg(target_os = "windows")] use winapi; @@ -665,10 +665,11 @@ struct XRWindowPose { } impl webxr::glwindow::GlWindow for XRWindow { - fn get_native_widget(&self, device: &Device) -> NativeWidget { - device.connection() + fn get_render_target(&self, device: &mut Device, _context: &mut Context) -> webxr::glwindow::GlWindowRenderTarget { + let native_widget = device.connection() .create_native_widget_from_winit_window(&self.winit_window) - .expect("Failed to create native widget") + .expect("Failed to create native widget"); + webxr::glwindow::GlWindowRenderTarget::NativeWidget(native_widget) } fn get_rotation(&self) -> Rotation3D { diff --git a/ports/winit/headless_window.rs b/ports/winit/headless_window.rs index b2c7a59d36e..3f8f5c7ea4c 100644 --- a/ports/winit/headless_window.rs +++ b/ports/winit/headless_window.rs @@ -18,8 +18,8 @@ use servo::webrender_surfman::WebrenderSurfman; use std::cell::Cell; use std::rc::Rc; use surfman::Connection; +use surfman::Context; use surfman::Device; -use surfman::NativeWidget; use surfman::SurfaceType; pub struct Window { @@ -148,7 +148,7 @@ impl WindowMethods for Window { } impl webxr::glwindow::GlWindow for Window { - fn get_native_widget(&self, _device: &Device) -> NativeWidget { + fn get_render_target(&self, _device: &mut Device, _context: &mut Context) -> webxr::glwindow::GlWindowRenderTarget { unimplemented!() }