From bc07154e05f5666b3a8c0e7d1e4a92bd395423a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Tue, 17 Sep 2019 18:04:50 +0200 Subject: [PATCH 1/3] Always hold current frame Regardless if it contains raw data or a gl texture. It is required to hold current frame because it can be used for webgl's get_image_pixels --- components/script/dom/htmlmediaelement.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index e3cb0dfb5b2..99b75748687 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -193,12 +193,12 @@ impl FrameRenderer for MediaFrameRenderer { ImageData::Raw(frame.get_data()), &webrender_api::DirtyRect::All, ); - } else if self.player_id.is_some() { - self.current_frame_holder - .get_or_insert_with(|| FrameHolder::new(frame.clone())) - .set(frame); } + self.current_frame_holder + .get_or_insert_with(|| FrameHolder::new(frame.clone())) + .set(frame); + if let Some(old_image_key) = self.old_frame.take() { txn.delete_image(old_image_key); } @@ -220,9 +220,6 @@ impl FrameRenderer for MediaFrameRenderer { TextureTarget::Default }; - self.current_frame_holder - .get_or_insert_with(|| FrameHolder::new(frame.clone())) - .set(frame); ImageData::External(ExternalImageData { id: ExternalImageId(self.player_id.unwrap()), channel_index: 0, @@ -231,6 +228,11 @@ impl FrameRenderer for MediaFrameRenderer { } else { ImageData::Raw(frame.get_data()) }; + + self.current_frame_holder + .get_or_insert_with(|| FrameHolder::new(frame.clone())) + .set(frame); + txn.add_image(new_image_key, descriptor, image_data, None); }, None => { @@ -244,7 +246,6 @@ impl FrameRenderer for MediaFrameRenderer { TextureTarget::Default }; - self.current_frame_holder = Some(FrameHolder::new(frame)); ImageData::External(ExternalImageData { id: ExternalImageId(self.player_id.unwrap()), channel_index: 0, @@ -253,6 +254,9 @@ impl FrameRenderer for MediaFrameRenderer { } else { ImageData::Raw(frame.get_data()) }; + + self.current_frame_holder = Some(FrameHolder::new(frame)); + txn.add_image(image_key, descriptor, image_data, None); }, } From b75d45416220cd69e6d3091f65c70793cb89b2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Fri, 13 Sep 2019 15:59:26 +0200 Subject: [PATCH 2/3] implement get_image_pixels() for video element --- components/script/dom/bindings/trace.rs | 2 ++ components/script/dom/htmlmediaelement.rs | 11 ++++++++ components/script/dom/htmlvideoelement.rs | 27 +++++++++++++++++++ .../script/dom/webglrenderingcontext.rs | 11 +++++--- 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/components/script/dom/bindings/trace.rs b/components/script/dom/bindings/trace.rs index 95bad8b4c4e..bf65eb1e87d 100644 --- a/components/script/dom/bindings/trace.rs +++ b/components/script/dom/bindings/trace.rs @@ -103,6 +103,7 @@ use servo_media::audio::context::AudioContext; use servo_media::audio::graph::NodeId; use servo_media::audio::panner_node::{DistanceModel, PanningModel}; use servo_media::audio::param::ParamType; +use servo_media::player::frame::Frame; use servo_media::player::Player; use servo_media::streams::registry::MediaStreamId; use servo_media::streams::MediaStreamType; @@ -516,6 +517,7 @@ unsafe_no_jsmanaged_fields!(Point2D, Rect); unsafe_no_jsmanaged_fields!(Rect); unsafe_no_jsmanaged_fields!(CascadeData); unsafe_no_jsmanaged_fields!(WindowGLContext); +unsafe_no_jsmanaged_fields!(Frame); unsafe impl<'a> JSTraceable for &'a str { #[inline] diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 99b75748687..398a3ad40f1 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -136,6 +136,10 @@ impl FrameHolder { unreachable!(); } } + + fn get_frame(&self) -> Frame { + self.1.clone() + } } pub struct MediaFrameRenderer { @@ -1857,6 +1861,13 @@ impl HTMLMediaElement { document_from_node(self).unregister_media_controls(&id); } } + + pub fn get_current_frame(&self) -> Option { + match self.frame_renderer.lock().unwrap().current_frame_holder { + Some(ref holder) => Some(holder.get_frame()), + None => return None, + } + } } // XXX Placeholder for [https://github.com/servo/servo/issues/22293] diff --git a/components/script/dom/htmlvideoelement.rs b/components/script/dom/htmlvideoelement.rs index e95a2ca8ccd..bca18c35a2b 100644 --- a/components/script/dom/htmlvideoelement.rs +++ b/components/script/dom/htmlvideoelement.rs @@ -23,6 +23,7 @@ use crate::fetch::FetchCanceller; use crate::image_listener::{add_cache_listener_for_element, ImageCacheListener}; use crate::network_listener::{self, NetworkListener, PreInvoke, ResourceTimingListener}; use dom_struct::dom_struct; +use euclid::default::Size2D; use html5ever::{LocalName, Prefix}; use ipc_channel::ipc; use ipc_channel::router::ROUTER; @@ -34,6 +35,7 @@ use net_traits::{ CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseListener, FetchResponseMsg, }; use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType}; +use servo_media::player::frame::Frame; use servo_url::ServoUrl; use std::cell::Cell; use std::sync::{Arc, Mutex}; @@ -55,6 +57,9 @@ pub struct HTMLVideoElement { /// Load event blocker. Will block the load event while the poster frame /// is being fetched. load_blocker: DomRefCell>, + /// A copy of the last frame + #[ignore_malloc_size_of = "Frame"] + last_frame: DomRefCell>, } impl HTMLVideoElement { @@ -70,6 +75,7 @@ impl HTMLVideoElement { generation_id: Cell::new(0), poster_frame_canceller: DomRefCell::new(Default::default()), load_blocker: Default::default(), + last_frame: Default::default(), } } @@ -108,6 +114,27 @@ impl HTMLVideoElement { LoadBlocker::terminate(&mut *self.load_blocker.borrow_mut()); } + pub fn get_current_frame_data(&self) -> Option<(Option, Size2D)> { + let frame = self.htmlmediaelement.get_current_frame(); + if frame.is_some() { + *self.last_frame.borrow_mut() = frame; + } + + match self.last_frame.borrow().as_ref() { + Some(frame) => { + let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32); + if !frame.is_gl_texture() { + let data = Some(ipc::IpcSharedMemory::from_bytes(&frame.get_data())); + Some((data, size)) + } else { + // XXX(victor): here we only have the GL texture ID. + Some((None, size)) + } + }, + None => None, + } + } + /// https://html.spec.whatwg.org/multipage/#poster-frame fn fetch_poster_frame(&self, poster_url: &str) { // Step 1. diff --git a/components/script/dom/webglrenderingcontext.rs b/components/script/dom/webglrenderingcontext.rs index 46b9c72c6ca..f8350759dd8 100644 --- a/components/script/dom/webglrenderingcontext.rs +++ b/components/script/dom/webglrenderingcontext.rs @@ -604,9 +604,14 @@ impl WebGLRenderingContext { return Ok(None); } }, - TexImageSource::HTMLVideoElement(_) => { - // TODO: https://github.com/servo/servo/issues/6711 - return Ok(None); + TexImageSource::HTMLVideoElement(video) => match video.get_current_frame_data() { + Some((data, size)) => { + let data = data.unwrap_or_else(|| { + IpcSharedMemory::from_bytes(&vec![0; size.area() as usize * 4]) + }); + TexPixels::new(data, size, PixelFormat::BGRA8, false) + }, + None => return Ok(None), }, })) } From 2010d896131cb0536634373edec99000ecc0c225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Thu, 19 Sep 2019 11:51:03 +0200 Subject: [PATCH 3/3] Enable related WPT tests --- .../extensions/oes-texture-float-with-video.html.ini | 2 +- .../extensions/oes-texture-half-float-with-video.html.ini | 2 +- .../textures/misc/texture-corner-case-videos.html.ini | 3 --- .../conformance/textures/misc/texture-npot-video.html.ini | 5 ++++- tests/wpt/webgl/meta/conformance/textures/video/__dir__.ini | 1 - .../textures/video/tex-2d-alpha-alpha-unsigned_byte.html.ini | 2 ++ .../video/tex-2d-luminance-luminance-unsigned_byte.html.ini | 2 ++ ...2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini | 2 ++ .../textures/video/tex-2d-rgb-rgb-unsigned_byte.html.ini | 2 ++ .../video/tex-2d-rgb-rgb-unsigned_short_5_6_5.html.ini | 2 ++ .../textures/video/tex-2d-rgba-rgba-unsigned_byte.html.ini | 2 ++ .../video/tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html.ini | 2 ++ .../video/tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html.ini | 2 ++ 13 files changed, 22 insertions(+), 7 deletions(-) delete mode 100644 tests/wpt/webgl/meta/conformance/textures/misc/texture-corner-case-videos.html.ini delete mode 100644 tests/wpt/webgl/meta/conformance/textures/video/__dir__.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-alpha-alpha-unsigned_byte.html.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance-luminance-unsigned_byte.html.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_short_5_6_5.html.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_byte.html.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html.ini create mode 100644 tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html.ini diff --git a/tests/wpt/webgl/meta/conformance/extensions/oes-texture-float-with-video.html.ini b/tests/wpt/webgl/meta/conformance/extensions/oes-texture-float-with-video.html.ini index 053f3f5bd0f..a7c620be5fb 100644 --- a/tests/wpt/webgl/meta/conformance/extensions/oes-texture-float-with-video.html.ini +++ b/tests/wpt/webgl/meta/conformance/extensions/oes-texture-float-with-video.html.ini @@ -1,2 +1,2 @@ [oes-texture-float-with-video.html] - disabled: https://github.com/servo/servo/issues/6711 + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/extensions/oes-texture-half-float-with-video.html.ini b/tests/wpt/webgl/meta/conformance/extensions/oes-texture-half-float-with-video.html.ini index 7f7fee93649..9ffdb1e39d6 100644 --- a/tests/wpt/webgl/meta/conformance/extensions/oes-texture-half-float-with-video.html.ini +++ b/tests/wpt/webgl/meta/conformance/extensions/oes-texture-half-float-with-video.html.ini @@ -1,2 +1,2 @@ [oes-texture-half-float-with-video.html] - disabled: https://github.com/servo/servo/issues/6711 + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/misc/texture-corner-case-videos.html.ini b/tests/wpt/webgl/meta/conformance/textures/misc/texture-corner-case-videos.html.ini deleted file mode 100644 index 82be9764f19..00000000000 --- a/tests/wpt/webgl/meta/conformance/textures/misc/texture-corner-case-videos.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[texture-corner-case-videos.html] - disabled: https://github.com/servo/servo/issues/6711 - diff --git a/tests/wpt/webgl/meta/conformance/textures/misc/texture-npot-video.html.ini b/tests/wpt/webgl/meta/conformance/textures/misc/texture-npot-video.html.ini index a9b07796ef5..07641b6d1e7 100644 --- a/tests/wpt/webgl/meta/conformance/textures/misc/texture-npot-video.html.ini +++ b/tests/wpt/webgl/meta/conformance/textures/misc/texture-npot-video.html.ini @@ -1,3 +1,6 @@ [texture-npot-video.html] - disabled: https://github.com/servo/servo/issues/6711 + [WebGL test #5: shouldBe 0,0,0\nat (4, 4) expected: 0,0,0 was 255,0,0] + expected: FAIL + [WebGL test #6: shouldBe 0,0,0\nat (4, 40) expected: 0,0,0 was 0,255,0] + expected: FAIL diff --git a/tests/wpt/webgl/meta/conformance/textures/video/__dir__.ini b/tests/wpt/webgl/meta/conformance/textures/video/__dir__.ini deleted file mode 100644 index 50edf46e40e..00000000000 --- a/tests/wpt/webgl/meta/conformance/textures/video/__dir__.ini +++ /dev/null @@ -1 +0,0 @@ -disabled: https://github.com/servo/servo/issues/6711 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-alpha-alpha-unsigned_byte.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-alpha-alpha-unsigned_byte.html.ini new file mode 100644 index 00000000000..cbc876b6aed --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-alpha-alpha-unsigned_byte.html.ini @@ -0,0 +1,2 @@ +[tex-2d-alpha-alpha-unsigned_byte.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance-luminance-unsigned_byte.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance-luminance-unsigned_byte.html.ini new file mode 100644 index 00000000000..ac9cb1c3e25 --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance-luminance-unsigned_byte.html.ini @@ -0,0 +1,2 @@ +[tex-2d-luminance-luminance-unsigned_byte.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini new file mode 100644 index 00000000000..113a3919110 --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini @@ -0,0 +1,2 @@ +[tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html.ini new file mode 100644 index 00000000000..b0a5ba5bba2 --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html.ini @@ -0,0 +1,2 @@ +[tex-2d-rgb-rgb-unsigned_byte.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_short_5_6_5.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_short_5_6_5.html.ini new file mode 100644 index 00000000000..3d2218bed69 --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_short_5_6_5.html.ini @@ -0,0 +1,2 @@ +[tex-2d-rgb-rgb-unsigned_short_5_6_5.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_byte.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_byte.html.ini new file mode 100644 index 00000000000..d5bbb7bf2ae --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_byte.html.ini @@ -0,0 +1,2 @@ +[tex-2d-rgba-rgba-unsigned_byte.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html.ini new file mode 100644 index 00000000000..1117d048625 --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html.ini @@ -0,0 +1,2 @@ +[tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437 diff --git a/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html.ini b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html.ini new file mode 100644 index 00000000000..b5baa6b4322 --- /dev/null +++ b/tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html.ini @@ -0,0 +1,2 @@ +[tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html] + disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437