Auto merge of #24233 - ceyusa:get-image-pixels, r=jdm

get_image_pixels() implementation for video element

<!-- Please describe your changes on the following line: -->

Implements the webgl's get_image_pixels() call for video element

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [ ] These changes fix #21995 (GitHub issue number if applicable)

<!-- Either: -->
- [ ] There are tests for these changes OR
- [ ] These changes do not require tests because ___

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/24233)
<!-- Reviewable:end -->

components/script/dom/bindings/trace.rs

@@ -103,6 +103,7 @@ use servo_media::audio::context::AudioContext;
 use servo_media::audio::graph::NodeId;
 use servo_media::audio::panner_node::{DistanceModel, PanningModel};
 use servo_media::audio::param::ParamType;
+use servo_media::player::frame::Frame;
 use servo_media::player::Player;
 use servo_media::streams::registry::MediaStreamId;
 use servo_media::streams::MediaStreamType;
@@ -516,6 +517,7 @@ unsafe_no_jsmanaged_fields!(Point2D<f32>, Rect<Au>);
 unsafe_no_jsmanaged_fields!(Rect<f32>);
 unsafe_no_jsmanaged_fields!(CascadeData);
 unsafe_no_jsmanaged_fields!(WindowGLContext);
+unsafe_no_jsmanaged_fields!(Frame);
 
 unsafe impl<'a> JSTraceable for &'a str {
     #[inline]

components/script/dom/htmlmediaelement.rs

@@ -136,6 +136,10 @@ impl FrameHolder {
             unreachable!();
         }
     }
+
+    fn get_frame(&self) -> Frame {
+        self.1.clone()
+    }
 }
 
 pub struct MediaFrameRenderer {
@@ -193,12 +197,12 @@ impl FrameRenderer for MediaFrameRenderer {
                         ImageData::Raw(frame.get_data()),
                         &webrender_api::DirtyRect::All,
                     );
-                } else if self.player_id.is_some() {
-                    self.current_frame_holder
-                        .get_or_insert_with(|| FrameHolder::new(frame.clone()))
-                        .set(frame);
                 }
 
+                self.current_frame_holder
+                    .get_or_insert_with(|| FrameHolder::new(frame.clone()))
+                    .set(frame);
+
                 if let Some(old_image_key) = self.old_frame.take() {
                     txn.delete_image(old_image_key);
                 }
@@ -220,9 +224,6 @@ impl FrameRenderer for MediaFrameRenderer {
                         TextureTarget::Default
                     };
 
-                    self.current_frame_holder
-                        .get_or_insert_with(|| FrameHolder::new(frame.clone()))
-                        .set(frame);
                     ImageData::External(ExternalImageData {
                         id: ExternalImageId(self.player_id.unwrap()),
                         channel_index: 0,
@@ -231,6 +232,11 @@ impl FrameRenderer for MediaFrameRenderer {
                 } else {
                     ImageData::Raw(frame.get_data())
                 };
+
+                self.current_frame_holder
+                    .get_or_insert_with(|| FrameHolder::new(frame.clone()))
+                    .set(frame);
+
                 txn.add_image(new_image_key, descriptor, image_data, None);
             },
             None => {
@@ -244,7 +250,6 @@ impl FrameRenderer for MediaFrameRenderer {
                         TextureTarget::Default
                     };
 
-                    self.current_frame_holder = Some(FrameHolder::new(frame));
                     ImageData::External(ExternalImageData {
                         id: ExternalImageId(self.player_id.unwrap()),
                         channel_index: 0,
@@ -253,6 +258,9 @@ impl FrameRenderer for MediaFrameRenderer {
                 } else {
                     ImageData::Raw(frame.get_data())
                 };
+
+                self.current_frame_holder = Some(FrameHolder::new(frame));
+
                 txn.add_image(image_key, descriptor, image_data, None);
             },
         }
@@ -1853,6 +1861,13 @@ impl HTMLMediaElement {
             document_from_node(self).unregister_media_controls(&id);
         }
     }
+
+    pub fn get_current_frame(&self) -> Option<Frame> {
+        match self.frame_renderer.lock().unwrap().current_frame_holder {
+            Some(ref holder) => Some(holder.get_frame()),
+            None => return None,
+        }
+    }
 }
 
 // XXX Placeholder for [https://github.com/servo/servo/issues/22293]

components/script/dom/htmlvideoelement.rs

@@ -23,6 +23,7 @@ use crate::fetch::FetchCanceller;
 use crate::image_listener::{add_cache_listener_for_element, ImageCacheListener};
 use crate::network_listener::{self, NetworkListener, PreInvoke, ResourceTimingListener};
 use dom_struct::dom_struct;
+use euclid::default::Size2D;
 use html5ever::{LocalName, Prefix};
 use ipc_channel::ipc;
 use ipc_channel::router::ROUTER;
@@ -34,6 +35,7 @@ use net_traits::{
     CoreResourceMsg, FetchChannels, FetchMetadata, FetchResponseListener, FetchResponseMsg,
 };
 use net_traits::{NetworkError, ResourceFetchTiming, ResourceTimingType};
+use servo_media::player::frame::Frame;
 use servo_url::ServoUrl;
 use std::cell::Cell;
 use std::sync::{Arc, Mutex};
@@ -55,6 +57,9 @@ pub struct HTMLVideoElement {
     /// Load event blocker. Will block the load event while the poster frame
     /// is being fetched.
     load_blocker: DomRefCell<Option<LoadBlocker>>,
+    /// A copy of the last frame
+    #[ignore_malloc_size_of = "Frame"]
+    last_frame: DomRefCell<Option<Frame>>,
 }
 
 impl HTMLVideoElement {
@@ -70,6 +75,7 @@ impl HTMLVideoElement {
             generation_id: Cell::new(0),
             poster_frame_canceller: DomRefCell::new(Default::default()),
             load_blocker: Default::default(),
+            last_frame: Default::default(),
         }
     }
 
@@ -108,6 +114,27 @@ impl HTMLVideoElement {
         LoadBlocker::terminate(&mut *self.load_blocker.borrow_mut());
     }
 
+    pub fn get_current_frame_data(&self) -> Option<(Option<ipc::IpcSharedMemory>, Size2D<u32>)> {
+        let frame = self.htmlmediaelement.get_current_frame();
+        if frame.is_some() {
+            *self.last_frame.borrow_mut() = frame;
+        }
+
+        match self.last_frame.borrow().as_ref() {
+            Some(frame) => {
+                let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
+                if !frame.is_gl_texture() {
+                    let data = Some(ipc::IpcSharedMemory::from_bytes(&frame.get_data()));
+                    Some((data, size))
+                } else {
+                    // XXX(victor): here we only have the GL texture ID.
+                    Some((None, size))
+                }
+            },
+            None => None,
+        }
+    }
+
     /// https://html.spec.whatwg.org/multipage/#poster-frame
     fn fetch_poster_frame(&self, poster_url: &str) {
         // Step 1.

components/script/dom/webglrenderingcontext.rs

@@ -604,9 +604,14 @@ impl WebGLRenderingContext {
                     return Ok(None);
                 }
             },
-            TexImageSource::HTMLVideoElement(_) => {
-                // TODO: https://github.com/servo/servo/issues/6711
-                return Ok(None);
+            TexImageSource::HTMLVideoElement(video) => match video.get_current_frame_data() {
+                Some((data, size)) => {
+                    let data = data.unwrap_or_else(|| {
+                        IpcSharedMemory::from_bytes(&vec![0; size.area() as usize * 4])
+                    });
+                    TexPixels::new(data, size, PixelFormat::BGRA8, false)
+                },
+                None => return Ok(None),
             },
         }))
     }

tests/wpt/webgl/meta/conformance/extensions/oes-texture-float-with-video.html.ini

@@ -1,2 +1,2 @@
 [oes-texture-float-with-video.html]
-  disabled: https://github.com/servo/servo/issues/6711
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/extensions/oes-texture-half-float-with-video.html.ini

@@ -1,2 +1,2 @@
 [oes-texture-half-float-with-video.html]
-  disabled: https://github.com/servo/servo/issues/6711
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/misc/texture-corner-case-videos.html.ini

@@ -1,3 +0,0 @@
-[texture-corner-case-videos.html]
-  disabled: https://github.com/servo/servo/issues/6711
-

tests/wpt/webgl/meta/conformance/textures/misc/texture-npot-video.html.ini

@@ -1,3 +1,6 @@
 [texture-npot-video.html]
-  disabled: https://github.com/servo/servo/issues/6711
+  [WebGL test #5: shouldBe 0,0,0\nat (4, 4) expected: 0,0,0 was 255,0,0]
+    expected: FAIL
 
+  [WebGL test #6: shouldBe 0,0,0\nat (4, 40) expected: 0,0,0 was 0,255,0]
+    expected: FAIL

tests/wpt/webgl/meta/conformance/textures/video/__dir__.ini

@@ -1 +0,0 @@
-disabled: https://github.com/servo/servo/issues/6711

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-alpha-alpha-unsigned_byte.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-alpha-alpha-unsigned_byte.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance-luminance-unsigned_byte.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-luminance-luminance-unsigned_byte.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-rgb-rgb-unsigned_byte.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgb-rgb-unsigned_short_5_6_5.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-rgb-rgb-unsigned_short_5_6_5.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_byte.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-rgba-rgba-unsigned_byte.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-rgba-rgba-unsigned_short_4_4_4_4.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437

tests/wpt/webgl/meta/conformance/textures/video/tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html.ini

@@ -0,0 +1,2 @@
+[tex-2d-rgba-rgba-unsigned_short_5_5_5_1.html]
+  disabled: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/437