canvas: Add HTMLVideoElement to CanvasImageSource union type (#37135)

Follow to the specification and add HTMLVideoElement to
CanvasImageSource union type
to allow use it as image source for createPattern/drawImage operations.
https://html.spec.whatwg.org/multipage/#canvasimagesource

https://html.spec.whatwg.org/multipage/#dom-context-2d-createpattern
https://html.spec.whatwg.org/multipage/#dom-context-2d-drawimage

The HTMLVideoElement media resource has an associated origin:
- media provider object (MediaStream/MediaSource/Blob): CORS-same-origin
 - URL record: CORS-cross-origin/CORS-same-origin

https://html.spec.whatwg.org/multipage/media.html#media-resource

Testing:
 - html/canvas/element/*
 - html/semantics/embedded-content/the-canvas-element/*

Signed-off-by: Andrei Volykhin <andrei.volykhin@gmail.com>

components/script/canvas_state.rs

@@ -54,6 +54,7 @@ use crate::dom::dommatrix::DOMMatrix;
 use crate::dom::element::{Element, cors_setting_for_element};
 use crate::dom::globalscope::GlobalScope;
 use crate::dom::htmlcanvaselement::HTMLCanvasElement;
+use crate::dom::htmlvideoelement::HTMLVideoElement;
 use crate::dom::imagedata::ImageData;
 use crate::dom::node::{Node, NodeTraits};
 use crate::dom::offscreencanvas::OffscreenCanvas;
@@ -310,14 +311,15 @@ impl CanvasState {
         self.origin_clean.set(false)
     }
 
-    // https://html.spec.whatwg.org/multipage/#the-image-argument-is-not-origin-clean
+    /// <https://html.spec.whatwg.org/multipage/#the-image-argument-is-not-origin-clean>
-    fn is_origin_clean(&self, image: CanvasImageSource) -> bool {
+    fn is_origin_clean(&self, source: CanvasImageSource) -> bool {
-        match image {
+        match source {
-            CanvasImageSource::HTMLCanvasElement(canvas) => canvas.origin_is_clean(),
-            CanvasImageSource::OffscreenCanvas(canvas) => canvas.origin_is_clean(),
             CanvasImageSource::HTMLImageElement(image) => {
                 image.same_origin(GlobalScope::entry().origin())
             },
+            CanvasImageSource::HTMLVideoElement(video) => video.origin_is_clean(),
+            CanvasImageSource::HTMLCanvasElement(canvas) => canvas.origin_is_clean(),
+            CanvasImageSource::OffscreenCanvas(canvas) => canvas.origin_is_clean(),
             CanvasImageSource::CSSStyleValue(_) => true,
         }
     }
@@ -438,6 +440,17 @@ impl CanvasState {
         }
 
         let result = match image {
+            CanvasImageSource::HTMLVideoElement(ref video) => {
+                // <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
+                // Step 2. Let usability be the result of checking the usability of image.
+                // Step 3. If usability is bad, then return (without drawing anything).
+                if !video.is_usable() {
+                    return Ok(());
+                }
+
+                self.draw_html_video_element(video, htmlcanvas, sx, sy, sw, sh, dx, dy, dw, dh);
+                Ok(())
+            },
             CanvasImageSource::HTMLCanvasElement(ref canvas) => {
                 // <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
                 if canvas.get_size().is_empty() {
@@ -498,6 +511,52 @@ impl CanvasState {
         result
     }
 
+    /// <https://html.spec.whatwg.org/multipage/#dom-context-2d-drawimage>
+    #[allow(clippy::too_many_arguments)]
+    fn draw_html_video_element(
+        &self,
+        video: &HTMLVideoElement,
+        canvas: Option<&HTMLCanvasElement>,
+        sx: f64,
+        sy: f64,
+        sw: Option<f64>,
+        sh: Option<f64>,
+        dx: f64,
+        dy: f64,
+        dw: Option<f64>,
+        dh: Option<f64>,
+    ) {
+        let Some(snapshot) = video.get_current_frame_data() else {
+            return;
+        };
+
+        // Step 4. Establish the source and destination rectangles.
+        let video_size = snapshot.size().to_f64();
+        let dw = dw.unwrap_or(video_size.width);
+        let dh = dh.unwrap_or(video_size.height);
+        let sw = sw.unwrap_or(video_size.width);
+        let sh = sh.unwrap_or(video_size.height);
+
+        let (source_rect, dest_rect) =
+            self.adjust_source_dest_rects(video_size, sx, sy, sw, sh, dx, dy, dw, dh);
+
+        // Step 5. If one of the sw or sh arguments is zero, then return. Nothing is painted.
+        if !is_rect_valid(source_rect) || !is_rect_valid(dest_rect) {
+            return;
+        }
+
+        let smoothing_enabled = self.state.borrow().image_smoothing_enabled;
+
+        self.send_canvas_2d_msg(Canvas2dMsg::DrawImage(
+            snapshot.as_ipc(),
+            dest_rect,
+            source_rect,
+            smoothing_enabled,
+        ));
+
+        self.mark_as_dirty(canvas);
+    }
+
     #[allow(clippy::too_many_arguments)]
     fn draw_offscreen_canvas(
         &self,
@@ -966,7 +1025,7 @@ impl CanvasState {
         ))
     }
 
-    // https://html.spec.whatwg.org/multipage/#dom-context-2d-createpattern
+    /// <https://html.spec.whatwg.org/multipage/#dom-context-2d-createpattern>
     pub(crate) fn create_pattern(
         &self,
         global: &GlobalScope,
@@ -976,7 +1035,7 @@ impl CanvasState {
     ) -> Fallible<Option<DomRoot<CanvasPattern>>> {
         let snapshot = match image {
             CanvasImageSource::HTMLImageElement(ref image) => {
-                // https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument
+                // <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
                 if !image.is_usable()? {
                     return Ok(None);
                 }
@@ -988,10 +1047,28 @@ impl CanvasState {
                     })
                     .ok_or(Error::InvalidState)?
             },
+            CanvasImageSource::HTMLVideoElement(ref video) => {
+                // <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
+                if !video.is_usable() {
+                    return Ok(None);
+                }
+
+                video.get_current_frame_data().ok_or(Error::InvalidState)?
+            },
             CanvasImageSource::HTMLCanvasElement(ref canvas) => {
+                // <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
+                if canvas.get_size().is_empty() {
+                    return Err(Error::InvalidState);
+                }
+
                 canvas.get_image_data().ok_or(Error::InvalidState)?
             },
             CanvasImageSource::OffscreenCanvas(ref canvas) => {
+                // <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
+                if canvas.get_size().is_empty() {
+                    return Err(Error::InvalidState);
+                }
+
                 canvas.get_image_data().ok_or(Error::InvalidState)?
             },
             CanvasImageSource::CSSStyleValue(ref value) => value

components/script/dom/htmlmediaelement.rs

@@ -24,8 +24,8 @@ use js::jsapi::JSAutoRealm;
 use media::{GLPlayerMsg, GLPlayerMsgForward, WindowGLContext};
 use net_traits::request::{Destination, RequestId};
 use net_traits::{
-    FetchMetadata, FetchResponseListener, Metadata, NetworkError, ResourceFetchTiming,
+    FetchMetadata, FetchResponseListener, FilteredMetadata, Metadata, NetworkError,
-    ResourceTimingType,
+    ResourceFetchTiming, ResourceTimingType,
 };
 use pixels::RasterImage;
 use script_bindings::codegen::GenericBindings::TimeRangesBinding::TimeRangesMethods;
@@ -2069,6 +2069,28 @@ impl HTMLMediaElement {
             }
         }
     }
+
+    /// <https://html.spec.whatwg.org/multipage/#concept-media-load-resource>
+    pub(crate) fn origin_is_clean(&self) -> bool {
+        // Step 5.local (media provider object).
+        if self.src_object.borrow().is_some() {
+            // The resource described by the current media resource, if any,
+            // contains the media data. It is CORS-same-origin.
+            return true;
+        }
+
+        // Step 5.remote (URL record).
+        if self.resource_url.borrow().is_some() {
+            // Update the media data with the contents
+            // of response's unsafe response obtained in this fashion.
+            // Response can be CORS-same-origin or CORS-cross-origin;
+            if let Some(ref current_fetch_context) = *self.current_fetch_context.borrow() {
+                return current_fetch_context.origin_is_clean();
+            }
+        }
+
+        true
+    }
 }
 
 // XXX Placeholder for [https://github.com/servo/servo/issues/22293]
@@ -2654,6 +2676,8 @@ pub(crate) struct HTMLMediaElementFetchContext {
     cancel_reason: Option<CancelReason>,
     /// Indicates whether the fetched stream is seekable.
     is_seekable: bool,
+    /// Indicates whether the fetched stream is origin clean.
+    origin_clean: bool,
     /// Fetch canceller. Allows cancelling the current fetch request by
     /// manually calling its .cancel() method or automatically on Drop.
     fetch_canceller: FetchCanceller,
@@ -2664,6 +2688,7 @@ impl HTMLMediaElementFetchContext {
         HTMLMediaElementFetchContext {
             cancel_reason: None,
             is_seekable: false,
+            origin_clean: true,
             fetch_canceller: FetchCanceller::new(request_id),
         }
     }
@@ -2676,6 +2701,14 @@ impl HTMLMediaElementFetchContext {
         self.is_seekable = seekable;
     }
 
+    pub(crate) fn origin_is_clean(&self) -> bool {
+        self.origin_clean
+    }
+
+    fn set_origin_unclean(&mut self) {
+        self.origin_clean = false;
+    }
+
     fn cancel(&mut self, reason: CancelReason) {
         if self.cancel_reason.is_some() {
             return;
@@ -2730,6 +2763,16 @@ impl FetchResponseListener for HTMLMediaElementFetchListener {
             return;
         }
 
+        if let Ok(FetchMetadata::Filtered {
+            filtered: FilteredMetadata::Opaque | FilteredMetadata::OpaqueRedirect(_),
+            ..
+        }) = metadata
+        {
+            if let Some(ref mut current_fetch_context) = *elem.current_fetch_context.borrow_mut() {
+                current_fetch_context.set_origin_unclean();
+            }
+        }
+
         self.metadata = metadata.ok().map(|m| match m {
             FetchMetadata::Unfiltered(m) => m,
             FetchMetadata::Filtered { unsafe_, .. } => unsafe_,

components/script/dom/htmlvideoelement.rs

@@ -9,7 +9,6 @@ use content_security_policy as csp;
 use dom_struct::dom_struct;
 use euclid::default::Size2D;
 use html5ever::{LocalName, Prefix, local_name, ns};
-use ipc_channel::ipc;
 use js::rust::HandleObject;
 use net_traits::image_cache::{
     ImageCache, ImageCacheResult, ImageLoadListener, ImageOrMetadataAvailable, ImageResponse,
@@ -23,6 +22,7 @@ use net_traits::{
 use script_layout_interface::{HTMLMediaData, MediaMetadata};
 use servo_media::player::video::VideoFrame;
 use servo_url::ServoUrl;
+use snapshot::Snapshot;
 use style::attr::{AttrValue, LengthOrPercentageOrAuto};
 
 use crate::document_loader::{LoadBlocker, LoadType};
@@ -133,9 +133,7 @@ impl HTMLVideoElement {
         sent_resize
     }
 
-    pub(crate) fn get_current_frame_data(
+    pub(crate) fn get_current_frame_data(&self) -> Option<Snapshot> {
-        &self,
-    ) -> Option<(Option<ipc::IpcSharedMemory>, Size2D<u32>)> {
         let frame = self.htmlmediaelement.get_current_frame();
         if frame.is_some() {
             *self.last_frame.borrow_mut() = frame;
@@ -145,11 +143,19 @@ impl HTMLVideoElement {
             Some(frame) => {
                 let size = Size2D::new(frame.get_width() as u32, frame.get_height() as u32);
                 if !frame.is_gl_texture() {
-                    let data = Some(ipc::IpcSharedMemory::from_bytes(&frame.get_data()));
+                    let alpha_mode = snapshot::AlphaMode::Transparent {
-                    Some((data, size))
+                        premultiplied: false,
+                    };
+
+                    Some(Snapshot::from_vec(
+                        size.cast(),
+                        snapshot::PixelFormat::BGRA,
+                        alpha_mode,
+                        frame.get_data().to_vec(),
+                    ))
                 } else {
                     // XXX(victor): here we only have the GL texture ID.
-                    Some((None, size))
+                    Some(Snapshot::cleared(size.cast()))
                 }
             },
             None => None,
@@ -276,6 +282,18 @@ impl HTMLVideoElement {
             },
         }
     }
+
+    /// <https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument>
+    pub(crate) fn is_usable(&self) -> bool {
+        !matches!(
+            self.htmlmediaelement.get_ready_state(),
+            ReadyState::HaveNothing | ReadyState::HaveMetadata
+        )
+    }
+
+    pub(crate) fn origin_is_clean(&self) -> bool {
+        self.htmlmediaelement.origin_is_clean()
+    }
 }
 
 impl HTMLVideoElementMethods<crate::DomTypeHolder> for HTMLVideoElement {

components/script/dom/webgl2renderingcontext.rs

@@ -32,12 +32,11 @@ use crate::dom::bindings::codegen::Bindings::WebGL2RenderingContextBinding::{
     WebGL2RenderingContextConstants as constants, WebGL2RenderingContextMethods,
 };
 use crate::dom::bindings::codegen::Bindings::WebGLRenderingContextBinding::{
-    WebGLContextAttributes, WebGLRenderingContextMethods,
+    TexImageSource, WebGLContextAttributes, WebGLRenderingContextMethods,
 };
 use crate::dom::bindings::codegen::UnionTypes::{
     ArrayBufferViewOrArrayBuffer, Float32ArrayOrUnrestrictedFloatSequence,
-    HTMLCanvasElementOrOffscreenCanvas,
+    HTMLCanvasElementOrOffscreenCanvas, Int32ArrayOrLongSequence,
-    ImageDataOrHTMLImageElementOrHTMLCanvasElementOrHTMLVideoElement, Int32ArrayOrLongSequence,
     Uint32ArrayOrUnsignedLongSequence,
 };
 use crate::dom::bindings::error::{ErrorResult, Fallible};
@@ -3023,7 +3022,7 @@ impl WebGL2RenderingContextMethods<crate::DomTypeHolder> for WebGL2RenderingCont
         internal_format: i32,
         format: u32,
         data_type: u32,
-        source: ImageDataOrHTMLImageElementOrHTMLCanvasElementOrHTMLVideoElement,
+        source: TexImageSource,
     ) -> ErrorResult {
         self.base
             .TexImage2D_(target, level, internal_format, format, data_type, source)
@@ -3118,7 +3117,7 @@ impl WebGL2RenderingContextMethods<crate::DomTypeHolder> for WebGL2RenderingCont
         border: i32,
         format: u32,
         type_: u32,
-        source: ImageDataOrHTMLImageElementOrHTMLCanvasElementOrHTMLVideoElement,
+        source: TexImageSource,
     ) -> Fallible<()> {
         if self.bound_pixel_unpack_buffer.get().is_some() {
             self.base.webgl_error(InvalidOperation);
@@ -3301,7 +3300,7 @@ impl WebGL2RenderingContextMethods<crate::DomTypeHolder> for WebGL2RenderingCont
         yoffset: i32,
         format: u32,
         data_type: u32,
-        source: ImageDataOrHTMLImageElementOrHTMLCanvasElementOrHTMLVideoElement,
+        source: TexImageSource,
     ) -> ErrorResult {
         self.base
             .TexSubImage2D_(target, level, xoffset, yoffset, format, data_type, source)

components/script/dom/webglrenderingcontext.rs

@@ -658,14 +658,23 @@ impl WebGLRenderingContext {
                     return Ok(None);
                 }
             },
-            TexImageSource::HTMLVideoElement(video) => match video.get_current_frame_data() {
+            TexImageSource::HTMLVideoElement(video) => {
-                Some((data, size)) => {
+                if !video.origin_is_clean() {
-                    let data = data.unwrap_or_else(|| {
+                    return Err(Error::Security);
-                        IpcSharedMemory::from_bytes(&vec![0; size.area() as usize * 4])
+                }
-                    });
+
-                    TexPixels::new(data, size, PixelFormat::BGRA8, false)
+                let Some(snapshot) = video.get_current_frame_data() else {
-                },
+                    return Ok(None);
-                None => return Ok(None),
+                };
+
+                let snapshot = snapshot.as_ipc();
+                let size = snapshot.size().cast();
+                let format: PixelFormat = match snapshot.format() {
+                    snapshot::PixelFormat::RGBA => PixelFormat::RGBA8,
+                    snapshot::PixelFormat::BGRA => PixelFormat::BGRA8,
+                };
+                let premultiply = snapshot.alpha_mode().is_premultiplied();
+                TexPixels::new(snapshot.to_ipc_shared_memory(), size, format, premultiply)
             },
         }))
     }

components/script_bindings/webidls/CanvasRenderingContext2D.webidl

@@ -9,7 +9,7 @@
 typedef HTMLImageElement HTMLOrSVGImageElement;
 
 typedef (HTMLOrSVGImageElement or
-         /*HTMLVideoElement or*/
+         HTMLVideoElement or
          HTMLCanvasElement or
          /*ImageBitmap or*/
          OffscreenCanvas or

tests/wpt/meta/html/canvas/element/manual/wide-gamut-canvas/canvas-display-p3-drawImage-video.html.ini

@@ -1,70 +1,34 @@
 [canvas-display-p3-drawImage-video.html]
-  [sRGB-FF0100, Context srgb, ImageData srgb, scaleImage=false]
-    expected: FAIL
-
-  [sRGB-FF0100, Context srgb, ImageData srgb, scaleImage=true]
-    expected: FAIL
-
   [sRGB-FF0100, Context srgb, ImageData display-p3, scaleImage=false]
     expected: FAIL
 
   [sRGB-FF0100, Context srgb, ImageData display-p3, scaleImage=true]
     expected: FAIL
 
-  [sRGB-FF0100, Context display-p3, ImageData srgb, scaleImage=false]
-    expected: FAIL
-
-  [sRGB-FF0100, Context display-p3, ImageData srgb, scaleImage=true]
-    expected: FAIL
-
   [sRGB-FF0100, Context display-p3, ImageData display-p3, scaleImage=false]
     expected: FAIL
 
   [sRGB-FF0100, Context display-p3, ImageData display-p3, scaleImage=true]
     expected: FAIL
 
-  [sRGB-BB0000, Context srgb, ImageData srgb, scaleImage=false]
-    expected: FAIL
-
-  [sRGB-BB0000, Context srgb, ImageData srgb, scaleImage=true]
-    expected: FAIL
-
   [sRGB-BB0000, Context srgb, ImageData display-p3, scaleImage=false]
     expected: FAIL
 
   [sRGB-BB0000, Context srgb, ImageData display-p3, scaleImage=true]
     expected: FAIL
 
-  [sRGB-BB0000, Context display-p3, ImageData srgb, scaleImage=false]
-    expected: FAIL
-
-  [sRGB-BB0000, Context display-p3, ImageData srgb, scaleImage=true]
-    expected: FAIL
-
   [sRGB-BB0000, Context display-p3, ImageData display-p3, scaleImage=false]
     expected: FAIL
 
   [sRGB-BB0000, Context display-p3, ImageData display-p3, scaleImage=true]
     expected: FAIL
 
-  [Rec2020-3FF000000, Context srgb, ImageData srgb, scaleImage=false]
-    expected: FAIL
-
-  [Rec2020-3FF000000, Context srgb, ImageData srgb, scaleImage=true]
-    expected: FAIL
-
   [Rec2020-3FF000000, Context srgb, ImageData display-p3, scaleImage=false]
     expected: FAIL
 
   [Rec2020-3FF000000, Context srgb, ImageData display-p3, scaleImage=true]
     expected: FAIL
 
-  [Rec2020-3FF000000, Context display-p3, ImageData srgb, scaleImage=false]
-    expected: FAIL
-
-  [Rec2020-3FF000000, Context display-p3, ImageData srgb, scaleImage=true]
-    expected: FAIL
-
   [Rec2020-3FF000000, Context display-p3, ImageData display-p3, scaleImage=false]
     expected: FAIL
 

tests/wpt/meta/html/canvas/element/video/2d.video.invalid.html.ini

@@ -1,4 +0,0 @@
-[2d.video.invalid.html]
-  [Verify test doesn't crash with invalid video.]
-    expected: FAIL
-

tests/wpt/meta/html/semantics/embedded-content/the-canvas-element/security.pattern.fillStyle.sub.html.ini

@@ -1,5 +1,4 @@
 [security.pattern.fillStyle.sub.html]
-  expected: TIMEOUT
   [cross-origin SVGImageElement: Setting fillStyle to an origin-unclean pattern makes the canvas origin-unclean]
     expected: FAIL
 
@@ -10,31 +9,22 @@
     expected: FAIL
 
   [redirected to same-origin HTMLVideoElement: Setting fillStyle to an origin-unclean pattern makes the canvas origin-unclean]
-    expected: TIMEOUT
+    expected: FAIL
-
-  [unclean HTMLCanvasElement: Setting fillStyle to an origin-unclean pattern makes the canvas origin-unclean]
-    expected: NOTRUN
 
   [unclean ImageBitmap: Setting fillStyle to an origin-unclean pattern makes the canvas origin-unclean]
-    expected: NOTRUN
+    expected: FAIL
-
-  [cross-origin HTMLImageElement: Setting fillStyle to an origin-unclean offscreen canvas pattern makes the canvas origin-unclean]
-    expected: NOTRUN
 
   [cross-origin SVGImageElement: Setting fillStyle to an origin-unclean offscreen canvas pattern makes the canvas origin-unclean]
-    expected: NOTRUN
+    expected: FAIL
 
   [cross-origin HTMLVideoElement: Setting fillStyle to an origin-unclean offscreen canvas pattern makes the canvas origin-unclean]
-    expected: NOTRUN
+    expected: FAIL
 
   [redirected to cross-origin HTMLVideoElement: Setting fillStyle to an origin-unclean offscreen canvas pattern makes the canvas origin-unclean]
-    expected: NOTRUN
+    expected: FAIL
 
   [redirected to same-origin HTMLVideoElement: Setting fillStyle to an origin-unclean offscreen canvas pattern makes the canvas origin-unclean]
-    expected: NOTRUN
+    expected: FAIL
-
-  [unclean HTMLCanvasElement: Setting fillStyle to an origin-unclean offscreen canvas pattern makes the canvas origin-unclean]
-    expected: NOTRUN
 
   [unclean ImageBitmap: Setting fillStyle to an origin-unclean offscreen canvas pattern makes the canvas origin-unclean]
-    expected: NOTRUN
+    expected: FAIL