Improve how intrinsic sizes work for videos (#31746)

* feat: patch for video layout sizes

added rebase from main 2024/10/05

Co-authored-by: Josh Matthews <josh@joshmatthews.net>
Signed-off-by: eri <epazos@igalia.com>

* feat: take width and height parameters if provided

Signed-off-by: eri <epazos@igalia.com>

* chore: tidy the code and update test expectations

Signed-off-by: eri <epazos@igalia.com>

* feat: handle removing poster

Signed-off-by: eri <epazos@igalia.com>

* chore: update test expectations and remove debug code

Signed-off-by: eri <epazos@igalia.com>

* fix: issues after rebasing to main

Signed-off-by: eri <epazos@igalia.com>

* feat: pass src remove test and tidy

Signed-off-by: eri <epazos@igalia.com>

* chore: clippy fixes

Signed-off-by: eri <epazos@igalia.com>

* chore: update passing test expectations

Signed-off-by: eri <epazos@igalia.com>

* fix object-position-svg test

Signed-off-by: eri <epazos@igalia.com>

* fix unintentional override of video size and resize events

Signed-off-by: eri <epazos@igalia.com>

* change how resize events are sent to better match the spec

Signed-off-by: eri <epazos@igalia.com>

* simplify poster mutation handling

Co-authored-by: Oriol Brufau <obrufau@igalia.com>
Signed-off-by: eri <eri@inventati.org>

* improved handling of intrinsic sizes

- differentiate between natural size and css size
- presentational attributes
- fallback ratio for video element
- handle more cases where the src/poster are added/removed
- aspect ratio hints

Signed-off-by: eri <epazos@igalia.com>

* update test expectations

Signed-off-by: eri <epazos@igalia.com>

* fix cleaning current frame

Signed-off-by: eri <epazos@igalia.com>

* update test expectations

Signed-off-by: eri <epazos@igalia.com>

* Apply suggestions from code review

Co-authored-by: Oriol Brufau <obrufau@igalia.com>
Signed-off-by: eri <eri@inventati.org>

* More code review suggestions

Signed-off-by: eri <epazos@igalia.com>

* Prevent aspect-ratio:auto from pulling the ratio from the default object size

As resolved in https://github.com/w3c/csswg-drafts/issues/7524#issuecomment-1204462924

Signed-off-by: Oriol Brufau <obrufau@igalia.com>

---------

Signed-off-by: eri <epazos@igalia.com>
Signed-off-by: eri <eri@inventati.org>
Signed-off-by: Oriol Brufau <obrufau@igalia.com>
Co-authored-by: Josh Matthews <josh@joshmatthews.net>
Co-authored-by: Oriol Brufau <obrufau@igalia.com>

components/layout_2020/display_list/mod.rs

@@ -291,14 +291,16 @@ impl Fragment {
                         .to_webrender();
                     let common = builder.common_properties(clip, &image.style);
 
-                    builder.wr().push_image(
-                        &common,
-                        rect,
-                        image_rendering,
-                        wr::AlphaType::PremultipliedAlpha,
-                        image.image_key,
-                        wr::ColorF::WHITE,
-                    );
+                    if let Some(image_key) = image.image_key {
+                        builder.wr().push_image(
+                            &common,
+                            rect,
+                            image_rendering,
+                            wr::AlphaType::PremultipliedAlpha,
+                            image_key,
+                            wr::ColorF::WHITE,
+                        );
+                    }
                 },
                 Visibility::Hidden => (),
                 Visibility::Collapse => (),

components/layout_2020/dom.rs

@@ -102,7 +102,7 @@ pub(crate) trait NodeExt<'dom>: 'dom + LayoutNode<'dom> {
     fn as_image(self) -> Option<(Option<Arc<Image>>, PhysicalSize<f64>)>;
     fn as_canvas(self) -> Option<(CanvasInfo, PhysicalSize<f64>)>;
     fn as_iframe(self) -> Option<(PipelineId, BrowsingContextId)>;
-    fn as_video(self) -> Option<(webrender_api::ImageKey, PhysicalSize<f64>)>;
+    fn as_video(self) -> Option<(Option<webrender_api::ImageKey>, Option<PhysicalSize<f64>>)>;
     fn as_typeless_object_with_data_attribute(self) -> Option<String>;
     fn style(self, context: &LayoutContext) -> ServoArc<ComputedValues>;
 
@@ -136,11 +136,19 @@ where
         Some((resource, PhysicalSize::new(width, height)))
     }
 
-    fn as_video(self) -> Option<(webrender_api::ImageKey, PhysicalSize<f64>)> {
+    fn as_video(self) -> Option<(Option<webrender_api::ImageKey>, Option<PhysicalSize<f64>>)> {
         let node = self.to_threadsafe();
-        let frame_data = node.media_data()?.current_frame?;
-        let (width, height) = (frame_data.1 as f64, frame_data.2 as f64);
-        Some((frame_data.0, PhysicalSize::new(width, height)))
+        let data = node.media_data()?;
+        let natural_size = if let Some(frame) = data.current_frame {
+            Some(PhysicalSize::new(frame.width.into(), frame.height.into()))
+        } else {
+            data.metadata
+                .map(|meta| PhysicalSize::new(meta.width.into(), meta.height.into()))
+        };
+        Some((
+            data.current_frame.map(|frame| frame.image_key),
+            natural_size,
+        ))
     }
 
     fn as_canvas(self) -> Option<(CanvasInfo, PhysicalSize<f64>)> {

components/layout_2020/fragment_tree/fragment.rs

@@ -85,7 +85,7 @@ pub(crate) struct ImageFragment {
     pub rect: PhysicalRect<Au>,
     pub clip: PhysicalRect<Au>,
     #[serde(skip_serializing)]
-    pub image_key: ImageKey,
+    pub image_key: Option<ImageKey>,
 }
 
 #[derive(Serialize)]

components/layout_2020/replaced.rs

@@ -137,7 +137,7 @@ pub(crate) enum ReplacedContentKind {
     Image(Option<Arc<Image>>),
     IFrame(IFrameInfo),
     Canvas(CanvasInfo),
-    Video(VideoInfo),
+    Video(Option<VideoInfo>),
 }
 
 impl ReplacedContent {
@@ -173,24 +173,25 @@ impl ReplacedContent {
                 )
             } else if let Some((image_key, natural_size_in_dots)) = element.as_video() {
                 (
-                    ReplacedContentKind::Video(VideoInfo { image_key }),
-                    Some(natural_size_in_dots),
+                    ReplacedContentKind::Video(image_key.map(|key| VideoInfo { image_key: key })),
+                    natural_size_in_dots,
                 )
             } else {
                 return None;
             }
         };
 
-        let natural_size =
-            natural_size_in_dots.map_or_else(NaturalSizes::empty, |naturalc_size_in_dots| {
-                // FIXME: should 'image-resolution' (when implemented) be used *instead* of
-                // `script::dom::htmlimageelement::ImageRequest::current_pixel_density`?
-                // https://drafts.csswg.org/css-images-4/#the-image-resolution
-                let dppx = 1.0;
-                let width = (naturalc_size_in_dots.width as CSSFloat) / dppx;
-                let height = (naturalc_size_in_dots.height as CSSFloat) / dppx;
-                NaturalSizes::from_width_and_height(width, height)
-            });
+        let natural_size = if let Some(naturalc_size_in_dots) = natural_size_in_dots {
+            // FIXME: should 'image-resolution' (when implemented) be used *instead* of
+            // `script::dom::htmlimageelement::ImageRequest::current_pixel_density`?
+            // https://drafts.csswg.org/css-images-4/#the-image-resolution
+            let dppx = 1.0;
+            let width = (naturalc_size_in_dots.width as CSSFloat) / dppx;
+            let height = (naturalc_size_in_dots.height as CSSFloat) / dppx;
+            NaturalSizes::from_width_and_height(width, height)
+        } else {
+            NaturalSizes::empty()
+        };
 
         let base_fragment_info = BaseFragmentInfo::new_for_node(element.opaque());
         Some(Self {
@@ -354,7 +355,7 @@ impl ReplacedContent {
                         style: style.clone(),
                         rect,
                         clip,
-                        image_key,
+                        image_key: Some(image_key),
                     })
                 })
                 .into_iter()
@@ -364,7 +365,7 @@ impl ReplacedContent {
                 style: style.clone(),
                 rect,
                 clip,
-                image_key: video.image_key,
+                image_key: video.as_ref().map(|video| video.image_key),
             })],
             ReplacedContentKind::IFrame(iframe) => {
                 vec![Fragment::IFrame(IFrameFragment {
@@ -403,7 +404,7 @@ impl ReplacedContent {
                     style: style.clone(),
                     rect,
                     clip,
-                    image_key,
+                    image_key: Some(image_key),
                 })]
             },
         }
@@ -449,6 +450,16 @@ impl ReplacedContent {
         )
     }
 
+    pub(crate) fn default_object_size() -> PhysicalSize<Au> {
+        // FIXME:
+        // https://drafts.csswg.org/css-images/#default-object-size
+        // “If 300px is too wide to fit the device, UAs should use the width of
+        //  the largest rectangle that has a 2:1 ratio and fits the device instead.”
+        // “height of the largest rectangle that has a 2:1 ratio, has a height not greater
+        //  than 150px, and has a width not greater than the device width.”
+        PhysicalSize::new(Au::from_px(300), Au::from_px(150))
+    }
+
     /// <https://drafts.csswg.org/css2/visudet.html#inline-replaced-width>
     /// <https://drafts.csswg.org/css2/visudet.html#inline-replaced-height>
     ///
@@ -464,20 +475,19 @@ impl ReplacedContent {
     ) -> LogicalVec2<Au> {
         let mode = style.writing_mode;
         let intrinsic_size = self.flow_relative_intrinsic_size(style);
-        let intrinsic_ratio = self.preferred_aspect_ratio(&containing_block.into(), style);
+        let intrinsic_ratio = self
+            .preferred_aspect_ratio(&containing_block.into(), style)
+            .or_else(|| {
+                matches!(self.kind, ReplacedContentKind::Video(_)).then(|| {
+                    let size = Self::default_object_size();
+                    AspectRatio::from_content_ratio(
+                        size.width.to_f32_px() / size.height.to_f32_px(),
+                    )
+                })
+            });
 
-        let default_object_size = || {
-            // FIXME:
-            // https://drafts.csswg.org/css-images/#default-object-size
-            // “If 300px is too wide to fit the device, UAs should use the width of
-            //  the largest rectangle that has a 2:1 ratio and fits the device instead.”
-            // “height of the largest rectangle that has a 2:1 ratio, has a height not greater
-            //  than 150px, and has a width not greater than the device width.”
-            LogicalVec2::from_physical_size(
-                &PhysicalSize::new(Au::from_px(300), Au::from_px(150)),
-                mode,
-            )
-        };
+        let default_object_size =
+            || LogicalVec2::from_physical_size(&Self::default_object_size(), mode);
 
         let get_tentative_size = |LogicalVec2 { inline, block }| -> LogicalVec2<Au> {
             match (inline, block) {

components/layout_2020/style_ext.rs

@@ -177,6 +177,13 @@ impl AspectRatio {
             },
         }
     }
+
+    pub(crate) fn from_content_ratio(i_over_b: CSSFloat) -> Self {
+        Self {
+            box_sizing_adjustment: LogicalVec2::zero(),
+            i_over_b,
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -905,25 +912,18 @@ impl ComputedValuesExt for ComputedValues {
             // ratio; otherwise the box has no preferred aspect ratio. Size
             // calculations involving the aspect ratio work with the content box
             // dimensions always."
-            (_, PreferredRatio::None) => natural_aspect_ratio.map(|natural_ratio| AspectRatio {
-                i_over_b: natural_ratio,
-                box_sizing_adjustment: LogicalVec2::zero(),
-            }),
+            (_, PreferredRatio::None) => natural_aspect_ratio.map(AspectRatio::from_content_ratio),
             // "If both auto and a <ratio> are specified together, the preferred
             // aspect ratio is the specified ratio of width / height unless it
             // is a replaced element with a natural aspect ratio, in which case
             // that aspect ratio is used instead. In all cases, size
             // calculations involving the aspect ratio work with the content box
             // dimensions always."
-            (true, PreferredRatio::Ratio(preferred_ratio)) => match natural_aspect_ratio {
-                Some(natural_ratio) => Some(AspectRatio {
-                    i_over_b: natural_ratio,
-                    box_sizing_adjustment: LogicalVec2::zero(),
-                }),
-                None => Some(AspectRatio {
-                    i_over_b: (preferred_ratio.0).0 / (preferred_ratio.1).0,
-                    box_sizing_adjustment: LogicalVec2::zero(),
-                }),
+            (true, PreferredRatio::Ratio(preferred_ratio)) => {
+                Some(AspectRatio::from_content_ratio(
+                    natural_aspect_ratio
+                        .unwrap_or_else(|| (preferred_ratio.0).0 / (preferred_ratio.1).0),
+                ))
             },
 
             // "The box’s preferred aspect ratio is the specified ratio of width