Animation: update image active frame when update the rendering (#36286)

When no CSS animation exist, register timer for image animation, update
animated image active image frame as part of update_the_rendering, mark
node as dirty if the corresponding image need update. Added unit test to
test ImageAnimationState.

Part of https://github.com/servo/servo/issues/36057, the last step to
let the Animated Image "Move".

Testing: Introduced new WPT RefTest for animated image, but fail because
of https://github.com/servo/servo/issues/36931. New unit test for
`ImageAnimationState`.
Fixes: https://github.com/servo/servo/issues/22903
https://github.com/servo/servo/issues/36057

[Try](1472472966)

---------

Signed-off-by: rayguo17 <rayguo17@gmail.com>

components/layout/context.rs

@@ -137,14 +137,23 @@ impl LayoutContext<'_> {
             if image_state.image_key() != image.id {
                 if image.should_animate() {
                     // i. Register/Replace tracking item in image_animation_manager.
-                    store.insert(node, ImageAnimationState::new(image));
+                    store.insert(
+                        node,
+                        ImageAnimationState::new(
+                            image,
+                            self.shared_context().current_time_for_animations,
+                        ),
+                    );
                 } else {
                     // ii. Cancel Action if the node's image is no longer animated.
                     store.remove(&node);
                 }
             }
         } else if image.should_animate() {
-            store.insert(node, ImageAnimationState::new(image));
+            store.insert(
+                node,
+                ImageAnimationState::new(image, self.shared_context().current_time_for_animations),
+            );
         }
     }
 

components/script/animations.rs

@@ -76,6 +76,10 @@ impl Animations {
         self.pending_events.borrow_mut().clear();
     }
 
+    pub(crate) fn animations_present(&self) -> bool {
+        self.has_running_animations.get() || !self.pending_events.borrow().is_empty()
+    }
+
     // Mark all animations dirty, if they haven't been marked dirty since the
     // specified `current_timeline_value`. Returns true if animations were marked
     // dirty or false otherwise.

components/script/dom/document.rs

@@ -5051,6 +5051,35 @@ impl Document {
         self.image_animation_manager.borrow_mut()
     }
 
+    pub(crate) fn update_animating_images(&self) {
+        let mut image_animation_manager = self.image_animation_manager.borrow_mut();
+        if !image_animation_manager.image_animations_present() {
+            return;
+        }
+        image_animation_manager
+            .update_active_frames(&self.window, self.current_animation_timeline_value());
+
+        if !self.animations().animations_present() {
+            let next_scheduled_time =
+                image_animation_manager.next_schedule_time(self.current_animation_timeline_value());
+            // TODO: Once we have refresh signal from the compositor,
+            // we should get rid of timer for animated image update.
+            if let Some(next_scheduled_time) = next_scheduled_time {
+                self.schedule_image_animation_update(next_scheduled_time);
+            }
+        }
+    }
+
+    fn schedule_image_animation_update(&self, next_scheduled_time: f64) {
+        let callback = ImageAnimationUpdateCallback {
+            document: Trusted::new(self),
+        };
+        self.global().schedule_callback(
+            OneshotTimerCallback::ImageAnimationUpdate(callback),
+            Duration::from_secs_f64(next_scheduled_time),
+        );
+    }
+
     /// <https://html.spec.whatwg.org/multipage/#shared-declarative-refresh-steps>
     pub(crate) fn shared_declarative_refresh_steps(&self, content: &[u8]) {
         // 1. If document's will declaratively refresh is true, then return.
@@ -6747,6 +6776,21 @@ impl FakeRequestAnimationFrameCallback {
     }
 }
 
+/// This is a temporary workaround to update animated images,
+/// we should get rid of this after we have refresh driver #3406
+#[derive(JSTraceable, MallocSizeOf)]
+pub(crate) struct ImageAnimationUpdateCallback {
+    /// The document.
+    #[ignore_malloc_size_of = "non-owning"]
+    document: Trusted<Document>,
+}
+
+impl ImageAnimationUpdateCallback {
+    pub(crate) fn invoke(self, can_gc: CanGc) {
+        with_script_thread(|script_thread| script_thread.update_the_rendering(true, can_gc))
+    }
+}
+
 #[derive(JSTraceable, MallocSizeOf)]
 pub(crate) enum AnimationFrameCallback {
     DevtoolsFramerateTick {

components/script/image_animation.rs

@@ -2,20 +2,37 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
-use fxhash::FxHashMap;
+use compositing_traits::{ImageUpdate, SerializableImageData};
+use embedder_traits::UntrustedNodeAddress;
+use fxhash::{FxHashMap, FxHashSet};
+use ipc_channel::ipc::IpcSharedMemory;
+use libc::c_void;
+use script_bindings::root::Dom;
 use script_layout_interface::ImageAnimationState;
 use style::dom::OpaqueNode;
+use webrender_api::units::DeviceIntSize;
+use webrender_api::{ImageDescriptor, ImageDescriptorFlags, ImageFormat};
+
+use crate::dom::bindings::cell::DomRefCell;
+use crate::dom::bindings::trace::NoTrace;
+use crate::dom::node::{Node, from_untrusted_node_address};
+use crate::dom::window::Window;
 
 #[derive(Clone, Debug, Default, JSTraceable, MallocSizeOf)]
+#[cfg_attr(crown, crown::unrooted_must_root_lint::must_root)]
 pub struct ImageAnimationManager {
     #[no_trace]
     pub node_to_image_map: FxHashMap<OpaqueNode, ImageAnimationState>,
+
+    /// A list of nodes with in-progress image animations.
+    rooted_nodes: DomRefCell<FxHashMap<NoTrace<OpaqueNode>, Dom<Node>>>,
 }
 
 impl ImageAnimationManager {
     pub fn new() -> Self {
         ImageAnimationManager {
             node_to_image_map: Default::default(),
+            rooted_nodes: DomRefCell::new(FxHashMap::default()),
         }
     }
 
@@ -25,5 +42,81 @@ impl ImageAnimationManager {
 
     pub fn restore_image_animate_set(&mut self, map: FxHashMap<OpaqueNode, ImageAnimationState>) {
         let _ = std::mem::replace(&mut self.node_to_image_map, map);
+        self.root_newly_animating_dom_nodes();
+        self.unroot_unused_nodes();
+    }
+
+    pub fn next_schedule_time(&self, now: f64) -> Option<f64> {
+        self.node_to_image_map
+            .values()
+            .map(|state| state.time_to_next_frame(now))
+            .min_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
+    }
+
+    pub fn image_animations_present(&self) -> bool {
+        !self.node_to_image_map.is_empty()
+    }
+
+    pub fn update_active_frames(&mut self, window: &Window, now: f64) {
+        let rooted_nodes = self.rooted_nodes.borrow();
+        let updates: Vec<ImageUpdate> = self
+            .node_to_image_map
+            .iter_mut()
+            .filter_map(|(node, state)| {
+                if state.update_frame_for_animation_timeline_value(now) {
+                    let image = &state.image;
+                    let frame = image
+                        .frames()
+                        .nth(state.active_frame)
+                        .expect("active_frame should within range of frames");
+
+                    if let Some(node) = rooted_nodes.get(&NoTrace(*node)) {
+                        node.dirty(crate::dom::node::NodeDamage::OtherNodeDamage);
+                    }
+                    Some(ImageUpdate::UpdateImage(
+                        image.id.unwrap(),
+                        ImageDescriptor {
+                            format: ImageFormat::BGRA8,
+                            size: DeviceIntSize::new(image.width as i32, image.height as i32),
+                            stride: None,
+                            offset: 0,
+                            flags: ImageDescriptorFlags::ALLOW_MIPMAPS,
+                        },
+                        SerializableImageData::Raw(IpcSharedMemory::from_bytes(frame.bytes)),
+                    ))
+                } else {
+                    None
+                }
+            })
+            .collect();
+        window.compositor_api().update_images(updates);
+    }
+
+    // Unroot any nodes that we have rooted but no longer have animating images.
+    fn unroot_unused_nodes(&self) {
+        let nodes: FxHashSet<&OpaqueNode> = self.node_to_image_map.keys().collect();
+        self.rooted_nodes
+            .borrow_mut()
+            .retain(|node, _| nodes.contains(&node.0));
+    }
+
+    /// Ensure that all nodes with Image animations are rooted. This should be called
+    /// immediately after a restyle, to ensure that these addresses are still valid.
+    #[allow(unsafe_code)]
+    fn root_newly_animating_dom_nodes(&self) {
+        let mut rooted_nodes = self.rooted_nodes.borrow_mut();
+        for opaque_node in self.node_to_image_map.keys() {
+            let opaque_node = *opaque_node;
+            if rooted_nodes.contains_key(&NoTrace(opaque_node)) {
+                continue;
+            }
+            let address = UntrustedNodeAddress(opaque_node.0 as *const c_void);
+            unsafe {
+                rooted_nodes.insert(
+                    NoTrace(opaque_node),
+                    Dom::from_ref(&*from_untrusted_node_address(address)),
+                )
+            };
+        }
     }
 }

components/script/script_thread.rs

@@ -1289,6 +1289,10 @@ impl ScriptThread {
 
             // TODO: Mark paint timing from https://w3c.github.io/paint-timing.
 
+            // Update the rendering of those does not require a reflow.
+            // e.g. animated images.
+            document.update_animating_images();
+
             #[cfg(feature = "webgpu")]
             document.update_rendering_of_webgpu_canvases();
 

components/script/timers.rs

@@ -24,7 +24,9 @@ use crate::dom::bindings::refcounted::Trusted;
 use crate::dom::bindings::reflector::{DomGlobal, DomObject};
 use crate::dom::bindings::root::Dom;
 use crate::dom::bindings::str::DOMString;
-use crate::dom::document::{FakeRequestAnimationFrameCallback, RefreshRedirectDue};
+use crate::dom::document::{
+    FakeRequestAnimationFrameCallback, ImageAnimationUpdateCallback, RefreshRedirectDue,
+};
 use crate::dom::eventsource::EventSourceTimeoutCallback;
 use crate::dom::globalscope::GlobalScope;
 #[cfg(feature = "testbinding")]
@@ -83,6 +85,7 @@ pub(crate) enum OneshotTimerCallback {
     TestBindingCallback(TestBindingCallback),
     FakeRequestAnimationFrame(FakeRequestAnimationFrameCallback),
     RefreshRedirectDue(RefreshRedirectDue),
+    ImageAnimationUpdate(ImageAnimationUpdateCallback),
 }
 
 impl OneshotTimerCallback {
@@ -95,6 +98,7 @@ impl OneshotTimerCallback {
             OneshotTimerCallback::TestBindingCallback(callback) => callback.invoke(),
             OneshotTimerCallback::FakeRequestAnimationFrame(callback) => callback.invoke(can_gc),
             OneshotTimerCallback::RefreshRedirectDue(callback) => callback.invoke(can_gc),
+            OneshotTimerCallback::ImageAnimationUpdate(callback) => callback.invoke(can_gc),
         }
     }
 }

components/shared/script_layout/lib.rs

@@ -507,21 +507,116 @@ pub fn node_id_from_scroll_id(id: usize) -> Option<usize> {
 #[derive(Clone, Debug, MallocSizeOf)]
 pub struct ImageAnimationState {
     #[ignore_malloc_size_of = "Arc is hard"]
-    image: Arc<Image>,
-    active_frame: usize,
+    pub image: Arc<Image>,
+    pub active_frame: usize,
     last_update_time: f64,
 }
 
 impl ImageAnimationState {
-    pub fn new(image: Arc<Image>) -> Self {
+    pub fn new(image: Arc<Image>, last_update_time: f64) -> Self {
         Self {
             image,
             active_frame: 0,
-            last_update_time: 0.,
+            last_update_time,
         }
     }
 
     pub fn image_key(&self) -> Option<ImageKey> {
         self.image.id
     }
+
+    pub fn time_to_next_frame(&self, now: f64) -> f64 {
+        let frame_delay = self
+            .image
+            .frames
+            .get(self.active_frame)
+            .expect("Image frame should always be valid")
+            .delay
+            .map_or(0., |delay| delay.as_secs_f64());
+        (frame_delay - now + self.last_update_time).max(0.0)
+    }
+
+    /// check whether image active frame need to be updated given current time,
+    /// return true if there are image that need to be updated.
+    /// false otherwise.
+    pub fn update_frame_for_animation_timeline_value(&mut self, now: f64) -> bool {
+        if self.image.frames.len() <= 1 {
+            return false;
+        }
+        let image = &self.image;
+        let time_interval_since_last_update = now - self.last_update_time;
+        let mut remain_time_interval = time_interval_since_last_update -
+            image
+                .frames
+                .get(self.active_frame)
+                .unwrap()
+                .delay
+                .unwrap()
+                .as_secs_f64();
+        let mut next_active_frame_id = self.active_frame;
+        while remain_time_interval > 0.0 {
+            next_active_frame_id = (next_active_frame_id + 1) % image.frames.len();
+            remain_time_interval -= image
+                .frames
+                .get(next_active_frame_id)
+                .unwrap()
+                .delay
+                .unwrap()
+                .as_secs_f64();
+        }
+        if self.active_frame == next_active_frame_id {
+            return false;
+        }
+        self.active_frame = next_active_frame_id;
+        self.last_update_time = now;
+        true
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use ipc_channel::ipc::IpcSharedMemory;
+    use pixels::{CorsStatus, Image, ImageFrame, PixelFormat};
+
+    use crate::ImageAnimationState;
+
+    #[test]
+    fn test() {
+        let image_frames: Vec<ImageFrame> = std::iter::repeat_with(|| ImageFrame {
+            delay: Some(Duration::from_millis(100)),
+            byte_range: 0..1,
+            width: 100,
+            height: 100,
+        })
+        .take(10)
+        .collect();
+        let image = Image {
+            width: 100,
+            height: 100,
+            format: PixelFormat::BGRA8,
+            id: None,
+            bytes: IpcSharedMemory::from_byte(1, 1),
+            frames: image_frames,
+            cors_status: CorsStatus::Unsafe,
+        };
+        let mut image_animation_state = ImageAnimationState::new(Arc::new(image), 0.0);
+
+        assert_eq!(image_animation_state.active_frame, 0);
+        assert_eq!(image_animation_state.last_update_time, 0.0);
+        assert_eq!(
+            image_animation_state.update_frame_for_animation_timeline_value(0.101),
+            true
+        );
+        assert_eq!(image_animation_state.active_frame, 1);
+        assert_eq!(image_animation_state.last_update_time, 0.101);
+        assert_eq!(
+            image_animation_state.update_frame_for_animation_timeline_value(0.116),
+            false
+        );
+        assert_eq!(image_animation_state.active_frame, 1);
+        assert_eq!(image_animation_state.last_update_time, 0.101);
+    }
 }

python/servo/testing_commands.py

@@ -175,6 +175,7 @@ class MachCommands(CommandBase):
             "net",
             "net_traits",
             "pixels",
+            "script_layout_interface",
             "script_traits",
             "selectors",
             "servo_config",

tests/wpt/meta/MANIFEST.json

@@ -353540,6 +353540,19 @@
          {}
         ]
        ],
+       "animated-image-update.tentative.html": [
+        "6875188ad1698155b7690ccdcbc3478fff4fff1c",
+        [
+         null,
+         [
+          [
+           "/html/semantics/embedded-content/the-img-element/animated-image-update-ref.tentative.html",
+           "=="
+          ]
+         ],
+         {}
+        ]
+       ],
        "available-images.html": [
         "779ff978689e4f5565b8d45d383efa75ac78b8b2",
         [
@@ -481177,6 +481190,10 @@
         "28d0ead2dbc3f9db5e7c2714cc72434b4bc367a2",
         []
        ],
+       "animated-image-update-ref.tentative.html": [
+        "f2d7fb6cff15f2a87144330249bbdc2698fad550",
+        []
+       ],
        "available-images-ref.html": [
         "8061abae50899a2befe286723d8bd5c285b356ab",
         []
@@ -481254,6 +481271,10 @@
         ]
        },
        "resources": {
+        "animated.gif": [
+         "7694add55e0c98ec3ee5d9110e5fb16b4d819137",
+         []
+        ],
         "blue-10.png": [
          "62949e08d87dfcdc0987eaef67692c7a1c16aa50",
          []

tests/wpt/tests/html/semantics/embedded-content/the-img-element/animated-image-update-ref.tentative.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<head>
+  <title>Reference for Animated image active frame update when passing delay time</title>
+</head>
+
+<body>
+    <img src="resources/green.png">
+</body>
+</html>

tests/wpt/tests/html/semantics/embedded-content/the-img-element/animated-image-update.tentative.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html class="reftest-wait">
+<head>
+  <title>Animated image active frame update when passing delay time</title>
+  <link rel="match" href="animated-image-update-ref.tentative.html">
+  <link rel="author" title="Tin Tun Aung" href="mailto:rayguo17@gmail.com">
+  <script src="/common/reftest-wait.js"></script>
+</head>
+
+<body>
+  <img id="image" src="resources/animated.gif">
+<script>
+  const image = document.getElementById('image');
+  requestAnimationFrame(()=>{
+    setTimeout(() => {
+      takeScreenshot();
+    }, 150);
+  })
+</script>
+</body>
+</html>