compositor: Allow canvas to upload rendered contents asynchronously (#37776)

Adds epoch to each WR image op command that is sent to compositor. The
renderer now has a `FrameDelayer` data structure that is responsible for
tracking when a frame is ready to be displayed. When asking canvases to
update their rendering, they are given an optional `Epoch` which denotes
the `Document`'s canvas epoch. When all image updates for that `Epoch`
are seen in the renderer, the frame can be displayed.

Testing: Existing WPT tests
Fixes: #35733

Signed-off-by: sagudev <16504129+sagudev@users.noreply.github.com>
Signed-off-by: Martin Robinson <mrobinson@igalia.com>
Co-authored-by: Martin Robinson <mrobinson@igalia.com>

components/script/canvas_context.rs

@@ -4,6 +4,7 @@
 
 //! Common interfaces for Canvas Contexts
 
+use base::Epoch;
 use euclid::default::Size2D;
 use layout_api::HTMLCanvasData;
 use pixels::Snapshot;
@@ -66,7 +67,14 @@ pub(crate) trait CanvasContext {
         }
     }
 
-    fn update_rendering(&self) {}
+    /// The WebRender [`ImageKey`] of this [`CanvasContext`] if any.
+    fn image_key(&self) -> Option<ImageKey>;
+
+    /// Request that the [`CanvasContext`] update the rendering of its contents, returning
+    /// the new [`Epoch`] of the image produced, if one was.
+    fn update_rendering(&self, _canvas_epoch: Option<Epoch>) -> bool {
+        false
+    }
 
     fn onscreen(&self) -> bool {
         let Some(canvas) = self.canvas() else {
@@ -228,19 +236,31 @@ impl CanvasContext for RenderingContext {
         }
     }
 
-    fn update_rendering(&self) {
+    fn image_key(&self) -> Option<ImageKey> {
         match self {
-            RenderingContext::Placeholder(offscreen_canvas) => {
-                if let Some(context) = offscreen_canvas.context() {
-                    context.update_rendering()
-                }
-            },
-            RenderingContext::Context2d(context) => context.update_rendering(),
-            RenderingContext::BitmapRenderer(context) => context.update_rendering(),
-            RenderingContext::WebGL(context) => context.update_rendering(),
-            RenderingContext::WebGL2(context) => context.update_rendering(),
+            RenderingContext::Placeholder(offscreen_canvas) => offscreen_canvas
+                .context()
+                .and_then(|context| context.image_key()),
+            RenderingContext::Context2d(context) => context.image_key(),
+            RenderingContext::BitmapRenderer(context) => context.image_key(),
+            RenderingContext::WebGL(context) => context.image_key(),
+            RenderingContext::WebGL2(context) => context.image_key(),
             #[cfg(feature = "webgpu")]
-            RenderingContext::WebGPU(context) => context.update_rendering(),
+            RenderingContext::WebGPU(context) => context.image_key(),
+        }
+    }
+
+    fn update_rendering(&self, canvas_epoch: Option<Epoch>) -> bool {
+        match self {
+            RenderingContext::Placeholder(offscreen_canvas) => offscreen_canvas
+                .context()
+                .is_some_and(|context| context.update_rendering(canvas_epoch)),
+            RenderingContext::Context2d(context) => context.update_rendering(canvas_epoch),
+            RenderingContext::BitmapRenderer(context) => context.update_rendering(canvas_epoch),
+            RenderingContext::WebGL(context) => context.update_rendering(canvas_epoch),
+            RenderingContext::WebGL2(context) => context.update_rendering(canvas_epoch),
+            #[cfg(feature = "webgpu")]
+            RenderingContext::WebGPU(context) => context.update_rendering(canvas_epoch),
         }
     }
 
@@ -333,11 +353,17 @@ impl CanvasContext for OffscreenRenderingContext {
         }
     }
 
-    fn update_rendering(&self) {
+    fn image_key(&self) -> Option<ImageKey> {
+        None
+    }
+
+    fn update_rendering(&self, canvas_epoch: Option<Epoch>) -> bool {
         match self {
-            OffscreenRenderingContext::Context2d(context) => context.update_rendering(),
-            OffscreenRenderingContext::BitmapRenderer(context) => context.update_rendering(),
-            OffscreenRenderingContext::Detached => {},
+            OffscreenRenderingContext::Context2d(context) => context.update_rendering(canvas_epoch),
+            OffscreenRenderingContext::BitmapRenderer(context) => {
+                context.update_rendering(canvas_epoch)
+            },
+            OffscreenRenderingContext::Detached => false,
         }
     }
 

components/script/canvas_state.rs

@@ -8,6 +8,7 @@ use std::str::FromStr;
 use std::sync::Arc;
 
 use app_units::Au;
+use base::Epoch;
 use canvas_traits::canvas::{
     Canvas2dMsg, CanvasFont, CanvasId, CanvasMsg, CompositionOptions, CompositionOrBlending,
     FillOrStrokeStyle, FillRule, GlyphAndPosition, LineCapStyle, LineJoinStyle, LineOptions,
@@ -287,19 +288,18 @@ impl CanvasState {
     }
 
     /// Updates WR image and blocks on completion
-    pub(crate) fn update_rendering(&self) {
+    pub(crate) fn update_rendering(&self, canvas_epoch: Option<Epoch>) -> bool {
         if !self.is_paintable() {
-            return;
+            return false;
         }
 
-        let (sender, receiver) = ipc::channel().unwrap();
         self.ipc_renderer
             .send(CanvasMsg::Canvas2d(
-                Canvas2dMsg::UpdateImage(sender),
+                Canvas2dMsg::UpdateImage(canvas_epoch),
                 self.canvas_id,
             ))
             .unwrap();
-        receiver.recv().unwrap();
+        true
     }
 
     /// <https://html.spec.whatwg.org/multipage/#concept-canvas-set-bitmap-dimensions>

components/script/dom/canvasrenderingcontext2d.rs

@@ -2,6 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+use base::Epoch;
 use canvas_traits::canvas::{Canvas2dMsg, CanvasId};
 use dom_struct::dom_struct;
 use euclid::default::Size2D;
@@ -122,8 +123,11 @@ impl CanvasContext for CanvasRenderingContext2D {
         Some(self.canvas.clone())
     }
 
-    fn update_rendering(&self) {
-        self.canvas_state.update_rendering();
+    fn update_rendering(&self, canvas_epoch: Option<Epoch>) -> bool {
+        if !self.onscreen() {
+            return false;
+        }
+        self.canvas_state.update_rendering(canvas_epoch)
     }
 
     fn resize(&self) {
@@ -155,6 +159,10 @@ impl CanvasContext for CanvasRenderingContext2D {
             canvas.owner_document().add_dirty_2d_canvas(self);
         }
     }
+
+    fn image_key(&self) -> Option<ImageKey> {
+        Some(self.canvas_state.image_key())
+    }
 }
 
 // We add a guard to each of methods by the spec:

components/script/dom/document.rs

@@ -12,10 +12,11 @@ use std::str::FromStr;
 use std::sync::{LazyLock, Mutex};
 use std::time::Duration;
 
+use base::Epoch;
 use base::cross_process_instant::CrossProcessInstant;
 use base::id::WebViewId;
 use canvas_traits::canvas::CanvasId;
-use canvas_traits::webgl::{self, WebGLContextId, WebGLMsg};
+use canvas_traits::webgl::{WebGLContextId, WebGLMsg};
 use chrono::Local;
 use constellation_traits::{NavigationHistoryBehavior, ScriptToConstellationMessage};
 use content_security_policy::{CspList, PolicyDisposition};
@@ -556,6 +557,15 @@ pub(crate) struct Document {
     /// When a `ResizeObserver` starts observing a target, this becomes true, which in turn is a
     /// signal to the [`ScriptThread`] that a rendering update should happen.
     resize_observer_started_observing_target: Cell<bool>,
+    /// Whether or not this [`Document`] is waiting on canvas image updates. If it is
+    /// waiting it will not do any new layout until the canvas images are up-to-date in
+    /// the renderer.
+    waiting_on_canvas_image_updates: Cell<bool>,
+    /// The current canvas epoch, which is used to track when canvas images have been
+    /// uploaded to the renderer after a rendering update. Until those images are uploaded
+    /// this `Document` will not perform any more rendering updates.
+    #[no_trace]
+    current_canvas_epoch: RefCell<Epoch>,
 }
 
 #[allow(non_snake_case)]
@@ -2671,40 +2681,69 @@ impl Document {
         }
 
         // All dirty canvases are flushed before updating the rendering.
-        #[cfg(feature = "webgpu")]
-        self.webgpu_contexts
-            .borrow_mut()
-            .iter()
-            .filter_map(|(_, context)| context.root())
-            .filter(|context| context.onscreen())
-            .for_each(|context| context.update_rendering());
+        self.current_canvas_epoch.borrow_mut().next();
+        let canvas_epoch = *self.current_canvas_epoch.borrow();
+        let mut image_keys = Vec::new();
 
-        self.dirty_2d_contexts
-            .borrow_mut()
-            .drain()
-            .filter(|(_, context)| context.onscreen())
-            .for_each(|(_, context)| context.update_rendering());
+        #[cfg(feature = "webgpu")]
+        image_keys.extend(
+            self.webgpu_contexts
+                .borrow_mut()
+                .iter()
+                .filter_map(|(_, context)| context.root())
+                .filter(|context| context.update_rendering(Some(canvas_epoch)))
+                .map(|context| context.image_key()),
+        );
+
+        image_keys.extend(
+            self.dirty_2d_contexts
+                .borrow_mut()
+                .drain()
+                .filter(|(_, context)| context.update_rendering(Some(canvas_epoch)))
+                .map(|(_, context)| context.image_key()),
+        );
 
         let dirty_webgl_context_ids: Vec<_> = self
             .dirty_webgl_contexts
             .borrow_mut()
             .drain()
             .filter(|(_, context)| context.onscreen())
-            .map(|(id, _)| id)
+            .map(|(id, context)| {
+                image_keys.push(context.image_key());
+                id
+            })
             .collect();
+
         if !dirty_webgl_context_ids.is_empty() {
-            let (sender, receiver) = webgl::webgl_channel().unwrap();
             self.window
                 .webgl_chan()
                 .expect("Where's the WebGL channel?")
-                .send(WebGLMsg::SwapBuffers(dirty_webgl_context_ids, sender, 0))
+                .send(WebGLMsg::SwapBuffers(
+                    dirty_webgl_context_ids,
+                    Some(canvas_epoch),
+                    0,
+                ))
                 .unwrap();
-            receiver.recv().unwrap();
+        }
+
+        // The renderer should wait to display the frame until all canvas images are
+        // uploaded. This allows canvas image uploading to happen asynchronously.
+        if !image_keys.is_empty() {
+            self.waiting_on_canvas_image_updates.set(true);
+            self.window().compositor_api().delay_new_frame_for_canvas(
+                self.window().pipeline_id(),
+                canvas_epoch,
+                image_keys.into_iter().flatten().collect(),
+            );
         }
 
         self.window().reflow(ReflowGoal::UpdateTheRendering)
     }
 
+    pub(crate) fn handle_no_longer_waiting_on_asynchronous_image_updates(&self) {
+        self.waiting_on_canvas_image_updates.set(false);
+    }
+
     /// From <https://drafts.csswg.org/css-font-loading/#fontfaceset-pending-on-the-environment>:
     ///
     /// > A FontFaceSet is pending on the environment if any of the following are true:
@@ -3390,6 +3429,8 @@ impl Document {
             adopted_stylesheets_frozen_types: CachedFrozenArray::new(),
             pending_scroll_event_targets: Default::default(),
             resize_observer_started_observing_target: Cell::new(false),
+            waiting_on_canvas_image_updates: Cell::new(false),
+            current_canvas_epoch: RefCell::new(Epoch(0)),
         }
     }
 

components/script/dom/gpucanvascontext.rs

@@ -2,6 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
+use base::Epoch;
 use dom_struct::dom_struct;
 use webrender_api::ImageKey;
 

components/script/dom/htmlmediaelement.rs

@@ -224,6 +224,7 @@ impl VideoFrameRenderer for MediaFrameRenderer {
                         current_frame.image_key,
                         descriptor,
                         SerializableImageData::Raw(IpcSharedMemory::from_bytes(&frame.get_data())),
+                        None,
                     ));
                 }
 

components/script/dom/imagebitmaprenderingcontext.rs

@@ -150,6 +150,10 @@ impl CanvasContext for ImageBitmapRenderingContext {
             .as_ref()
             .map_or_else(|| self.canvas.size(), |bitmap| bitmap.size())
     }
+
+    fn image_key(&self) -> Option<ImageKey> {
+        None
+    }
 }
 
 impl ImageBitmapRenderingContextMethods<crate::DomTypeHolder> for ImageBitmapRenderingContext {

components/script/dom/offscreencanvasrenderingcontext2d.rs

@@ -97,6 +97,10 @@ impl CanvasContext for OffscreenCanvasRenderingContext2D {
     fn origin_is_clean(&self) -> bool {
         self.context.origin_is_clean()
     }
+
+    fn image_key(&self) -> Option<webrender_api::ImageKey> {
+        None
+    }
 }
 
 impl OffscreenCanvasRenderingContext2DMethods<crate::DomTypeHolder>

components/script/dom/paintrenderingcontext2d.rs

@@ -63,9 +63,12 @@ impl PaintRenderingContext2D {
         ))
     }
 
+    pub(crate) fn update_rendering(&self) -> bool {
+        self.canvas_state.update_rendering(None)
+    }
+
     /// Send update to canvas paint thread and returns [`ImageKey`]
     pub(crate) fn image_key(&self) -> ImageKey {
-        self.canvas_state.update_rendering();
         self.canvas_state.image_key()
     }
 

components/script/dom/paintworkletglobalscope.rs

@@ -348,13 +348,13 @@ impl PaintWorkletGlobalScope {
             return self.invalid_image(size_in_dpx, missing_image_urls);
         }
 
-        let image_key = rendering_context.image_key();
+        rendering_context.update_rendering();
 
         DrawAPaintImageResult {
             width: size_in_dpx.width,
             height: size_in_dpx.height,
             format: PixelFormat::BGRA8,
-            image_key: Some(image_key),
+            image_key: Some(rendering_context.image_key()),
             missing_image_urls,
         }
     }

components/script/dom/webgl/webgl2renderingcontext.rs

@@ -992,6 +992,10 @@ impl CanvasContext for WebGL2RenderingContext {
     fn mark_as_dirty(&self) {
         self.base.mark_as_dirty()
     }
+
+    fn image_key(&self) -> Option<ImageKey> {
+        self.base.image_key()
+    }
 }
 
 impl WebGL2RenderingContextMethods<crate::DomTypeHolder> for WebGL2RenderingContext {

components/script/dom/webgl/webglrenderingcontext.rs

@@ -2042,6 +2042,10 @@ impl CanvasContext for WebGLRenderingContext {
             HTMLCanvasElementOrOffscreenCanvas::OffscreenCanvas(_) => {},
         }
     }
+
+    fn image_key(&self) -> Option<ImageKey> {
+        Some(self.webrender_image)
+    }
 }
 
 #[cfg(not(feature = "webgl_backtrace"))]

components/script/dom/webgpu/gpucanvascontext.rs

@@ -6,6 +6,7 @@ use std::borrow::Cow;
 use std::cell::RefCell;
 
 use arrayvec::ArrayVec;
+use base::Epoch;
 use dom_struct::dom_struct;
 use ipc_channel::ipc::{self};
 use pixels::Snapshot;
@@ -183,19 +184,29 @@ impl GPUCanvasContext {
     }
 
     /// <https://gpuweb.github.io/gpuweb/#abstract-opdef-expire-the-current-texture>
-    fn expire_current_texture(&self) {
-        if let Some(current_texture) = self.current_texture.take() {
-            // Make copy of texture content
-            self.send_swap_chain_present(current_texture.id());
-            // Step 1
-            current_texture.Destroy()
-        }
+    fn expire_current_texture(&self, canvas_epoch: Option<Epoch>) -> bool {
+        // Step 1: If context.[[currentTexture]] is not null:
+        let Some(current_texture) = self.current_texture.take() else {
+            return false;
+        };
+
+        // Make copy of texture content
+        let did_swap = self.send_swap_chain_present(current_texture.id(), canvas_epoch);
+
+        // Step 1.1: Call context.currentTexture.destroy() (without destroying
+        // context.drawingBuffer) to terminate write access to the image.
+        current_texture.Destroy();
+
+        // Step 1.2: Set context.[[currentTexture]] to null.
+        // This is handled by the call to `.take()` above.
+
+        did_swap
     }
 
     /// <https://gpuweb.github.io/gpuweb/#abstract-opdef-replace-the-drawing-buffer>
     fn replace_drawing_buffer(&self) {
         // Step 1
-        self.expire_current_texture();
+        self.expire_current_texture(None);
         // Step 2
         let configuration = self.configuration.borrow();
         // Step 3
@@ -234,19 +245,28 @@ impl GPUCanvasContext {
         }
     }
 
-    fn send_swap_chain_present(&self, texture_id: WebGPUTexture) {
+    fn send_swap_chain_present(
+        &self,
+        texture_id: WebGPUTexture,
+        canvas_epoch: Option<Epoch>,
+    ) -> bool {
         self.drawing_buffer.borrow_mut().cleared = false;
         let encoder_id = self.global().wgpu_id_hub().create_command_encoder_id();
-        if let Err(e) = self.channel.0.send(WebGPURequest::SwapChainPresent {
+        let send_result = self.channel.0.send(WebGPURequest::SwapChainPresent {
             context_id: self.context_id,
             texture_id: texture_id.0,
             encoder_id,
-        }) {
+            canvas_epoch,
+        });
+
+        if let Err(error) = &send_result {
             warn!(
-                "Failed to send UpdateWebrenderData({:?}) ({})",
-                self.context_id, e
+                "Failed to send UpdateWebrenderData({:?}) ({error})",
+                self.context_id,
             );
         }
+
+        send_result.is_ok()
     }
 }
 
@@ -258,9 +278,19 @@ impl CanvasContext for GPUCanvasContext {
     }
 
     /// <https://gpuweb.github.io/gpuweb/#abstract-opdef-updating-the-rendering-of-a-webgpu-canvas>
-    fn update_rendering(&self) {
-        // Step 1
-        self.expire_current_texture();
+    fn update_rendering(&self, canvas_epoch: Option<Epoch>) -> bool {
+        if !self.onscreen() {
+            return false;
+        }
+
+        // Step 1: Expire the current texture of context.
+        self.expire_current_texture(canvas_epoch)
+        // Step 2: Set context.[[lastPresentedImage]] to context.[[drawingBuffer]].
+        // TODO: Implement this.
+    }
+
+    fn image_key(&self) -> Option<ImageKey> {
+        Some(self.webrender_image)
     }
 
     /// <https://gpuweb.github.io/gpuweb/#abstract-opdef-update-the-canvas-size>

components/script/image_animation.rs

@@ -101,6 +101,7 @@ impl ImageAnimationManager {
                         flags: ImageDescriptorFlags::ALLOW_MIPMAPS,
                     },
                     SerializableImageData::Raw(IpcSharedMemory::from_bytes(frame.bytes)),
+                    None,
                 ))
             })
             .collect();

components/script/messaging.rs

@@ -98,6 +98,7 @@ impl MixedMessage {
                 ScriptThreadMessage::EvaluateJavaScript(id, _, _) => Some(*id),
                 ScriptThreadMessage::SendImageKeysBatch(..) => None,
                 ScriptThreadMessage::PreferencesUpdated(..) => None,
+                ScriptThreadMessage::NoLongerWaitingOnAsychronousImageUpdates(_) => None,
             },
             MixedMessage::FromScript(inner_msg) => match inner_msg {
                 MainThreadScriptMsg::Common(CommonScriptMsg::Task(_, _, pipeline_id, _)) => {

components/script/script_thread.rs

@@ -1424,6 +1424,13 @@ impl ScriptThread {
                 )) => {
                     self.set_needs_rendering_update();
                 },
+                MixedMessage::FromConstellation(
+                    ScriptThreadMessage::NoLongerWaitingOnAsychronousImageUpdates(pipeline_id),
+                ) => {
+                    if let Some(document) = self.documents.borrow().find_document(pipeline_id) {
+                        document.handle_no_longer_waiting_on_asynchronous_image_updates();
+                    }
+                },
                 MixedMessage::FromConstellation(ScriptThreadMessage::SendInputEvent(id, event)) => {
                     self.handle_input_event(id, event)
                 },
@@ -1891,6 +1898,7 @@ impl ScriptThread {
             msg @ ScriptThreadMessage::ExitFullScreen(..) |
             msg @ ScriptThreadMessage::SendInputEvent(..) |
             msg @ ScriptThreadMessage::TickAllAnimations(..) |
+            msg @ ScriptThreadMessage::NoLongerWaitingOnAsychronousImageUpdates(..) |
             msg @ ScriptThreadMessage::ExitScriptThread => {
                 panic!("should have handled {:?} already", msg)
             },