From 73b778e67f2f399846af57a2c93e22da05bb7657 Mon Sep 17 00:00:00 2001 From: sagudev <16504129+sagudev@users.noreply.github.com> Date: Wed, 23 Apr 2025 09:32:47 +0200 Subject: [PATCH] Introduce snapshot concept of canvas (#36119) Each canvas context returns snapshot instead of just raw bytes. This allows as to hold off conversions (BGRA <-> RGBA, (un)premultiply) to when/if they are actually needed. For example when loading snapshot into webgl we can load both RGBA and BGRA so no conversion is really needed. Currently whole thing is designed to be able to be extend on https://github.com/servo/ipc-channel/pull/356, to make less copies. Hence some commented out code. Fixes #35759 There are tests for these changes in WPT --------- Signed-off-by: sagudev <16504129+sagudev@users.noreply.github.com> --- Cargo.lock | 15 + Cargo.toml | 1 + components/canvas/Cargo.toml | 1 + components/canvas/canvas_data.rs | 65 ++-- components/canvas/canvas_paint_thread.rs | 45 +-- components/canvas/raqote_backend.rs | 17 +- components/canvas/webgl_thread.rs | 9 +- components/pixels/lib.rs | 64 ++++ components/script/Cargo.toml | 1 + components/script/canvas_context.rs | 11 +- components/script/canvas_state.rs | 64 ++-- .../script/dom/canvasrenderingcontext2d.rs | 16 +- components/script/dom/globalscope.rs | 19 +- components/script/dom/htmlcanvaselement.rs | 78 +++-- components/script/dom/offscreencanvas.rs | 27 +- .../dom/offscreencanvasrenderingcontext2d.rs | 6 +- .../script/dom/webgl2renderingcontext.rs | 11 +- .../script/dom/webglrenderingcontext.rs | 36 ++- .../script/dom/webgpu/gpucanvascontext.rs | 9 +- components/shared/canvas/Cargo.toml | 1 + components/shared/canvas/canvas.rs | 9 +- components/shared/canvas/webgl.rs | 7 +- components/shared/compositing/Cargo.toml | 1 - components/shared/snapshot/Cargo.toml | 19 ++ components/shared/snapshot/lib.rs | 305 ++++++++++++++++++ components/shared/webgpu/Cargo.toml | 1 + components/shared/webgpu/messages/recv.rs | 3 +- components/webgpu/Cargo.toml | 1 + components/webgpu/swapchain.rs | 27 +- .../toDataURL.jpeg.alpha.html.ini | 3 - .../webgl/tex_image_2d_canvas.html.ini | 2 - .../context/premultiplyalpha-test.html.ini | 17 - .../more/functions/copyTexSubImage2D.html.ini | 3 - .../textures/misc/tex-image-webgl.html.ini | 6 - ...luminance-luminance-unsigned_byte.html.ini | 36 +++ ...pha-luminance_alpha-unsigned_byte.html.ini | 36 +++ ...nvas_colorspace_rgba16float.https.html.ini | 3 - 37 files changed, 724 insertions(+), 251 deletions(-) create mode 100644 components/shared/snapshot/Cargo.toml create mode 100644 components/shared/snapshot/lib.rs delete mode 100644 tests/wpt/meta/html/semantics/embedded-content/the-canvas-element/toDataURL.jpeg.alpha.html.ini delete mode 100644 tests/wpt/mozilla/meta/mozilla/webgl/tex_image_2d_canvas.html.ini delete mode 100644 tests/wpt/webgl/meta/conformance/more/functions/copyTexSubImage2D.html.ini delete mode 100644 tests/wpt/webgl/meta/conformance/textures/misc/tex-image-webgl.html.ini delete mode 100644 tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_colorspace_rgba16float.https.html.ini diff --git a/Cargo.lock b/Cargo.lock index 856e243b854..9df8210ab64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -805,6 +805,7 @@ dependencies = [ "range", "raqote", "servo_arc", + "snapshot", "stylo", "surfman", "unicode-script", @@ -829,6 +830,7 @@ dependencies = [ "serde_bytes", "servo_config", "servo_malloc_size_of", + "snapshot", "stylo", "webrender_api", "webxr-api", @@ -6303,6 +6305,7 @@ dependencies = [ "servo_rand", "servo_url", "smallvec", + "snapshot", "strum", "strum_macros", "stylo", @@ -7081,6 +7084,16 @@ dependencies = [ "serde", ] +[[package]] +name = "snapshot" +version = "0.0.1" +dependencies = [ + "euclid", + "ipc-channel", + "pixels", + "serde", +] + [[package]] name = "socket2" version = "0.5.9" @@ -8489,6 +8502,7 @@ dependencies = [ "serde", "servo_config", "servo_malloc_size_of", + "snapshot", "webgpu_traits", "webrender", "webrender_api", @@ -8505,6 +8519,7 @@ dependencies = [ "ipc-channel", "serde", "servo_malloc_size_of", + "snapshot", "webrender_api", "wgpu-core", "wgpu-types", diff --git a/Cargo.toml b/Cargo.toml index 9edae9c058b..cd2cbb54b63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,6 +129,7 @@ servo-tracing = { path = "components/servo_tracing" } servo_arc = { git = "https://github.com/servo/stylo", branch = "2025-03-15" } smallbitvec = "2.6.0" smallvec = "1.15" +snapshot = { path = "./components/shared/snapshot" } static_assertions = "1.1" string_cache = "0.8" string_cache_codegen = "0.5" diff --git a/components/canvas/Cargo.toml b/components/canvas/Cargo.toml index a9c06784d18..7e7b00efe11 100644 --- a/components/canvas/Cargo.toml +++ b/components/canvas/Cargo.toml @@ -38,6 +38,7 @@ pixels = { path = "../pixels" } range = { path = "../range" } raqote = "0.8.5" servo_arc = { workspace = true } +snapshot = { workspace = true } stylo = { workspace = true } surfman = { workspace = true } unicode-script = { workspace = true } diff --git a/components/canvas/canvas_data.rs b/components/canvas/canvas_data.rs index d6e35b2cbaf..99d6273813e 100644 --- a/components/canvas/canvas_data.rs +++ b/components/canvas/canvas_data.rs @@ -19,6 +19,7 @@ use log::warn; use num_traits::ToPrimitive; use range::Range; use servo_arc::Arc as ServoArc; +use snapshot::Snapshot; use style::color::AbsoluteColor; use style::properties::style_structs::Font as FontStyleStruct; use unicode_script::Script; @@ -521,8 +522,7 @@ pub trait GenericDrawTarget { stroke_options: &StrokeOptions, draw_options: &DrawOptions, ); - fn snapshot_data(&self, f: &dyn Fn(&[u8]) -> Vec) -> Vec; - fn snapshot_data_owned(&self) -> Vec; + fn snapshot_data(&self) -> &[u8]; } pub enum GradientStop { @@ -605,9 +605,8 @@ impl<'a> CanvasData<'a> { offset: 0, flags: ImageDescriptorFlags::empty(), }; - let data = SerializableImageData::Raw(IpcSharedMemory::from_bytes( - &draw_target.snapshot_data_owned(), - )); + let data = + SerializableImageData::Raw(IpcSharedMemory::from_bytes(draw_target.snapshot_data())); compositor_api.update_images(vec![ImageUpdate::AddImage(image_key, descriptor, data)]); CanvasData { backend, @@ -628,7 +627,7 @@ impl<'a> CanvasData<'a> { pub fn draw_image( &mut self, image_data: &[u8], - image_size: Size2D, + image_size: Size2D, dest_rect: Rect, source_rect: Rect, smoothing_enabled: bool, @@ -637,8 +636,8 @@ impl<'a> CanvasData<'a> { // We round up the floating pixel values to draw the pixels let source_rect = source_rect.ceil(); // It discards the extra pixels (if any) that won't be painted - let image_data = if Rect::from_size(image_size).contains_rect(&source_rect) { - pixels::rgba8_get_rect(image_data, image_size.to_u64(), source_rect.to_u64()).into() + let image_data = if Rect::from_size(image_size.to_f64()).contains_rect(&source_rect) { + pixels::rgba8_get_rect(image_data, image_size, source_rect.to_u64()).into() } else { image_data.into() }; @@ -1412,12 +1411,8 @@ impl<'a> CanvasData<'a> { self.update_image_rendering(); } - pub fn send_pixels(&mut self, chan: IpcSender) { - self.drawtarget.snapshot_data(&|bytes| { - let data = IpcSharedMemory::from_bytes(bytes); - chan.send(data).unwrap(); - vec![] - }); + pub fn snapshot(&self) { + self.drawtarget.snapshot_data(); } /// Update image in WebRender @@ -1430,7 +1425,7 @@ impl<'a> CanvasData<'a> { flags: ImageDescriptorFlags::empty(), }; let data = SerializableImageData::Raw(IpcSharedMemory::from_bytes( - &self.drawtarget.snapshot_data_owned(), + self.drawtarget.snapshot_data(), )); self.compositor_api @@ -1529,18 +1524,36 @@ impl<'a> CanvasData<'a> { /// canvas_size: The size of the canvas we're reading from /// read_rect: The area of the canvas we want to read from #[allow(unsafe_code)] - pub fn read_pixels(&self, read_rect: Rect, canvas_size: Size2D) -> Vec { - let canvas_rect = Rect::from_size(canvas_size); - if canvas_rect - .intersection(&read_rect) - .is_none_or(|rect| rect.is_empty()) - { - return vec![]; - } + pub fn read_pixels( + &self, + read_rect: Option>, + canvas_size: Option>, + ) -> Snapshot { + let canvas_size = canvas_size.unwrap_or(self.drawtarget.get_size().cast()); - self.drawtarget.snapshot_data(&|bytes| { - pixels::rgba8_get_rect(bytes, canvas_size, read_rect).into_owned() - }) + let data = if let Some(read_rect) = read_rect { + let canvas_rect = Rect::from_size(canvas_size); + if canvas_rect + .intersection(&read_rect) + .is_none_or(|rect| rect.is_empty()) + { + vec![] + } else { + let bytes = self.drawtarget.snapshot_data(); + pixels::rgba8_get_rect(bytes, canvas_size, read_rect).to_vec() + } + } else { + self.drawtarget.snapshot_data().to_vec() + }; + + Snapshot::from_vec( + canvas_size, + snapshot::PixelFormat::BGRA, + snapshot::AlphaMode::Transparent { + premultiplied: true, + }, + data, + ) } } diff --git a/components/canvas/canvas_paint_thread.rs b/components/canvas/canvas_paint_thread.rs index 8b1b5038334..bb940d7ef81 100644 --- a/components/canvas/canvas_paint_thread.rs +++ b/components/canvas/canvas_paint_thread.rs @@ -76,7 +76,11 @@ impl<'a> CanvasPaintThread<'a> { }, Ok(CanvasMsg::FromScript(message, canvas_id)) => match message { FromScriptMsg::SendPixels(chan) => { - canvas_paint_thread.canvas(canvas_id).send_pixels(chan); + chan.send(canvas_paint_thread + .canvas(canvas_id) + .read_pixels(None, None) + .as_ipc() + ).unwrap(); }, }, Err(e) => { @@ -159,24 +163,21 @@ impl<'a> CanvasPaintThread<'a> { Canvas2dMsg::IsPointInPath(path, x, y, fill_rule, chan) => self .canvas(canvas_id) .is_point_in_path_(&path[..], x, y, fill_rule, chan), - Canvas2dMsg::DrawImage( - ref image_data, - image_size, - dest_rect, - source_rect, - smoothing_enabled, - ) => self.canvas(canvas_id).draw_image( - image_data, - image_size, - dest_rect, - source_rect, - smoothing_enabled, - true, - ), + Canvas2dMsg::DrawImage(snapshot, dest_rect, source_rect, smoothing_enabled) => { + let snapshot = snapshot.to_owned(); + self.canvas(canvas_id).draw_image( + snapshot.data(), + snapshot.size(), + dest_rect, + source_rect, + smoothing_enabled, + !snapshot.alpha_mode().is_premultiplied(), + ) + }, Canvas2dMsg::DrawEmptyImage(image_size, dest_rect, source_rect) => { self.canvas(canvas_id).draw_image( &vec![0; image_size.area() as usize * 4], - image_size, + image_size.to_u64(), dest_rect, source_rect, false, @@ -192,10 +193,10 @@ impl<'a> CanvasPaintThread<'a> { ) => { let image_data = self .canvas(canvas_id) - .read_pixels(source_rect.to_u64(), image_size.to_u64()); + .read_pixels(Some(source_rect.to_u64()), Some(image_size.to_u64())); self.canvas(other_canvas_id).draw_image( - &image_data, - source_rect.size, + image_data.data(), + source_rect.size.to_u64(), dest_rect, source_rect, smoothing, @@ -244,8 +245,10 @@ impl<'a> CanvasPaintThread<'a> { self.canvas(canvas_id).set_global_composition(op) }, Canvas2dMsg::GetImageData(dest_rect, canvas_size, sender) => { - let pixels = self.canvas(canvas_id).read_pixels(dest_rect, canvas_size); - sender.send(&pixels).unwrap(); + let snapshot = self + .canvas(canvas_id) + .read_pixels(Some(dest_rect), Some(canvas_size)); + sender.send(snapshot.as_ipc()).unwrap(); }, Canvas2dMsg::PutImageData(rect, receiver) => { self.canvas(canvas_id) diff --git a/components/canvas/raqote_backend.rs b/components/canvas/raqote_backend.rs index e40367a4ee8..12137e41f41 100644 --- a/components/canvas/raqote_backend.rs +++ b/components/canvas/raqote_backend.rs @@ -630,7 +630,7 @@ impl GenericDrawTarget for raqote::DrawTarget { self.set_transform(matrix); } fn snapshot(&self) -> SourceSurface { - SourceSurface::Raqote(self.snapshot_data_owned()) + SourceSurface::Raqote(self.snapshot_data().to_vec()) } fn stroke( &mut self, @@ -694,20 +694,9 @@ impl GenericDrawTarget for raqote::DrawTarget { ); } #[allow(unsafe_code)] - fn snapshot_data(&self, f: &dyn Fn(&[u8]) -> Vec) -> Vec { + fn snapshot_data(&self) -> &[u8] { let v = self.get_data(); - f( - unsafe { - std::slice::from_raw_parts(v.as_ptr() as *const u8, std::mem::size_of_val(v)) - }, - ) - } - #[allow(unsafe_code)] - fn snapshot_data_owned(&self) -> Vec { - let v = self.get_data(); - unsafe { - std::slice::from_raw_parts(v.as_ptr() as *const u8, std::mem::size_of_val(v)).into() - } + unsafe { std::slice::from_raw_parts(v.as_ptr() as *const u8, std::mem::size_of_val(v)) } } } diff --git a/components/canvas/webgl_thread.rs b/components/canvas/webgl_thread.rs index e3f0c77b4b3..b1ac2b2d3c4 100644 --- a/components/canvas/webgl_thread.rs +++ b/components/canvas/webgl_thread.rs @@ -31,6 +31,7 @@ use glow::{ bytes_per_type, components_per_format, }; use half::f16; +use ipc_channel::ipc::IpcSharedMemory; use log::{debug, error, trace, warn}; use pixels::{self, PixelFormat, unmultiply_inplace}; use surfman::chains::{PreserveBuffer, SwapChains, SwapChainsAPI}; @@ -1212,7 +1213,13 @@ impl WebGLImpl { glow::PixelPackData::Slice(Some(&mut pixels)), ) }; - sender.send(&pixels).unwrap(); + let alpha_mode = match (attributes.alpha, attributes.premultiplied_alpha) { + (true, premultiplied) => snapshot::AlphaMode::Transparent { premultiplied }, + (false, _) => snapshot::AlphaMode::Opaque, + }; + sender + .send((IpcSharedMemory::from_bytes(&pixels), alpha_mode)) + .unwrap(); }, WebGLCommand::ReadPixelsPP(rect, format, pixel_type, offset) => unsafe { gl.read_pixels( diff --git a/components/pixels/lib.rs b/components/pixels/lib.rs index 7cba060ec74..24386ff830a 100644 --- a/components/pixels/lib.rs +++ b/components/pixels/lib.rs @@ -84,6 +84,7 @@ pub fn rgba8_premultiply_inplace(pixels: &mut [u8]) -> bool { is_opaque } +#[inline(always)] pub fn multiply_u8_color(a: u8, b: u8) -> u8 { (a as u32 * b as u32 / 255) as u8 } @@ -254,6 +255,69 @@ pub fn unmultiply_inplace(pixels: &mut [u8]) { } } +#[repr(u8)] +pub enum Multiply { + None = 0, + PreMultiply = 1, + UnMultiply = 2, +} + +pub fn transform_inplace(pixels: &mut [u8], multiply: Multiply, swap_rb: bool, clear_alpha: bool) { + match (multiply, swap_rb, clear_alpha) { + (Multiply::None, true, true) => generic_transform_inplace::<0, true, true>(pixels), + (Multiply::None, true, false) => generic_transform_inplace::<0, true, false>(pixels), + (Multiply::None, false, true) => generic_transform_inplace::<0, false, true>(pixels), + (Multiply::None, false, false) => generic_transform_inplace::<0, false, false>(pixels), + (Multiply::PreMultiply, true, true) => generic_transform_inplace::<1, true, true>(pixels), + (Multiply::PreMultiply, true, false) => generic_transform_inplace::<1, true, false>(pixels), + (Multiply::PreMultiply, false, true) => generic_transform_inplace::<1, false, true>(pixels), + (Multiply::PreMultiply, false, false) => { + generic_transform_inplace::<1, false, false>(pixels) + }, + (Multiply::UnMultiply, true, true) => generic_transform_inplace::<2, true, true>(pixels), + (Multiply::UnMultiply, true, false) => generic_transform_inplace::<2, true, false>(pixels), + (Multiply::UnMultiply, false, true) => generic_transform_inplace::<2, false, true>(pixels), + (Multiply::UnMultiply, false, false) => { + generic_transform_inplace::<2, false, false>(pixels) + }, + } +} + +pub fn generic_transform_inplace< + const MULTIPLY: u8, // 1 premultiply, 2 unmultiply + const SWAP_RB: bool, + const CLEAR_ALPHA: bool, +>( + pixels: &mut [u8], +) { + for rgba in pixels.chunks_mut(4) { + match MULTIPLY { + 1 => { + let a = rgba[3]; + multiply_u8_color(rgba[0], a); + multiply_u8_color(rgba[1], a); + multiply_u8_color(rgba[2], a); + }, + 2 => { + let a = rgba[3] as u32; + + if a > 0 { + rgba[0] = (rgba[0] as u32 * 255 / a) as u8; + rgba[1] = (rgba[1] as u32 * 255 / a) as u8; + rgba[2] = (rgba[2] as u32 * 255 / a) as u8; + } + }, + _ => {}, + } + if SWAP_RB { + rgba.swap(0, 2); + } + if CLEAR_ALPHA { + rgba[3] = u8::MAX; + } + } +} + fn is_gif(buffer: &[u8]) -> bool { buffer.starts_with(b"GIF87a") || buffer.starts_with(b"GIF89a") } diff --git a/components/script/Cargo.toml b/components/script/Cargo.toml index 37e80d0e748..1aa821cdbd3 100644 --- a/components/script/Cargo.toml +++ b/components/script/Cargo.toml @@ -114,6 +114,7 @@ servo_config = { path = "../config" } servo_geometry = { path = "../geometry" } servo_rand = { path = "../rand" } servo_url = { path = "../url" } +snapshot = { workspace = true } smallvec = { workspace = true, features = ["union"] } strum = { workspace = true } strum_macros = { workspace = true } diff --git a/components/script/canvas_context.rs b/components/script/canvas_context.rs index 8bf188a5aa9..d49d31997e1 100644 --- a/components/script/canvas_context.rs +++ b/components/script/canvas_context.rs @@ -5,8 +5,8 @@ //! Common interfaces for Canvas Contexts use euclid::default::Size2D; -use ipc_channel::ipc::IpcSharedMemory; use script_layout_interface::{HTMLCanvasData, HTMLCanvasDataSource}; +use snapshot::Snapshot; use crate::dom::bindings::codegen::UnionTypes::HTMLCanvasElementOrOffscreenCanvas; use crate::dom::bindings::inheritance::Castable; @@ -30,11 +30,10 @@ pub(crate) trait CanvasContext { fn resize(&self); - fn get_image_data_as_shared_memory(&self) -> Option; - - fn get_image_data(&self) -> Option> { - self.get_image_data_as_shared_memory().map(|sm| sm.to_vec()) - } + /// Returns none if area of canvas is zero. + /// + /// In case of other errors it returns cleared snapshot + fn get_image_data(&self) -> Option; fn origin_is_clean(&self) -> bool { true diff --git a/components/script/canvas_state.rs b/components/script/canvas_state.rs index aea3012b365..408c94c124a 100644 --- a/components/script/canvas_state.rs +++ b/components/script/canvas_state.rs @@ -17,7 +17,7 @@ use cssparser::color::clamp_unit_f32; use cssparser::{Parser, ParserInput}; use euclid::default::{Point2D, Rect, Size2D, Transform2D}; use euclid::vec2; -use ipc_channel::ipc::{self, IpcSender, IpcSharedMemory}; +use ipc_channel::ipc::{self, IpcSender}; use net_traits::image_cache::{ImageCache, ImageResponse}; use net_traits::request::CorsSettings; use pixels::PixelFormat; @@ -298,7 +298,7 @@ impl CanvasState { &self, url: ServoUrl, cors_setting: Option, - ) -> Option<(IpcSharedMemory, Size2D)> { + ) -> Option { let img = match self.request_image_from_cache(url, cors_setting) { ImageResponse::Loaded(img, _) => img, ImageResponse::PlaceholderLoaded(_, _) | @@ -308,13 +308,22 @@ impl CanvasState { }, }; - let image_size = Size2D::new(img.width, img.height); - let image_data = match img.format { - PixelFormat::BGRA8 => img.bytes(), + let size = Size2D::new(img.width, img.height); + let format = match img.format { + PixelFormat::BGRA8 => snapshot::PixelFormat::BGRA, + PixelFormat::RGBA8 => snapshot::PixelFormat::RGBA, pixel_format => unimplemented!("unsupported pixel format ({:?})", pixel_format), }; + let alpha_mode = snapshot::AlphaMode::Transparent { + premultiplied: false, + }; - Some((image_data, image_size)) + Some(snapshot::Snapshot::from_shared_memory( + size.cast(), + format, + alpha_mode, + img.bytes(), + )) } fn request_image_from_cache( @@ -341,13 +350,16 @@ impl CanvasState { assert!(Rect::from_size(canvas_size).contains_rect(&rect)); - let (sender, receiver) = ipc::bytes_channel().unwrap(); + let (sender, receiver) = ipc::channel().unwrap(); self.send_canvas_2d_msg(Canvas2dMsg::GetImageData(rect, canvas_size, sender)); - let mut pixels = receiver.recv().unwrap().to_vec(); - - pixels::unmultiply_inplace::(&mut pixels); - - pixels + let mut snapshot = receiver.recv().unwrap().to_owned(); + snapshot.transform( + snapshot::AlphaMode::Transparent { + premultiplied: false, + }, + snapshot::PixelFormat::RGBA, + ); + snapshot.to_vec() } /// @@ -594,10 +606,10 @@ impl CanvasState { dh: Option, ) -> ErrorResult { debug!("Fetching image {}.", url); - let (image_data, image_size) = self + let snapshot = self .fetch_image_data(url, cors_setting) .ok_or(Error::InvalidState)?; - let image_size = image_size.to_f64(); + let image_size = snapshot.size().to_f64(); let dw = dw.unwrap_or(image_size.width); let dh = dh.unwrap_or(image_size.height); @@ -614,8 +626,7 @@ impl CanvasState { let smoothing_enabled = self.state.borrow().image_smoothing_enabled; self.send_canvas_2d_msg(Canvas2dMsg::DrawImage( - image_data, - image_size, + snapshot.as_ipc(), dest_rect, source_rect, smoothing_enabled, @@ -929,7 +940,7 @@ impl CanvasState { mut repetition: DOMString, can_gc: CanGc, ) -> Fallible>> { - let (image_data, image_size) = match image { + let snapshot = match image { CanvasImageSource::HTMLImageElement(ref image) => { // https://html.spec.whatwg.org/multipage/#check-the-usability-of-the-image-argument if !image.is_usable()? { @@ -941,27 +952,17 @@ impl CanvasState { .and_then(|url| { self.fetch_image_data(url, cors_setting_for_element(image.upcast())) }) - .map(|data| (data.0.to_vec(), data.1)) .ok_or(Error::InvalidState)? }, CanvasImageSource::HTMLCanvasElement(ref canvas) => { - let (data, size) = canvas.fetch_all_data().ok_or(Error::InvalidState)?; - let data = data - .map(|data| data.to_vec()) - .unwrap_or_else(|| vec![0; size.area() as usize * 4]); - (data, size) + canvas.get_image_data().ok_or(Error::InvalidState)? }, CanvasImageSource::OffscreenCanvas(ref canvas) => { - let (data, size) = canvas.fetch_all_data().ok_or(Error::InvalidState)?; - let data = data - .map(|data| data.to_vec()) - .unwrap_or_else(|| vec![0; size.area() as usize * 4]); - (data, size) + canvas.get_image_data().ok_or(Error::InvalidState)? }, CanvasImageSource::CSSStyleValue(ref value) => value .get_url(self.base_url.clone()) .and_then(|url| self.fetch_image_data(url, None)) - .map(|data| (data.0.to_vec(), data.1)) .ok_or(Error::InvalidState)?, }; @@ -970,10 +971,11 @@ impl CanvasState { } if let Ok(rep) = RepetitionStyle::from_str(&repetition) { + let size = snapshot.size(); Ok(Some(CanvasPattern::new( global, - image_data, - image_size, + snapshot.to_vec(), + size.cast(), rep, self.is_origin_clean(image), can_gc, diff --git a/components/script/dom/canvasrenderingcontext2d.rs b/components/script/dom/canvasrenderingcontext2d.rs index f6bf432de69..73052e6906e 100644 --- a/components/script/dom/canvasrenderingcontext2d.rs +++ b/components/script/dom/canvasrenderingcontext2d.rs @@ -5,11 +5,11 @@ use canvas_traits::canvas::{Canvas2dMsg, CanvasId, CanvasMsg, FromScriptMsg}; use dom_struct::dom_struct; use euclid::default::{Point2D, Rect, Size2D}; -use ipc_channel::ipc::IpcSharedMemory; use profile_traits::ipc; use script_bindings::inheritance::Castable; use script_layout_interface::HTMLCanvasDataSource; use servo_url::ServoUrl; +use snapshot::Snapshot; use crate::canvas_context::{CanvasContext, CanvasHelpers, LayoutCanvasRenderingContextHelpers}; use crate::canvas_state::CanvasState; @@ -142,16 +142,18 @@ impl CanvasContext for CanvasRenderingContext2D { self.set_bitmap_dimensions(self.size().cast()) } - fn get_image_data_as_shared_memory(&self) -> Option { + fn get_image_data(&self) -> Option { + let size = self.size(); + + if size.is_empty() { + return None; + } + let (sender, receiver) = ipc::channel(self.global().time_profiler_chan().clone()).unwrap(); let msg = CanvasMsg::FromScript(FromScriptMsg::SendPixels(sender), self.get_canvas_id()); self.canvas_state.get_ipc_renderer().send(msg).unwrap(); - Some(receiver.recv().unwrap()) - } - - fn get_image_data(&self) -> Option> { - Some(self.get_rect(Rect::from_size(self.size().cast()))) + Some(receiver.recv().unwrap().to_owned()) } fn origin_is_clean(&self) -> bool { diff --git a/components/script/dom/globalscope.rs b/components/script/dom/globalscope.rs index 2658911c795..77d1ee37c03 100644 --- a/components/script/dom/globalscope.rs +++ b/components/script/dom/globalscope.rs @@ -2880,15 +2880,11 @@ impl GlobalScope { return p; } - if let Some((data, size)) = canvas.fetch_all_data() { - let data = data - .map(|data| data.to_vec()) - .unwrap_or_else(|| vec![0; size.area() as usize * 4]); - + if let Some(snapshot) = canvas.get_image_data() { + let size = snapshot.size().cast(); let image_bitmap = ImageBitmap::new(self, size.width, size.height, can_gc).unwrap(); - - image_bitmap.set_bitmap_data(data); + image_bitmap.set_bitmap_data(snapshot.to_vec()); image_bitmap.set_origin_clean(canvas.origin_is_clean()); p.resolve_native(&(image_bitmap), can_gc); } @@ -2901,14 +2897,11 @@ impl GlobalScope { return p; } - if let Some((data, size)) = canvas.fetch_all_data() { - let data = data - .map(|data| data.to_vec()) - .unwrap_or_else(|| vec![0; size.area() as usize * 4]); - + if let Some(snapshot) = canvas.get_image_data() { + let size = snapshot.size().cast(); let image_bitmap = ImageBitmap::new(self, size.width, size.height, can_gc).unwrap(); - image_bitmap.set_bitmap_data(data); + image_bitmap.set_bitmap_data(snapshot.to_vec()); image_bitmap.set_origin_clean(canvas.origin_is_clean()); p.resolve_native(&(image_bitmap), can_gc); } diff --git a/components/script/dom/htmlcanvaselement.rs b/components/script/dom/htmlcanvaselement.rs index 9e20539ceca..303c781c8b3 100644 --- a/components/script/dom/htmlcanvaselement.rs +++ b/components/script/dom/htmlcanvaselement.rs @@ -17,7 +17,6 @@ use image::codecs::jpeg::JpegEncoder; use image::codecs::png::PngEncoder; use image::codecs::webp::WebPEncoder; use image::{ColorType, ImageEncoder}; -use ipc_channel::ipc::IpcSharedMemory; #[cfg(feature = "webgpu")] use ipc_channel::ipc::{self as ipcchan}; use js::error::throw_type_error; @@ -25,6 +24,7 @@ use js::rust::{HandleObject, HandleValue}; use script_layout_interface::{HTMLCanvasData, HTMLCanvasDataSource}; use servo_media::streams::MediaStreamType; use servo_media::streams::registry::MediaStreamId; +use snapshot::Snapshot; use style::attr::AttrValue; use crate::canvas_context::CanvasContext as _; @@ -69,6 +69,7 @@ use crate::script_runtime::{CanGc, JSContext}; const DEFAULT_WIDTH: u32 = 300; const DEFAULT_HEIGHT: u32 = 150; +#[derive(PartialEq)] enum EncodedImageType { Png, Jpeg, @@ -375,42 +376,21 @@ impl HTMLCanvasElement { self.Height() != 0 && self.Width() != 0 } - pub(crate) fn fetch_all_data(&self) -> Option<(Option, Size2D)> { - let size = self.get_size(); - - if size.width == 0 || size.height == 0 { - return None; - } - - let data = match self.context.borrow().as_ref() { - Some(CanvasContext::Context2d(context)) => context.get_image_data_as_shared_memory(), - Some(CanvasContext::WebGL(_context)) => { - // TODO: add a method in WebGLRenderingContext to get the pixels. - return None; - }, - Some(CanvasContext::WebGL2(_context)) => { - // TODO: add a method in WebGL2RenderingContext to get the pixels. - return None; - }, + pub(crate) fn get_image_data(&self) -> Option { + match self.context.borrow().as_ref() { + Some(CanvasContext::Context2d(context)) => context.get_image_data(), + Some(CanvasContext::WebGL(context)) => context.get_image_data(), + Some(CanvasContext::WebGL2(context)) => context.get_image_data(), #[cfg(feature = "webgpu")] - Some(CanvasContext::WebGPU(context)) => context.get_image_data_as_shared_memory(), - Some(CanvasContext::Placeholder(context)) => return context.fetch_all_data(), - None => None, - }; - - Some((data, size)) - } - - fn get_content(&self) -> Option> { - match *self.context.borrow() { - Some(CanvasContext::Context2d(ref context)) => context.get_image_data(), - Some(CanvasContext::WebGL(ref context)) => context.get_image_data(), - Some(CanvasContext::WebGL2(ref context)) => context.get_image_data(), - #[cfg(feature = "webgpu")] - Some(CanvasContext::WebGPU(ref context)) => context.get_image_data(), - Some(CanvasContext::Placeholder(_)) | None => { - // Each pixel is fully-transparent black. - Some(vec![0; (self.Width() * self.Height() * 4) as usize]) + Some(CanvasContext::WebGPU(context)) => context.get_image_data(), + Some(CanvasContext::Placeholder(context)) => context.get_image_data(), + None => { + let size = self.get_size(); + if size.width == 0 || size.height == 0 { + None + } else { + Some(Snapshot::cleared(size.cast())) + } }, } } @@ -560,11 +540,23 @@ impl HTMLCanvasElementMethods for HTMLCanvasElement { } // Step 3. - let Some(file) = self.get_content() else { + let Some(mut snapshot) = self.get_image_data() else { return Ok(USVString("data:,".into())); }; let image_type = EncodedImageType::from(mime_type); + snapshot.transform( + if image_type == EncodedImageType::Jpeg { + snapshot::AlphaMode::AsOpaque { + premultiplied: true, + } + } else { + snapshot::AlphaMode::Transparent { + premultiplied: false, + } + }, + snapshot::PixelFormat::RGBA, + ); let mut url = format!("data:{};base64,", image_type.as_mime_type()); let mut encoder = base64::write::EncoderStringWriter::from_consumer( @@ -575,7 +567,7 @@ impl HTMLCanvasElementMethods for HTMLCanvasElement { self.encode_for_mime_type( &image_type, Self::maybe_quality(quality), - &file, + snapshot.data(), &mut encoder, ); encoder.into_inner(); @@ -604,7 +596,7 @@ impl HTMLCanvasElementMethods for HTMLCanvasElement { let result = if self.Width() == 0 || self.Height() == 0 { None } else { - self.get_content() + self.get_image_data() }; let this = Trusted::new(self); @@ -625,13 +617,17 @@ impl HTMLCanvasElementMethods for HTMLCanvasElement { return error!("Expected blob callback, but found none!"); }; - if let Some(bytes) = result { + if let Some(mut snapshot) = result { + snapshot.transform( + snapshot::AlphaMode::Transparent{ premultiplied: false }, + snapshot::PixelFormat::RGBA + ); // Step 4.1 // If result is non-null, then set result to a serialization of result as a file with // type and quality if given. let mut encoded: Vec = vec![]; - this.encode_for_mime_type(&image_type, quality, &bytes, &mut encoded); + this.encode_for_mime_type(&image_type, quality, snapshot.data(), &mut encoded); let blob_impl = BlobImpl::new_from_bytes(encoded, image_type.as_mime_type()); // Step 4.2.1 & 4.2.2 // Set result to a new Blob object, created in the relevant realm of this canvas element diff --git a/components/script/dom/offscreencanvas.rs b/components/script/dom/offscreencanvas.rs index 0587fbad12b..aabe5955e12 100644 --- a/components/script/dom/offscreencanvas.rs +++ b/components/script/dom/offscreencanvas.rs @@ -6,8 +6,8 @@ use std::cell::Cell; use dom_struct::dom_struct; use euclid::default::Size2D; -use ipc_channel::ipc::IpcSharedMemory; use js::rust::{HandleObject, HandleValue}; +use snapshot::Snapshot; use crate::dom::bindings::cell::{DomRefCell, Ref, ref_filter_map}; use crate::dom::bindings::codegen::Bindings::OffscreenCanvasBinding::{ @@ -88,21 +88,18 @@ impl OffscreenCanvas { ref_filter_map(self.context.borrow(), |ctx| ctx.as_ref()) } - pub(crate) fn fetch_all_data(&self) -> Option<(Option, Size2D)> { - let size = self.get_size(); - - if size.width == 0 || size.height == 0 { - return None; - } - - let data = match self.context.borrow().as_ref() { - Some(OffscreenCanvasContext::OffscreenContext2d(context)) => { - context.get_image_data_as_shared_memory() + pub(crate) fn get_image_data(&self) -> Option { + match self.context.borrow().as_ref() { + Some(OffscreenCanvasContext::OffscreenContext2d(context)) => context.get_image_data(), + None => { + let size = self.get_size(); + if size.width == 0 || size.height == 0 { + None + } else { + Some(Snapshot::cleared(size)) + } }, - None => None, - }; - - Some((data, size.to_u32())) + } } pub(crate) fn get_or_init_2d_context( diff --git a/components/script/dom/offscreencanvasrenderingcontext2d.rs b/components/script/dom/offscreencanvasrenderingcontext2d.rs index 69a1d41af2e..2f9b52640e6 100644 --- a/components/script/dom/offscreencanvasrenderingcontext2d.rs +++ b/components/script/dom/offscreencanvasrenderingcontext2d.rs @@ -8,7 +8,7 @@ use crate::dom::bindings::codegen::UnionTypes::HTMLCanvasElementOrOffscreenCanva use canvas_traits::canvas::Canvas2dMsg; use dom_struct::dom_struct; use euclid::default::Size2D; -use ipc_channel::ipc::IpcSharedMemory; +use snapshot::Snapshot; use crate::dom::bindings::codegen::Bindings::CanvasRenderingContext2DBinding::{ CanvasDirection, CanvasFillRule, CanvasImageSource, CanvasLineCap, CanvasLineJoin, @@ -76,8 +76,8 @@ impl OffscreenCanvasRenderingContext2D { self.context.origin_is_clean() } - pub(crate) fn get_image_data_as_shared_memory(&self) -> Option { - self.context.get_image_data_as_shared_memory() + pub(crate) fn get_image_data(&self) -> Option { + self.context.get_image_data() } } diff --git a/components/script/dom/webgl2renderingcontext.rs b/components/script/dom/webgl2renderingcontext.rs index bb6ffa11849..416454d8719 100644 --- a/components/script/dom/webgl2renderingcontext.rs +++ b/components/script/dom/webgl2renderingcontext.rs @@ -24,6 +24,7 @@ use js::typedarray::{ArrayBufferView, CreateWith, Float32, Int32Array, Uint32, U use script_bindings::interfaces::WebGL2RenderingContextHelpers; use script_layout_interface::HTMLCanvasDataSource; use servo_config::pref; +use snapshot::Snapshot; use url::Host; use crate::canvas_context::CanvasContext; @@ -549,11 +550,11 @@ impl WebGL2RenderingContext { return ); - let (sender, receiver) = ipc::bytes_channel().unwrap(); + let (sender, receiver) = ipc::channel().unwrap(); self.base.send_command(WebGLCommand::ReadPixels( src_rect, format, pixel_type, sender, )); - let src = receiver.recv().unwrap(); + let (src, _) = receiver.recv().unwrap(); for i in 0..src_rect.size.height as usize { let src_start = i * src_row_bytes as usize; @@ -916,11 +917,7 @@ impl CanvasContext for WebGL2RenderingContext { self.base.resize(); } - fn get_image_data_as_shared_memory(&self) -> Option { - self.base.get_image_data_as_shared_memory() - } - - fn get_image_data(&self) -> Option> { + fn get_image_data(&self) -> Option { self.base.get_image_data() } diff --git a/components/script/dom/webglrenderingcontext.rs b/components/script/dom/webglrenderingcontext.rs index 1e51ac4baf9..9996a3cf504 100644 --- a/components/script/dom/webglrenderingcontext.rs +++ b/components/script/dom/webglrenderingcontext.rs @@ -34,6 +34,7 @@ use pixels::{self, PixelFormat}; use script_layout_interface::HTMLCanvasDataSource; use serde::{Deserialize, Serialize}; use servo_config::pref; +use snapshot::Snapshot; use webrender_api::ImageKey; use crate::canvas_context::CanvasContext; @@ -628,11 +629,15 @@ impl WebGLRenderingContext { if !canvas.origin_is_clean() { return Err(Error::Security); } - if let Some((data, size)) = canvas.fetch_all_data() { - let data = data.unwrap_or_else(|| { - IpcSharedMemory::from_bytes(&vec![0; size.area() as usize * 4]) - }); - TexPixels::new(data, size, PixelFormat::BGRA8, true) + if let Some(snapshot) = canvas.get_image_data() { + let snapshot = snapshot.as_ipc(); + let size = snapshot.size().cast(); + let format = match snapshot.format() { + snapshot::PixelFormat::RGBA => PixelFormat::RGBA8, + snapshot::PixelFormat::BGRA => PixelFormat::BGRA8, + }; + let premultiply = snapshot.alpha_mode().is_premultiplied(); + TexPixels::new(snapshot.to_ipc_shared_memory(), size, format, premultiply) } else { return Ok(None); } @@ -1922,18 +1927,13 @@ impl CanvasContext for WebGLRenderingContext { } } - fn get_image_data_as_shared_memory(&self) -> Option { - // TODO: add a method in WebGLRenderingContext to get the pixels. - None - } - // Used by HTMLCanvasElement.toDataURL // // This emits errors quite liberally, but the spec says that this operation // can fail and that it is UB what happens in that case. // // https://www.khronos.org/registry/webgl/specs/latest/1.0/#2.2 - fn get_image_data(&self) -> Option> { + fn get_image_data(&self) -> Option { handle_potential_webgl_error!(self, self.validate_framebuffer(), return None); let mut size = self.size().cast(); @@ -1945,14 +1945,20 @@ impl CanvasContext for WebGLRenderingContext { size.width = cmp::min(size.width, fb_width as u32); size.height = cmp::min(size.height, fb_height as u32); - let (sender, receiver) = ipc::bytes_channel().unwrap(); + let (sender, receiver) = ipc::channel().unwrap(); self.send_command(WebGLCommand::ReadPixels( Rect::from_size(size), constants::RGBA, constants::UNSIGNED_BYTE, sender, )); - Some(receiver.recv().unwrap()) + let (data, alpha_mode) = receiver.recv().unwrap(); + Some(Snapshot::from_vec( + size.cast(), + snapshot::PixelFormat::RGBA, + alpha_mode, + data.to_vec(), + )) } fn mark_as_dirty(&self) { @@ -3826,11 +3832,11 @@ impl WebGLRenderingContextMethods for WebGLRenderingContex dest_offset += -y * row_len; } - let (sender, receiver) = ipc::bytes_channel().unwrap(); + let (sender, receiver) = ipc::channel().unwrap(); self.send_command(WebGLCommand::ReadPixels( src_rect, format, pixel_type, sender, )); - let src = receiver.recv().unwrap(); + let (src, _) = receiver.recv().unwrap(); let src_row_len = src_rect.size.width as usize * bytes_per_pixel as usize; for i in 0..src_rect.size.height { diff --git a/components/script/dom/webgpu/gpucanvascontext.rs b/components/script/dom/webgpu/gpucanvascontext.rs index 595b54c58d7..c81f96f651f 100644 --- a/components/script/dom/webgpu/gpucanvascontext.rs +++ b/components/script/dom/webgpu/gpucanvascontext.rs @@ -7,8 +7,9 @@ use std::cell::RefCell; use arrayvec::ArrayVec; use dom_struct::dom_struct; -use ipc_channel::ipc::{self, IpcSharedMemory}; +use ipc_channel::ipc::{self}; use script_layout_interface::HTMLCanvasDataSource; +use snapshot::Snapshot; use webgpu_traits::{ ContextConfiguration, PRESENTATION_BUFFER_COUNT, WebGPU, WebGPUContextId, WebGPURequest, WebGPUTexture, @@ -277,10 +278,10 @@ impl CanvasContext for GPUCanvasContext { } /// - fn get_image_data_as_shared_memory(&self) -> Option { + fn get_image_data(&self) -> Option { // 1. Return a copy of the image contents of context. Some(if self.drawing_buffer.borrow().cleared { - IpcSharedMemory::from_byte(0, self.size().area() as usize * 4) + Snapshot::cleared(self.size()) } else { let (sender, receiver) = ipc::channel().unwrap(); self.channel @@ -290,7 +291,7 @@ impl CanvasContext for GPUCanvasContext { sender, }) .unwrap(); - receiver.recv().unwrap() + receiver.recv().unwrap().to_owned() }) } diff --git a/components/shared/canvas/Cargo.toml b/components/shared/canvas/Cargo.toml index d6e96711e1d..c77399ef847 100644 --- a/components/shared/canvas/Cargo.toml +++ b/components/shared/canvas/Cargo.toml @@ -26,6 +26,7 @@ pixels = { path = "../../pixels" } serde = { workspace = true } serde_bytes = { workspace = true } servo_config = { path = "../../config" } +snapshot = { workspace = true } stylo = { workspace = true } webrender_api = { workspace = true } webxr-api = { workspace = true, features = ["ipc"] } diff --git a/components/shared/canvas/canvas.rs b/components/shared/canvas/canvas.rs index 90ba569b5eb..850f5f9bd9a 100644 --- a/components/shared/canvas/canvas.rs +++ b/components/shared/canvas/canvas.rs @@ -6,10 +6,11 @@ use std::default::Default; use std::str::FromStr; use euclid::default::{Point2D, Rect, Size2D, Transform2D}; -use ipc_channel::ipc::{IpcBytesReceiver, IpcBytesSender, IpcSender, IpcSharedMemory}; +use ipc_channel::ipc::{IpcBytesReceiver, IpcSender}; use malloc_size_of_derive::MallocSizeOf; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; +use snapshot::IpcSnapshot; use style::color::AbsoluteColor; use style::properties::style_structs::Font as FontStyleStruct; @@ -87,7 +88,7 @@ pub enum CanvasMsg { pub enum Canvas2dMsg { Arc(Point2D, f32, f32, f32, bool), ArcTo(Point2D, Point2D, f32), - DrawImage(IpcSharedMemory, Size2D, Rect, Rect, bool), + DrawImage(IpcSnapshot, Rect, Rect, bool), DrawEmptyImage(Size2D, Rect, Rect), DrawImageInOther(CanvasId, Size2D, Rect, Rect, bool), BeginPath, @@ -101,7 +102,7 @@ pub enum Canvas2dMsg { FillPath(FillOrStrokeStyle, Vec), FillText(String, f64, f64, Option, FillOrStrokeStyle, bool), FillRect(Rect, FillOrStrokeStyle), - GetImageData(Rect, Size2D, IpcBytesSender), + GetImageData(Rect, Size2D, IpcSender), GetTransform(IpcSender>), IsPointInCurrentPath(f64, f64, FillRule, IpcSender), IsPointInPath(Vec, f64, f64, FillRule, IpcSender), @@ -137,7 +138,7 @@ pub enum Canvas2dMsg { #[derive(Clone, Debug, Deserialize, Serialize)] pub enum FromScriptMsg { - SendPixels(IpcSender), + SendPixels(IpcSender), } #[derive(Clone, Debug, Deserialize, MallocSizeOf, Serialize)] diff --git a/components/shared/canvas/webgl.rs b/components/shared/canvas/webgl.rs index eaa934fc145..7ffae5d5fc5 100644 --- a/components/shared/canvas/webgl.rs +++ b/components/shared/canvas/webgl.rs @@ -298,7 +298,12 @@ pub enum WebGLCommand { PolygonOffset(f32, f32), RenderbufferStorage(u32, u32, i32, i32), RenderbufferStorageMultisample(u32, i32, u32, i32, i32), - ReadPixels(Rect, u32, u32, IpcBytesSender), + ReadPixels( + Rect, + u32, + u32, + IpcSender<(IpcSharedMemory, snapshot::AlphaMode)>, + ), ReadPixelsPP(Rect, u32, u32, usize), SampleCoverage(f32, bool), Scissor(i32, i32, u32, u32), diff --git a/components/shared/compositing/Cargo.toml b/components/shared/compositing/Cargo.toml index e3ff81615e6..2e39ef5397b 100644 --- a/components/shared/compositing/Cargo.toml +++ b/components/shared/compositing/Cargo.toml @@ -38,4 +38,3 @@ stylo = { workspace = true } stylo_traits = { workspace = true } surfman = { workspace = true, features = ["sm-x11"] } webrender_api = { workspace = true } - diff --git a/components/shared/snapshot/Cargo.toml b/components/shared/snapshot/Cargo.toml new file mode 100644 index 00000000000..9d5bd555623 --- /dev/null +++ b/components/shared/snapshot/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "snapshot" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lib] +name = "snapshot" +path = "lib.rs" + + +[dependencies] +euclid = { workspace = true } +ipc-channel = { workspace = true } +serde = { workspace = true } +pixels = { path = "../../pixels" } diff --git a/components/shared/snapshot/lib.rs b/components/shared/snapshot/lib.rs new file mode 100644 index 00000000000..5a8c1a5fbc0 --- /dev/null +++ b/components/shared/snapshot/lib.rs @@ -0,0 +1,305 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use std::ops::{Deref, DerefMut}; + +use euclid::default::Size2D; +use ipc_channel::ipc::IpcSharedMemory; +use pixels::Multiply; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub enum PixelFormat { + #[default] + RGBA, + BGRA, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub enum AlphaMode { + /// Internal data is opaque (alpha is cleared to 1) + Opaque, + /// Internal data should be threated as opaque (does not mean it actually is) + AsOpaque { premultiplied: bool }, + /// Data is not opaque + Transparent { premultiplied: bool }, +} + +impl Default for AlphaMode { + fn default() -> Self { + Self::Transparent { + premultiplied: true, + } + } +} + +impl AlphaMode { + pub const fn is_premultiplied(&self) -> bool { + match self { + AlphaMode::Opaque => true, + AlphaMode::AsOpaque { premultiplied } => *premultiplied, + AlphaMode::Transparent { premultiplied } => *premultiplied, + } + } + + pub const fn is_opaque(&self) -> bool { + matches!(self, AlphaMode::Opaque | AlphaMode::AsOpaque { .. }) + } +} + +#[derive(Debug)] +pub enum Data { + // TODO: https://github.com/servo/servo/issues/36594 + //IPC(IpcSharedMemory), + Owned(Vec), +} + +impl Deref for Data { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + match &self { + //Data::IPC(ipc_shared_memory) => ipc_shared_memory, + Data::Owned(items) => items, + } + } +} + +impl DerefMut for Data { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + //Data::IPC(ipc_shared_memory) => unsafe { ipc_shared_memory.deref_mut() }, + Data::Owned(items) => items, + } + } +} + +pub type IpcSnapshot = Snapshot; + +/// Represents image bitmap with metadata, usually as snapshot of canvas +/// +/// This allows us to hold off conversions (BGRA <-> RGBA, (un)premultiply) +/// to when/if they are actually needed (WebGL/WebGPU can load both BGRA and RGBA). +/// +/// Inspired by snapshot for concept in WebGPU spec: +/// +#[derive(Debug, Deserialize, Serialize)] +pub struct Snapshot { + size: Size2D, + /// internal data (can be any format it will be converted on use if needed) + data: T, + /// RGBA/BGRA (reflect internal data) + format: PixelFormat, + /// How to treat alpha channel + alpha_mode: AlphaMode, +} + +impl Snapshot { + pub const fn size(&self) -> Size2D { + self.size + } + + pub const fn format(&self) -> PixelFormat { + self.format + } + + pub const fn alpha_mode(&self) -> AlphaMode { + self.alpha_mode + } + + pub const fn is_premultiplied(&self) -> bool { + self.alpha_mode().is_premultiplied() + } + + pub const fn is_opaque(&self) -> bool { + self.alpha_mode().is_opaque() + } +} + +impl Snapshot { + pub fn empty() -> Self { + Self { + size: Size2D::zero(), + data: Data::Owned(vec![]), + format: PixelFormat::RGBA, + alpha_mode: AlphaMode::Transparent { + premultiplied: true, + }, + } + } + + /// Returns snapshot with provided size that is black transparent alpha + pub fn cleared(size: Size2D) -> Self { + Self { + size, + data: Data::Owned(vec![0; size.area() as usize * 4]), + format: PixelFormat::RGBA, + alpha_mode: AlphaMode::Transparent { + premultiplied: true, + }, + } + } + + pub fn from_vec( + size: Size2D, + format: PixelFormat, + alpha_mode: AlphaMode, + data: Vec, + ) -> Self { + Self { + size, + data: Data::Owned(data), + format, + alpha_mode, + } + } + + pub fn from_shared_memory( + size: Size2D, + format: PixelFormat, + alpha_mode: AlphaMode, + ism: IpcSharedMemory, + ) -> Self { + Self { + size, + data: Data::Owned(ism.to_vec()), + format, + alpha_mode, + } + } + + // TODO: https://github.com/servo/servo/issues/36594 + /* + /// # Safety + /// + /// This is safe if data is owned by this process only + /// (ownership is transferred on send) + pub unsafe fn from_shared_memory( + size: Size2D, + format: PixelFormat, + alpha_mode: AlphaMode, + ism: IpcSharedMemory, + ) -> Self { + Self { + size, + data: Data::IPC(ism), + format, + alpha_mode, + } + } + */ + + pub fn data(&self) -> &[u8] { + &self.data + } + + /// Convert inner data of snapshot to target format and alpha mode. + /// If data is already in target format and alpha mode no work will be done. + pub fn transform(&mut self, target_alpha_mode: AlphaMode, target_format: PixelFormat) { + let swap_rb = target_format != self.format; + let multiply = match (self.alpha_mode, target_alpha_mode) { + (AlphaMode::Opaque, _) => Multiply::None, + (alpha_mode, AlphaMode::Opaque) => { + if alpha_mode.is_premultiplied() { + Multiply::UnMultiply + } else { + Multiply::None + } + }, + ( + AlphaMode::Transparent { premultiplied } | AlphaMode::AsOpaque { premultiplied }, + AlphaMode::Transparent { + premultiplied: target_premultiplied, + } | + AlphaMode::AsOpaque { + premultiplied: target_premultiplied, + }, + ) => { + if premultiplied == target_premultiplied { + Multiply::None + } else if target_premultiplied { + Multiply::PreMultiply + } else { + Multiply::UnMultiply + } + }, + }; + let clear_alpha = !matches!(self.alpha_mode, AlphaMode::Opaque) && + matches!(target_alpha_mode, AlphaMode::Opaque); + pixels::transform_inplace(self.data.deref_mut(), multiply, swap_rb, clear_alpha); + self.alpha_mode = target_alpha_mode; + self.format = target_format; + } + + pub fn as_ipc(self) -> Snapshot { + let Snapshot { + size, + data, + format, + alpha_mode, + } = self; + let data = match data { + //Data::IPC(ipc_shared_memory) => ipc_shared_memory, + Data::Owned(items) => IpcSharedMemory::from_bytes(&items), + }; + Snapshot { + size, + data, + format, + alpha_mode, + } + } + + pub fn to_vec(self) -> Vec { + match self.data { + Data::Owned(data) => data, + } + } +} + +impl Snapshot { + // TODO: https://github.com/servo/servo/issues/36594 + /* + /// # Safety + /// + /// This is safe if data is owned by this process only + /// (ownership is transferred on send) + pub unsafe fn to_data(self) -> Snapshot { + let Snapshot { + size, + data, + format, + alpha_mode, + } = self; + Snapshot { + size, + data: Data::IPC(data), + format, + alpha_mode, + } + } + */ + pub fn to_owned(self) -> Snapshot { + let Snapshot { + size, + data, + format, + alpha_mode, + } = self; + Snapshot { + size, + data: Data::Owned(data.to_vec()), + format, + alpha_mode, + } + } + + pub fn data(&self) -> &[u8] { + &self.data + } + + pub fn to_ipc_shared_memory(self) -> IpcSharedMemory { + self.data + } +} diff --git a/components/shared/webgpu/Cargo.toml b/components/shared/webgpu/Cargo.toml index 0a4aa4e7ff3..04efea0cb9c 100644 --- a/components/shared/webgpu/Cargo.toml +++ b/components/shared/webgpu/Cargo.toml @@ -20,3 +20,4 @@ serde = { workspace = true } webrender_api = { workspace = true } wgpu-core = { workspace = true, features = ["serde", "wgsl"] } wgpu-types = { workspace = true } +snapshot = { workspace = true } diff --git a/components/shared/webgpu/messages/recv.rs b/components/shared/webgpu/messages/recv.rs index 47c32437e45..4a30177cf0a 100644 --- a/components/shared/webgpu/messages/recv.rs +++ b/components/shared/webgpu/messages/recv.rs @@ -9,6 +9,7 @@ use arrayvec::ArrayVec; use base::id::PipelineId; use ipc_channel::ipc::{IpcSender, IpcSharedMemory}; use serde::{Deserialize, Serialize}; +use snapshot::IpcSnapshot; use webrender_api::ImageKey; use webrender_api::units::DeviceIntSize; use wgpu_core::Label; @@ -165,7 +166,7 @@ pub enum WebGPURequest { /// Obtains image from latest presentation buffer (same as wr update) GetImage { context_id: WebGPUContextId, - sender: IpcSender, + sender: IpcSender, }, ValidateTextureDescriptor { device_id: DeviceId, diff --git a/components/webgpu/Cargo.toml b/components/webgpu/Cargo.toml index f25f160b1b0..39ca562affe 100644 --- a/components/webgpu/Cargo.toml +++ b/components/webgpu/Cargo.toml @@ -22,6 +22,7 @@ malloc_size_of = { workspace = true } serde = { workspace = true, features = ["serde_derive"] } servo_config = { path = "../config" } webgpu_traits = { workspace = true } +snapshot = { workspace = true } webrender = { workspace = true } webrender_api = { workspace = true } wgpu-core = { workspace = true, features = ["serde", "wgsl"] } diff --git a/components/webgpu/swapchain.rs b/components/webgpu/swapchain.rs index 6d61540581b..a3ca15b638d 100644 --- a/components/webgpu/swapchain.rs +++ b/components/webgpu/swapchain.rs @@ -10,9 +10,10 @@ use std::sync::{Arc, Mutex}; use arrayvec::ArrayVec; use compositing_traits::{WebrenderExternalImageApi, WebrenderImageSource}; use euclid::default::Size2D; -use ipc_channel::ipc::{IpcSender, IpcSharedMemory}; +use ipc_channel::ipc::IpcSender; use log::{error, warn}; use serde::{Deserialize, Serialize}; +use snapshot::{IpcSnapshot, Snapshot}; use webgpu_traits::{ ContextConfiguration, Error, PRESENTATION_BUFFER_COUNT, WebGPUContextId, WebGPUMsg, }; @@ -364,20 +365,34 @@ impl crate::WGPU { ); } - pub(crate) fn get_image(&self, context_id: WebGPUContextId) -> IpcSharedMemory { + pub(crate) fn get_image(&self, context_id: WebGPUContextId) -> IpcSnapshot { let webgpu_contexts = self.wgpu_image_map.lock().unwrap(); let context_data = webgpu_contexts.get(&context_id).unwrap(); - let buffer_size = context_data.image_desc.buffer_size(); + let size = context_data.image_desc.size().cast().cast_unit(); let data = if let Some(present_buffer) = context_data .swap_chain .as_ref() .and_then(|swap_chain| swap_chain.data.as_ref()) { - IpcSharedMemory::from_bytes(present_buffer.slice()) + let format = match context_data.image_desc.0.format { + ImageFormat::RGBA8 => snapshot::PixelFormat::RGBA, + ImageFormat::BGRA8 => snapshot::PixelFormat::BGRA, + _ => unimplemented!(), + }; + let alpha_mode = if context_data.image_desc.0.is_opaque() { + snapshot::AlphaMode::AsOpaque { + premultiplied: false, + } + } else { + snapshot::AlphaMode::Transparent { + premultiplied: true, + } + }; + Snapshot::from_vec(size, format, alpha_mode, present_buffer.slice().to_vec()) } else { - IpcSharedMemory::from_byte(0, buffer_size as usize) + Snapshot::cleared(size) }; - data + data.as_ipc() } pub(crate) fn update_context( diff --git a/tests/wpt/meta/html/semantics/embedded-content/the-canvas-element/toDataURL.jpeg.alpha.html.ini b/tests/wpt/meta/html/semantics/embedded-content/the-canvas-element/toDataURL.jpeg.alpha.html.ini deleted file mode 100644 index 16ecb34bf33..00000000000 --- a/tests/wpt/meta/html/semantics/embedded-content/the-canvas-element/toDataURL.jpeg.alpha.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[toDataURL.jpeg.alpha.html] - [toDataURL with JPEG composites onto black] - expected: FAIL diff --git a/tests/wpt/mozilla/meta/mozilla/webgl/tex_image_2d_canvas.html.ini b/tests/wpt/mozilla/meta/mozilla/webgl/tex_image_2d_canvas.html.ini deleted file mode 100644 index dc950fa7b63..00000000000 --- a/tests/wpt/mozilla/meta/mozilla/webgl/tex_image_2d_canvas.html.ini +++ /dev/null @@ -1,2 +0,0 @@ -[tex_image_2d_canvas.html] - expected: FAIL diff --git a/tests/wpt/webgl/meta/conformance/context/premultiplyalpha-test.html.ini b/tests/wpt/webgl/meta/conformance/context/premultiplyalpha-test.html.ini index 9afbf256abc..ffc971bc861 100644 --- a/tests/wpt/webgl/meta/conformance/context/premultiplyalpha-test.html.ini +++ b/tests/wpt/webgl/meta/conformance/context/premultiplyalpha-test.html.ini @@ -1,22 +1,5 @@ [premultiplyalpha-test.html] bug: https://github.com/servo/servo/issues/21132 - [WebGL test #13] - expected: FAIL - - [WebGL test #20] - expected: FAIL - - [WebGL test #27] - expected: FAIL - - [WebGL test #48] - expected: FAIL - - [WebGL test #55] - expected: FAIL - - [WebGL test #6] - expected: FAIL [WebGL test #62] expected: FAIL diff --git a/tests/wpt/webgl/meta/conformance/more/functions/copyTexSubImage2D.html.ini b/tests/wpt/webgl/meta/conformance/more/functions/copyTexSubImage2D.html.ini deleted file mode 100644 index 22d4e847667..00000000000 --- a/tests/wpt/webgl/meta/conformance/more/functions/copyTexSubImage2D.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[copyTexSubImage2D.html] - [WebGL test #1] - expected: FAIL diff --git a/tests/wpt/webgl/meta/conformance/textures/misc/tex-image-webgl.html.ini b/tests/wpt/webgl/meta/conformance/textures/misc/tex-image-webgl.html.ini deleted file mode 100644 index 66ef4bda9ce..00000000000 --- a/tests/wpt/webgl/meta/conformance/textures/misc/tex-image-webgl.html.ini +++ /dev/null @@ -1,6 +0,0 @@ -[tex-image-webgl.html] - [WebGL test #2] - expected: FAIL - - [WebGL test #3] - expected: FAIL diff --git a/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance-luminance-unsigned_byte.html.ini b/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance-luminance-unsigned_byte.html.ini index f9e8abf7941..309ce090136 100644 --- a/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance-luminance-unsigned_byte.html.ini +++ b/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance-luminance-unsigned_byte.html.ini @@ -7,38 +7,74 @@ [WebGL test #0] expected: FAIL + [WebGL test #6] + expected: FAIL + [WebGL test #11] expected: FAIL + [WebGL test #12] + expected: FAIL + [WebGL test #17] expected: FAIL + [WebGL test #18] + expected: FAIL + [WebGL test #23] expected: FAIL + [WebGL test #24] + expected: FAIL + [WebGL test #29] expected: FAIL + [WebGL test #30] + expected: FAIL + [WebGL test #35] expected: FAIL + [WebGL test #36] + expected: FAIL + [WebGL test #41] expected: FAIL + [WebGL test #42] + expected: FAIL + [WebGL test #47] expected: FAIL + [WebGL test #48] + expected: FAIL + [WebGL test #5] expected: FAIL [WebGL test #53] expected: FAIL + [WebGL test #54] + expected: FAIL + [WebGL test #59] expected: FAIL + [WebGL test #60] + expected: FAIL + [WebGL test #65] expected: FAIL + [WebGL test #66] + expected: FAIL + [WebGL test #71] expected: FAIL + + [WebGL test #72] + expected: FAIL diff --git a/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini b/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini index 1d2e7e6eb01..81efc8044b3 100644 --- a/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini +++ b/tests/wpt/webgl/meta/conformance/textures/webgl_canvas/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html.ini @@ -7,38 +7,74 @@ [WebGL test #0] expected: FAIL + [WebGL test #6] + expected: FAIL + [WebGL test #11] expected: FAIL + [WebGL test #12] + expected: FAIL + [WebGL test #17] expected: FAIL + [WebGL test #18] + expected: FAIL + [WebGL test #23] expected: FAIL + [WebGL test #24] + expected: FAIL + [WebGL test #29] expected: FAIL + [WebGL test #30] + expected: FAIL + [WebGL test #35] expected: FAIL + [WebGL test #36] + expected: FAIL + [WebGL test #41] expected: FAIL + [WebGL test #42] + expected: FAIL + [WebGL test #47] expected: FAIL + [WebGL test #48] + expected: FAIL + [WebGL test #5] expected: FAIL [WebGL test #53] expected: FAIL + [WebGL test #54] + expected: FAIL + [WebGL test #59] expected: FAIL + [WebGL test #60] + expected: FAIL + [WebGL test #65] expected: FAIL + [WebGL test #66] + expected: FAIL + [WebGL test #71] expected: FAIL + + [WebGL test #72] + expected: FAIL diff --git a/tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_colorspace_rgba16float.https.html.ini b/tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_colorspace_rgba16float.https.html.ini deleted file mode 100644 index dd38fd7a4c2..00000000000 --- a/tests/wpt/webgpu/meta/webgpu/webgpu/web_platform/reftests/canvas_colorspace_rgba16float.https.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[canvas_colorspace_rgba16float.https.html] - expected: - if os == "linux" and not debug: PASS