Enable the zstd decoder (#36530)

Uses the `zstd` support from `async-compression` to support zstd
Content-Encoding.

Testing: Covered by wpt tests.

Signed-off-by: webbeef <me@webbeef.org>

Cargo.lock

@@ -1543,6 +1543,8 @@ dependencies = [
  "compression-core",
  "flate2",
  "memchr",
+ "zstd",
+ "zstd-safe",
 ]
 
 [[package]]
@@ -10950,6 +10952,34 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "zstd"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "7.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
+dependencies = [
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.16+zstd.1.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "zune-core"
 version = "0.4.12"

components/net/Cargo.toml

@@ -15,7 +15,7 @@ test = false
 doctest = false
 
 [dependencies]
-async-compression = { version = "0.4.12", default-features = false, features = ["tokio", "brotli", "gzip", "zlib"] }
+async-compression = { version = "0.4.12", default-features = false, features = ["tokio", "brotli", "gzip", "zlib", "zstd"] }
 async-recursion = "1.1"
 async-tungstenite = { workspace = true }
 base = { workspace = true }

components/net/decoder.rs

@@ -21,7 +21,7 @@ use std::fmt;
 use std::io::{self};
 use std::pin::Pin;
 
-use async_compression::tokio::bufread::{BrotliDecoder, GzipDecoder, ZlibDecoder};
+use async_compression::tokio::bufread::{BrotliDecoder, GzipDecoder, ZlibDecoder, ZstdDecoder};
 use bytes::Bytes;
 use futures::stream::Peekable;
 use futures::task::{Context, Poll};
@@ -51,6 +51,7 @@ enum DecoderType {
     Gzip,
     Brotli,
     Deflate,
+    Zstd,
 }
 
 enum Inner {
@@ -62,6 +63,8 @@ enum Inner {
     Deflate(FramedRead<ZlibDecoder<StreamReader<Peekable<BodyStream>, Bytes>>, BytesCodec>),
     /// A `Brotli` decoder will uncompress the brotli-encoded response content before returning it.
     Brotli(FramedRead<BrotliDecoder<StreamReader<Peekable<BodyStream>, Bytes>>, BytesCodec>),
+    /// A `Zstd` decoder will uncompress the zstd-encoded response content before returning it.
+    Zstd(FramedRead<ZstdDecoder<StreamReader<Peekable<BodyStream>, Bytes>>, BytesCodec>),
     /// A decoder that doesn't have a value yet.
     Pending(Pending),
 }
@@ -131,6 +134,8 @@ impl Decoder {
                     Some(DecoderType::Brotli)
                 } else if enc == HeaderValue::from_static("deflate") {
                     Some(DecoderType::Deflate)
+                } else if enc == HeaderValue::from_static("zstd") {
+                    Some(DecoderType::Zstd)
                 } else {
                     None
                 }
@@ -182,6 +187,13 @@ impl Stream for Decoder {
                     None => Poll::Ready(None),
                 }
             },
+            Inner::Zstd(ref mut decoder) => {
+                match futures_core::ready!(Pin::new(decoder).poll_next(cx)) {
+                    Some(Ok(bytes)) => Poll::Ready(Some(Ok(bytes.freeze()))),
+                    Some(Err(err)) => Poll::Ready(Some(Err(err))),
+                    None => Poll::Ready(None),
+                }
+            },
         }
     }
 }
@@ -223,6 +235,11 @@ impl Future for Pending {
                 BytesCodec::new(),
                 DECODER_BUFFER_SIZE,
             )))),
+            DecoderType::Zstd => Poll::Ready(Ok(Inner::Zstd(FramedRead::with_capacity(
+                ZstdDecoder::new(StreamReader::new(body)),
+                BytesCodec::new(),
+                DECODER_BUFFER_SIZE,
+            )))),
         }
     }
 }

components/net/http_loader.rs

@@ -184,7 +184,7 @@ fn set_default_accept_encoding(headers: &mut HeaderMap) {
     // TODO(eijebong): Change this once typed headers are done
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 }
 

components/net/tests/fetch.rs

@@ -1313,7 +1313,7 @@ fn test_fetch_with_devtools() {
 
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 
     // Append fetch metadata headers

components/net/tests/http_loader.rs

@@ -189,7 +189,7 @@ fn test_check_default_headers_loaded_in_every_request() {
 
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 
     headers.typed_insert(Host::from(
@@ -369,7 +369,7 @@ fn test_request_and_response_data_with_network_messages() {
 
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 
     // Append fetch metadata headers
@@ -1128,7 +1128,7 @@ fn test_load_sets_default_accept_encoding_to_gzip_and_deflate() {
                     .unwrap()
                     .to_str()
                     .unwrap(),
-                "gzip, deflate, br"
+                "gzip, deflate, br, zstd"
             );
             *response.body_mut() = make_body(b"Yay!".to_vec());
         };

tests/wpt/meta/fetch/content-encoding/zstd/big-zstd-body.https.any.js.ini

@@ -2,17 +2,11 @@
   expected: ERROR
 
 [big-zstd-body.https.any.worker.html]
-  [large zstd data should be decompressed successfully]
-    expected: FAIL
-
   [large zstd data should be decompressed successfully with byte stream]
     expected: FAIL
 
 
 [big-zstd-body.https.any.html]
-  [large zstd data should be decompressed successfully]
-    expected: FAIL
-
   [large zstd data should be decompressed successfully with byte stream]
     expected: FAIL
 

tests/wpt/meta/fetch/content-encoding/zstd/zstd-body.https.any.js.ini

@@ -2,19 +2,8 @@
   expected: ERROR
 
 [zstd-body.https.any.html]
-  [fetched zstd data with content type text should be decompressed.]
-    expected: FAIL
-
-  [fetched zstd data with content type octetstream should be decompressed.]
-    expected: FAIL
-
 
 [zstd-body.https.any.sharedworker.html]
   expected: ERROR
 
 [zstd-body.https.any.worker.html]
-  [fetched zstd data with content type text should be decompressed.]
-    expected: FAIL
-
-  [fetched zstd data with content type octetstream should be decompressed.]
-    expected: FAIL

tests/wpt/meta/fetch/content-encoding/zstd/zstd-navigation.https.window.js.ini

@@ -1,3 +0,0 @@
-[zstd-navigation.https.window.html]
-  [Naigation to zstd encoded page]
-    expected: FAIL