Enable the zstd decoder (#36530)

Uses the `zstd` support from `async-compression` to support zstd
Content-Encoding.

Testing: Covered by wpt tests.

Signed-off-by: webbeef <me@webbeef.org>

components/net/Cargo.toml

@@ -15,7 +15,7 @@ test = false
 doctest = false
 
 [dependencies]
-async-compression = { version = "0.4.12", default-features = false, features = ["tokio", "brotli", "gzip", "zlib"] }
+async-compression = { version = "0.4.12", default-features = false, features = ["tokio", "brotli", "gzip", "zlib", "zstd"] }
 async-recursion = "1.1"
 async-tungstenite = { workspace = true }
 base = { workspace = true }

components/net/decoder.rs

@@ -21,7 +21,7 @@ use std::fmt;
 use std::io::{self};
 use std::pin::Pin;
 
-use async_compression::tokio::bufread::{BrotliDecoder, GzipDecoder, ZlibDecoder};
+use async_compression::tokio::bufread::{BrotliDecoder, GzipDecoder, ZlibDecoder, ZstdDecoder};
 use bytes::Bytes;
 use futures::stream::Peekable;
 use futures::task::{Context, Poll};
@@ -51,6 +51,7 @@ enum DecoderType {
     Gzip,
     Brotli,
     Deflate,
+    Zstd,
 }
 
 enum Inner {
@@ -62,6 +63,8 @@ enum Inner {
     Deflate(FramedRead<ZlibDecoder<StreamReader<Peekable<BodyStream>, Bytes>>, BytesCodec>),
     /// A `Brotli` decoder will uncompress the brotli-encoded response content before returning it.
     Brotli(FramedRead<BrotliDecoder<StreamReader<Peekable<BodyStream>, Bytes>>, BytesCodec>),
+    /// A `Zstd` decoder will uncompress the zstd-encoded response content before returning it.
+    Zstd(FramedRead<ZstdDecoder<StreamReader<Peekable<BodyStream>, Bytes>>, BytesCodec>),
     /// A decoder that doesn't have a value yet.
     Pending(Pending),
 }
@@ -131,6 +134,8 @@ impl Decoder {
                     Some(DecoderType::Brotli)
                 } else if enc == HeaderValue::from_static("deflate") {
                     Some(DecoderType::Deflate)
+                } else if enc == HeaderValue::from_static("zstd") {
+                    Some(DecoderType::Zstd)
                 } else {
                     None
                 }
@@ -182,6 +187,13 @@ impl Stream for Decoder {
                     None => Poll::Ready(None),
                 }
             },
+            Inner::Zstd(ref mut decoder) => {
+                match futures_core::ready!(Pin::new(decoder).poll_next(cx)) {
+                    Some(Ok(bytes)) => Poll::Ready(Some(Ok(bytes.freeze()))),
+                    Some(Err(err)) => Poll::Ready(Some(Err(err))),
+                    None => Poll::Ready(None),
+                }
+            },
         }
     }
 }
@@ -223,6 +235,11 @@ impl Future for Pending {
                 BytesCodec::new(),
                 DECODER_BUFFER_SIZE,
             )))),
+            DecoderType::Zstd => Poll::Ready(Ok(Inner::Zstd(FramedRead::with_capacity(
+                ZstdDecoder::new(StreamReader::new(body)),
+                BytesCodec::new(),
+                DECODER_BUFFER_SIZE,
+            )))),
         }
     }
 }

components/net/http_loader.rs

@@ -184,7 +184,7 @@ fn set_default_accept_encoding(headers: &mut HeaderMap) {
     // TODO(eijebong): Change this once typed headers are done
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 }
 

components/net/tests/fetch.rs

@@ -1313,7 +1313,7 @@ fn test_fetch_with_devtools() {
 
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 
     // Append fetch metadata headers

components/net/tests/http_loader.rs

@@ -189,7 +189,7 @@ fn test_check_default_headers_loaded_in_every_request() {
 
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 
     headers.typed_insert(Host::from(
@@ -369,7 +369,7 @@ fn test_request_and_response_data_with_network_messages() {
 
     headers.insert(
         header::ACCEPT_ENCODING,
-        HeaderValue::from_static("gzip, deflate, br"),
+        HeaderValue::from_static("gzip, deflate, br, zstd"),
     );
 
     // Append fetch metadata headers
@@ -1128,7 +1128,7 @@ fn test_load_sets_default_accept_encoding_to_gzip_and_deflate() {
                     .unwrap()
                     .to_str()
                     .unwrap(),
-                "gzip, deflate, br"
+                "gzip, deflate, br, zstd"
             );
             *response.body_mut() = make_body(b"Yay!".to_vec());
         };