From e30440c9ccbd9bfb2bd62b0c2d667e37293d8a1d Mon Sep 17 00:00:00 2001 From: Bastien Orivel Date: Thu, 8 Nov 2018 02:36:59 +0100 Subject: [PATCH 1/3] Don't reset the buffers for each chunk we download Sometimes hyper sends data that can't completely decompressed, resetting the buffer means we're losing some data and thus breaking the body --- components/net/connector.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/net/connector.rs b/components/net/connector.rs index 38693e70e24..05e4ded2bd9 100644 --- a/components/net/connector.rs +++ b/components/net/connector.rs @@ -81,7 +81,7 @@ impl Stream for WrappedBody { Decoder::Plain => Some(chunk), Decoder::Gzip(Some(ref mut decoder)) => { let mut buf = vec![0; BUF_SIZE]; - *decoder.get_mut() = Cursor::new(chunk.into_bytes()); + decoder.get_mut().get_mut().extend(&chunk.into_bytes()); let len = decoder.read(&mut buf).ok()?; buf.truncate(len); Some(buf.into()) @@ -96,7 +96,7 @@ impl Stream for WrappedBody { }, Decoder::Deflate(ref mut decoder) => { let mut buf = vec![0; BUF_SIZE]; - *decoder.get_mut() = Cursor::new(chunk.into_bytes()); + decoder.get_mut().get_mut().extend(&chunk.into_bytes()); let len = decoder.read(&mut buf).ok()?; buf.truncate(len); Some(buf.into()) From dcbe7d36aea1423ff1330a4a0153db846139e945 Mon Sep 17 00:00:00 2001 From: Bastien Orivel Date: Thu, 8 Nov 2018 02:38:03 +0100 Subject: [PATCH 2/3] Continue decompressing chunks even after hyper is done downloading the body If hyper reads compressed enough data, we were decompressing 32k by 32k but we were throwing away the end of the body because we would end up having lots of backed up data in the cursor when hyper was done. --- components/net/connector.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/components/net/connector.rs b/components/net/connector.rs index 05e4ded2bd9..7e9dcddcba1 100644 --- a/components/net/connector.rs +++ b/components/net/connector.rs @@ -110,7 +110,37 @@ impl Stream for WrappedBody { }, } } else { - None + // Hyper is done downloading but we still have uncompressed data + match self.decoder { + Decoder::Gzip(Some(ref mut decoder)) => { + let mut buf = vec![0; BUF_SIZE]; + let len = decoder.read(&mut buf).ok()?; + if len == 0 { + return None; + } + buf.truncate(len); + Some(buf.into()) + }, + Decoder::Deflate(ref mut decoder) => { + let mut buf = vec![0; BUF_SIZE]; + let len = decoder.read(&mut buf).ok()?; + if len == 0 { + return None; + } + buf.truncate(len); + Some(buf.into()) + }, + Decoder::Brotli(ref mut decoder) => { + let mut buf = vec![0; BUF_SIZE]; + let len = decoder.read(&mut buf).ok()?; + if len == 0 { + return None; + } + buf.truncate(len); + Some(buf.into()) + }, + _ => None, + } } }) }) From 85c6eff94a7c1266fddd78ce12e7632db1b8a2ac Mon Sep 17 00:00:00 2001 From: Bastien Orivel Date: Thu, 8 Nov 2018 12:37:50 +0100 Subject: [PATCH 3/3] Use chunk.as_ref() instead of &chunk.into_bytes() --- components/net/connector.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/net/connector.rs b/components/net/connector.rs index 7e9dcddcba1..9151f1821fc 100644 --- a/components/net/connector.rs +++ b/components/net/connector.rs @@ -81,7 +81,7 @@ impl Stream for WrappedBody { Decoder::Plain => Some(chunk), Decoder::Gzip(Some(ref mut decoder)) => { let mut buf = vec![0; BUF_SIZE]; - decoder.get_mut().get_mut().extend(&chunk.into_bytes()); + decoder.get_mut().get_mut().extend(chunk.as_ref()); let len = decoder.read(&mut buf).ok()?; buf.truncate(len); Some(buf.into()) @@ -96,14 +96,14 @@ impl Stream for WrappedBody { }, Decoder::Deflate(ref mut decoder) => { let mut buf = vec![0; BUF_SIZE]; - decoder.get_mut().get_mut().extend(&chunk.into_bytes()); + decoder.get_mut().get_mut().extend(chunk.as_ref()); let len = decoder.read(&mut buf).ok()?; buf.truncate(len); Some(buf.into()) }, Decoder::Brotli(ref mut decoder) => { let mut buf = vec![0; BUF_SIZE]; - decoder.get_mut().get_mut().extend(&chunk.into_bytes()); + decoder.get_mut().get_mut().extend(chunk.as_ref()); let len = decoder.read(&mut buf).ok()?; buf.truncate(len); Some(buf.into())