Closes #2587 Adding HTTP compression capability

components/net/Cargo.toml

@@ -31,3 +31,4 @@ cookie="*"
 regex = "0.1.14"
 regex_macros = "0.1.8"
 hyper = "0.3"
+flate2 = "0.2.0"

components/net/http_loader.rs

@@ -3,15 +3,16 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 use cookie_storage::CookieSource;
-use resource_task::{Metadata, TargetedLoadResponse, LoadData, start_sending_opt, ResponseSenders};
+use resource_task::{Metadata, TargetedLoadResponse, LoadData, start_sending_opt, ResponseSenders, ProgressMsg};
 use resource_task::ControlMsg;
 use resource_task::ProgressMsg::{Payload, Done};
 
 use log;
 use std::collections::HashSet;
 use file_loader;
+use flate2::read::{DeflateDecoder, GzDecoder};
 use hyper::client::Request;
-use hyper::header::{ContentLength, ContentType, Host, Location};
+use hyper::header::{AcceptEncoding, ContentLength, ContentType, Host, Location};
 use hyper::HttpError;
 use hyper::method::Method;
 use hyper::mime::{Mime, TopLevel, SubLevel};
@@ -160,11 +161,9 @@ reason: \"certificate verify failed\" }]";
             req.headers_mut().set_raw("Cookie".to_owned(), v);
         }
 
-        // FIXME(seanmonstar): use AcceptEncoding from Hyper once available
+        if !req.headers().has::<AcceptEncoding>() {
-        //if !req.headers.has::<AcceptEncoding>() {
+            req.headers_mut().set_raw("Accept-Encoding".to_owned(), vec![b"gzip, deflate".to_vec()]);
-            // We currently don't support HTTP Compression (FIXME #2587)
+        }
-            req.headers_mut().set_raw("Accept-Encoding".to_owned(), vec![b"identity".to_vec()]);
-        //}
         if log_enabled!(log::INFO) {
             info!("{}", load_data.method);
             for header in req.headers().iter() {
@@ -296,24 +295,32 @@ reason: \"certificate verify failed\" }]";
             Ok(p) => p,
             _ => return
         };
-        loop {
-            let mut buf = Vec::with_capacity(1024);
 
-            unsafe { buf.set_len(1024); }
+        let mut encoding_str: Option<String> = None;
-            match response.read(buf.as_mut_slice()) {
+        //FIXME: Implement Content-Encoding Header https://github.com/hyperium/hyper/issues/391
-                Ok(len) if len > 0 => {
+        if let Some(encodings) = response.headers.get_raw("content-encoding") {
-                    unsafe { buf.set_len(len); }
+            for encoding in encodings.iter() {
-                    if progress_chan.send(Payload(buf)).is_err() {
+                if let Ok(encodings) = String::from_utf8(encoding.clone()) {
-                        // The send errors when the receiver is out of scope,
+                    if encodings == "gzip" || encodings == "deflate" {
-                        // which will happen if the fetch has timed out (or has been aborted)
+                        encoding_str = Some(encodings);
-                        // so we don't need to continue with the loading of the file here.
+                        break;
-                        return;
                     }
                 }
-                Ok(_) | Err(_) => {
+            }
-                    let _ = progress_chan.send(Done(Ok(())));
+        }
-                    break;
+
+        match encoding_str {
+            Some(encoding) => {
+                if encoding == "gzip" {
+                    let mut response_decoding = GzDecoder::new(response).unwrap();
+                    send_data(&mut response_decoding, progress_chan);
+                } else if encoding == "deflate" {
+                    let mut response_decoding = DeflateDecoder::new(response);
+                    send_data(&mut response_decoding, progress_chan);
                 }
+            },
+            None => {
+                send_data(&mut response, progress_chan);
             }
         }
 
@@ -321,3 +328,26 @@ reason: \"certificate verify failed\" }]";
         break;
     }
 }
+
+fn send_data<R: Read>(reader: &mut R, progress_chan: Sender<ProgressMsg>) {
+    loop {
+        let mut buf = Vec::with_capacity(1024);
+
+        unsafe { buf.set_len(1024); }
+        match reader.read(buf.as_mut_slice()) {
+            Ok(len) if len > 0 => {
+                unsafe { buf.set_len(len); }
+                if progress_chan.send(Payload(buf)).is_err() {
+                    // The send errors when the receiver is out of scope,
+                    // which will happen if the fetch has timed out (or has been aborted)
+                    // so we don't need to continue with the loading of the file here.
+                    return;
+                }
+            }
+            Ok(_) | Err(_) => {
+                let _ = progress_chan.send(Done(Ok(())));
+                break;
+            }
+        }
+    }
+}

components/net/lib.rs

@@ -20,6 +20,7 @@
 
 extern crate "cookie" as cookie_rs;
 extern crate collections;
+extern crate flate2;
 extern crate geom;
 extern crate hyper;
 extern crate png;

components/servo/Cargo.lock

@@ -240,6 +240,15 @@ name = "expat-sys"
 version = "2.1.0"
 source = "git+https://github.com/servo/libexpat#523a2f2f51b41adf7bb5c4c65e80db0cb615d70b"
 
+[[package]]
+name = "flate2"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "fontconfig"
 version = "0.1.0"
@@ -586,6 +595,15 @@ dependencies = [
  "log 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "miniz-sys"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "gcc 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "mod_path"
 version = "0.1.4"
@@ -617,6 +635,7 @@ name = "net"
 version = "0.0.1"
 dependencies = [
  "cookie 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "flate2 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "geom 0.1.0 (git+https://github.com/servo/rust-geom)",
  "hyper 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "openssl 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",

ports/cef/Cargo.lock

@@ -245,6 +245,15 @@ name = "expat-sys"
 version = "2.1.0"
 source = "git+https://github.com/servo/libexpat#523a2f2f51b41adf7bb5c4c65e80db0cb615d70b"
 
+[[package]]
+name = "flate2"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "fontconfig"
 version = "0.1.0"
@@ -591,6 +600,15 @@ dependencies = [
  "log 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "miniz-sys"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "gcc 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "mod_path"
 version = "0.1.4"
@@ -622,6 +640,7 @@ name = "net"
 version = "0.0.1"
 dependencies = [
  "cookie 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "flate2 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "geom 0.1.0 (git+https://github.com/servo/rust-geom)",
  "hyper 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "openssl 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",

ports/gonk/Cargo.lock

@@ -224,6 +224,15 @@ name = "expat-sys"
 version = "2.1.0"
 source = "git+https://github.com/servo/libexpat#523a2f2f51b41adf7bb5c4c65e80db0cb615d70b"
 
+[[package]]
+name = "flate2"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz-sys 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "fontconfig"
 version = "0.1.0"
@@ -516,6 +525,15 @@ dependencies = [
  "log 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "miniz-sys"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "gcc 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "mod_path"
 version = "0.1.4"
@@ -547,6 +565,7 @@ name = "net"
 version = "0.0.1"
 dependencies = [
  "cookie 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "flate2 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "geom 0.1.0 (git+https://github.com/servo/rust-geom)",
  "hyper 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "openssl 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",

tests/wpt/metadata/XMLHttpRequest/response-data-gzip.htm.ini

@@ -1,5 +0,0 @@
-[response-data-gzip.htm]
-  type: testharness
-  [XMLHttpRequest: GZIP response was correctly inflated]
-    expected: FAIL
-