Auto merge of #14114 - servo:fetch-unit-http, r=jdm

Start using fetch for the HTTP unit tests.

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/14114)
<!-- Reviewable:end -->

components/net/fetch/methods.rs

@@ -8,7 +8,7 @@ use data_loader::decode;
 use devtools_traits::DevtoolsControlMsg;
 use fetch::cors_cache::CORSCache;
 use filemanager_thread::{FileManager, UIProvider};
-use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies};
+use http_loader::{HttpState, set_default_accept_encoding, set_default_accept_language, set_request_cookies};
 use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block};
 use http_loader::{auth_from_cache, determine_request_referrer, set_cookies_from_headers};
 use http_loader::{send_response_to_devtools, send_request_to_devtools, LoadErrorType};
@@ -120,9 +120,7 @@ pub fn fetch_with_cors_cache<UI: 'static + UIProvider>(request: Rc<Request>,
     }
 
     // Step 4
-    if !request.headers.borrow().has::<AcceptLanguage>() {
+    set_default_accept_language(&mut request.headers.borrow_mut());
-        request.headers.borrow_mut().set(AcceptLanguage(vec![qitem("en-US".parse().unwrap())]));
-    }
 
     // Step 5
     // TODO: Figure out what a Priority object is

components/net/http_loader.rs

@@ -413,7 +413,7 @@ fn set_default_accept(headers: &mut Headers) {
     }
 }
 
-fn set_default_accept_language(headers: &mut Headers) {
+pub fn set_default_accept_language(headers: &mut Headers) {
     if headers.has::<AcceptLanguage>() {
         return;
     }

tests/unit/net/fetch.rs

@@ -2,7 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use {DEFAULT_USER_AGENT, FetchResponseCollector, new_fetch_context, fetch_async, fetch_sync};
+use {DEFAULT_USER_AGENT, FetchResponseCollector, new_fetch_context, fetch_async, fetch_sync, make_server};
 use devtools_traits::DevtoolsControlMsg;
 use devtools_traits::HttpRequest as DevtoolsHttpRequest;
 use devtools_traits::HttpResponse as DevtoolsHttpResponse;
@@ -12,11 +12,10 @@ use hyper::header::{Accept, AccessControlAllowCredentials, AccessControlAllowHea
 use hyper::header::{AcceptEncoding, AcceptLanguage, AccessControlAllowMethods, AccessControlMaxAge};
 use hyper::header::{AccessControlRequestHeaders, AccessControlRequestMethod, Date, UserAgent};
 use hyper::header::{CacheControl, ContentLanguage, ContentLength, ContentType, Expires, LastModified};
-use hyper::header::{Encoding, Location, Pragma, SetCookie, qitem};
+use hyper::header::{Encoding, Location, Pragma, Quality, QualityItem, SetCookie, qitem};
 use hyper::header::{Headers, Host, HttpDate, Referer as HyperReferer};
 use hyper::method::Method;
 use hyper::mime::{Mime, SubLevel, TopLevel};
-use hyper::server::{Handler, Listening, Server};
 use hyper::server::{Request as HyperRequest, Response as HyperResponse};
 use hyper::status::StatusCode;
 use hyper::uri::RequestUri;
@@ -39,16 +38,6 @@ use util::resource_files::resources_dir_path;
 
 // TODO write a struct that impls Handler for storing test values
 
-fn make_server<H: Handler + 'static>(handler: H) -> (Listening, Url) {
-    // this is a Listening server because of handle_threads()
-    let server = Server::http("0.0.0.0:0").unwrap().handle_threads(handler, 1).unwrap();
-    let port = server.socket.port().to_string();
-    let mut url_string = "http://localhost:".to_owned();
-    url_string.push_str(&port);
-    let url = Url::parse(&url_string).unwrap();
-    (server, url)
-}
-
 #[test]
 fn test_fetch_response_is_not_network_error() {
     static MESSAGE: &'static [u8] = b"";
@@ -787,7 +776,12 @@ fn test_fetch_with_devtools() {
     let mut en_us: LanguageTag = Default::default();
     en_us.language = Some("en".to_owned());
     en_us.region = Some("US".to_owned());
-    headers.set(AcceptLanguage(vec![qitem(en_us)]));
+    let mut en: LanguageTag = Default::default();
+    en.language = Some("en".to_owned());
+    headers.set(AcceptLanguage(vec![
+        qitem(en_us),
+        QualityItem::new(en, Quality(500)),
+    ]));
 
     headers.set(UserAgent(DEFAULT_USER_AGENT.to_owned()));
 

tests/unit/net/http_loader.rs

@@ -7,6 +7,7 @@ use cookie_rs::Cookie as CookiePair;
 use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, NetworkEvent};
 use devtools_traits::HttpRequest as DevtoolsHttpRequest;
 use devtools_traits::HttpResponse as DevtoolsHttpResponse;
+use fetch_sync;
 use flate2::Compression;
 use flate2::write::{DeflateEncoder, GzEncoder};
 use hyper::LanguageTag;
@@ -17,7 +18,9 @@ use hyper::header::{StrictTransportSecurity, UserAgent};
 use hyper::http::RawStatus;
 use hyper::method::Method;
 use hyper::mime::{Mime, SubLevel, TopLevel};
+use hyper::server::{Request as HyperRequest, Response as HyperResponse};
 use hyper::status::StatusCode;
+use make_server;
 use msg::constellation_msg::{PipelineId, TEST_PIPELINE_ID};
 use net::cookie::Cookie;
 use net::cookie_storage::CookieStorage;
@@ -27,9 +30,10 @@ use net::test::{HttpRequest, HttpRequestFactory, HttpState, LoadError, UIProvide
 use net::test::{HttpResponse, LoadErrorType};
 use net_traits::{CookieSource, IncludeSubdomains, LoadContext, LoadData};
 use net_traits::{CustomResponse, LoadOrigin, Metadata, ReferrerPolicy};
+use net_traits::request::{Request, RequestInit, Destination};
 use std::borrow::Cow;
 use std::io::{self, Cursor, Read, Write};
-use std::sync::{Arc, RwLock, mpsc};
+use std::sync::{Arc, Mutex, RwLock, mpsc};
 use std::sync::mpsc::Receiver;
 use std::thread;
 use url::Url;
@@ -229,20 +233,6 @@ impl HttpRequest for MockRequest {
     }
 }
 
-struct AssertMustHaveHeadersRequestFactory {
-    expected_headers: Headers,
-    body: Vec<u8>
-}
-
-impl HttpRequestFactory for AssertMustHaveHeadersRequestFactory {
-    type R = MockRequest;
-
-    fn create(&self, _: Url, _: Method, headers: Headers) -> Result<MockRequest, LoadError> {
-        assert_eq!(headers, self.expected_headers);
-        Ok(MockRequest::new(ResponseType::Text(self.body.clone())))
-    }
-}
-
 struct AssertAuthHeaderRequestFactory {
     expected_headers: Headers,
     body: Vec<u8>
@@ -373,14 +363,12 @@ pub fn expect_devtools_http_response(devtools_port: &Receiver<DevtoolsControlMsg
 
 #[test]
 fn test_check_default_headers_loaded_in_every_request() {
-    let url = Url::parse("http://mozilla.com").unwrap();
+    let expected_headers = Arc::new(Mutex::new(None));
-
+    let expected_headers_clone = expected_headers.clone();
-    let http_state = HttpState::new();
+    let handler = move |request: HyperRequest, _: HyperResponse| {
-    let ui_provider = TestProvider::new();
+        assert_eq!(request.headers, expected_headers_clone.lock().unwrap().take().unwrap());
-
+    };
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), &HttpTest);
+    let (mut server, url) = make_server(handler);
-    load_data.data = None;
-    load_data.method = Method::Get;
 
     let mut headers = Headers::new();
 
@@ -388,7 +376,12 @@ fn test_check_default_headers_loaded_in_every_request() {
                                     qitem(Encoding::Deflate),
                                     qitem(Encoding::EncodingExt("br".to_owned()))]));
 
-    headers.set(Host { hostname: "mozilla.com".to_owned() , port: None });
+    let hostname = match url.host_str() {
+        Some(hostname) => hostname.to_owned(),
+        _ => panic!()
+    };
+
+    headers.set(Host { hostname: hostname, port: url.port() });
 
     let accept = Accept(vec![
                             qitem(Mime(TopLevel::Text, SubLevel::Html, vec![])),
@@ -408,25 +401,37 @@ fn test_check_default_headers_loaded_in_every_request() {
         QualityItem::new(en, Quality(500)),
     ]));
 
-    headers.set(UserAgent(DEFAULT_USER_AGENT.to_owned()));
+    headers.set(UserAgent(::DEFAULT_USER_AGENT.to_owned()));
+
+    *expected_headers.lock().unwrap() = Some(headers.clone());
 
     // Testing for method.GET
-    let _ = load(&load_data, &ui_provider, &http_state, None,
+    let request = Request::from_init(RequestInit {
-                 &AssertMustHaveHeadersRequestFactory {
+        url: url.clone(),
-                     expected_headers: headers.clone(),
+        method: Method::Get,
-                     body: <[_]>::to_vec(&[])
+        destination: Destination::Document,
-                 }, DEFAULT_USER_AGENT.into(), &CancellationListener::new(None), None);
+        origin: url.clone(),
+        pipeline_id: Some(TEST_PIPELINE_ID),
+        .. RequestInit::default()
+    });
+    let response = fetch_sync(request, None);
+    assert!(response.status.unwrap().is_success());
 
     // Testing for method.POST
-    load_data.method = Method::Post;
-
     headers.set(ContentLength(0 as u64));
+    *expected_headers.lock().unwrap() = Some(headers.clone());
+    let request = Request::from_init(RequestInit {
+        url: url.clone(),
+        method: Method::Post,
+        destination: Destination::Document,
+        origin: url.clone(),
+        pipeline_id: Some(TEST_PIPELINE_ID),
+        .. RequestInit::default()
+    });
+    let response = fetch_sync(request, None);
+    assert!(response.status.unwrap().is_success());
 
-    let _ = load(&load_data, &ui_provider, &http_state, None,
+    let _ = server.close();
-                 &AssertMustHaveHeadersRequestFactory {
-                     expected_headers: headers,
-                     body: <[_]>::to_vec(&[])
-                 }, DEFAULT_USER_AGENT.into(), &CancellationListener::new(None), None);
 }
 
 #[test]

tests/unit/net/lib.rs

@@ -35,6 +35,7 @@ extern crate util;
 
 use devtools_traits::DevtoolsControlMsg;
 use filemanager_thread::{TestProvider, TEST_PROVIDER};
+use hyper::server::{Handler, Listening, Server};
 use net::fetch::methods::{FetchContext, fetch};
 use net::filemanager_thread::FileManager;
 use net::test::HttpState;
@@ -44,6 +45,7 @@ use net_traits::response::Response;
 use std::rc::Rc;
 use std::sync::mpsc::Sender;
 use std::thread;
+use url::Url;
 
 const DEFAULT_USER_AGENT: &'static str = "Such Browser. Very Layout. Wow.";
 
@@ -79,3 +81,11 @@ fn fetch_async(request: Request, target: Box<FetchTaskTarget + Send>, dc: Option
 fn fetch_sync(request: Request, dc: Option<Sender<DevtoolsControlMsg>>) -> Response {
     fetch(Rc::new(request), &mut None, new_fetch_context(dc))
 }
+
+fn make_server<H: Handler + 'static>(handler: H) -> (Listening, Url) {
+    // this is a Listening server because of handle_threads()
+    let server = Server::http("0.0.0.0:0").unwrap().handle_threads(handler, 1).unwrap();
+    let url_string = format!("http://localhost:{}", server.socket.port());
+    let url = Url::parse(&url_string).unwrap();
+    (server, url)
+}