Auto merge of #11556 - Manishearth:make-fetch-happen, r=jdm

Make fetch happen

<!-- Please describe your changes on the following line: -->
Moves XHR over to the fetch backend.

Previous PR: https://github.com/servo/servo/pull/114

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [ ] `./mach test-tidy` does not report any errors (Will fix later)

<!-- Either: -->
- [x] There are tests for these changes OR
- [ ] These changes do not require tests because _____

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/11556)
<!-- Reviewable:end -->

components/net/fetch/methods.rs

@@ -5,20 +5,24 @@
 use connector::create_http_connector;
 use data_loader::decode;
 use fetch::cors_cache::CORSCache;
-use http_loader::{NetworkHttpRequestFactory, ReadResult, obtain_response, read_block};
+use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies};
+use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block};
+use http_loader::{auth_from_cache, determine_request_referrer};
 use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials};
 use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods};
 use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic};
 use hyper::header::{CacheControl, CacheDirective, ContentEncoding, ContentLength, ContentLanguage, ContentType};
-use hyper::header::{Encoding, HeaderView, Headers, IfMatch, IfRange, IfUnmodifiedSince, IfModifiedSince};
+use hyper::header::{Encoding, HeaderView, Headers, Host, IfMatch, IfRange, IfUnmodifiedSince, IfModifiedSince};
 use hyper::header::{IfNoneMatch, Pragma, Location, QualityItem, Referer as RefererHeader, UserAgent, q, qitem};
 use hyper::method::Method;
 use hyper::mime::{Mime, SubLevel, TopLevel};
 use hyper::status::StatusCode;
 use mime_guess::guess_mime_type;
-use net_traits::AsyncFetchListener;
+use msg::constellation_msg::ReferrerPolicy;
-use net_traits::request::{CacheMode, CredentialsMode, Type, Origin, Window};
+use net_traits::FetchTaskTarget;
+use net_traits::request::{CacheMode, CredentialsMode};
 use net_traits::request::{RedirectMode, Referer, Request, RequestMode, ResponseTainting};
+use net_traits::request::{Type, Origin, Window};
 use net_traits::response::{HttpsState, TerminationReason};
 use net_traits::response::{Response, ResponseBody, ResponseType};
 use resource_thread::CancellationListener;
@@ -26,27 +30,36 @@ use std::collections::HashSet;
 use std::fs::File;
 use std::io::Read;
 use std::iter::FromIterator;
+use std::mem::swap;
 use std::rc::Rc;
-use std::thread;
+use std::sync::mpsc::{channel, Sender, Receiver};
 use unicase::UniCase;
 use url::{Origin as UrlOrigin, Url};
 use util::thread::spawn_named;
 
-pub fn fetch_async(request: Request, listener: Box<AsyncFetchListener + Send>) {
+pub type Target = Option<Box<FetchTaskTarget + Send>>;
-    spawn_named(format!("fetch for {:?}", request.current_url_string()), move || {
+
-        let request = Rc::new(request);
+enum Data {
-        let fetch_response = fetch(request);
+    Payload(Vec<u8>),
-        fetch_response.wait_until_done();
+    Done,
-        listener.response_available(fetch_response);
-    })
 }
 
+pub struct FetchContext {
+    pub state: HttpState,
+    pub user_agent: String,
+}
+
+type DoneChannel = Option<(Sender<Data>, Receiver<Data>)>;
+
 /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch)
-pub fn fetch(request: Rc<Request>) -> Response {
+pub fn fetch(request: Rc<Request>, target: &mut Target, context: FetchContext) -> Response {
-    fetch_with_cors_cache(request, &mut CORSCache::new())
+    fetch_with_cors_cache(request, &mut CORSCache::new(), target, context)
 }
 
-pub fn fetch_with_cors_cache(request: Rc<Request>, cache: &mut CORSCache) -> Response {
+pub fn fetch_with_cors_cache(request: Rc<Request>,
+                             cache: &mut CORSCache,
+                             target: &mut Target,
+                             context: FetchContext) -> Response {
     // Step 1
     if request.window.get() == Window::Client {
         // TODO: Set window to request's client object if client is a Window object
@@ -105,12 +118,15 @@ pub fn fetch_with_cors_cache(request: Rc<Request>, cache: &mut CORSCache) -> Res
     if request.is_subresource_request() {
         // TODO: create a fetch record and append it to request's client's fetch group list
     }
+
     // Step 7
-    main_fetch(request, cache, false, false)
+    main_fetch(request, cache, false, false, target, &mut None, &context)
 }
 
 /// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch)
-fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recursive_flag: bool) -> Response {
+fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
+              recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel,
+              context: &FetchContext) -> Response {
     // TODO: Implement main fetch spec
 
     // Step 1
@@ -128,23 +144,39 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
     // TODO be able to execute report CSP
 
     // Step 4
-    // TODO this step, based off of http_loader.rs
+    // TODO this step, based off of http_loader.rs (upgrade)
 
     // Step 5
-    // TODO this step
+    // TODO this step (CSP port/content blocking)
 
     // Step 6
-    if request.referer != Referer::NoReferer {
+    // TODO this step (referer policy)
-        // TODO be able to invoke "determine request's referer"
+    // currently the clients themselves set referer policy in RequestInit
-    }
 
     // Step 7
-    // TODO this step
+    if request.referrer_policy.get().is_none() {
+        request.referrer_policy.set(Some(ReferrerPolicy::NoRefWhenDowngrade));
+    }
 
     // Step 8
-    // this step is obsoleted by fetch_async
+    if *request.referer.borrow() != Referer::NoReferer {
+        // remove Referer headers set in past redirects/preflights
+        // this stops the assertion in determine_request_referrer from failing
+        request.headers.borrow_mut().remove::<RefererHeader>();
+        let referrer_url = determine_request_referrer(&mut *request.headers.borrow_mut(),
+                                                      request.referrer_policy.get(),
+                                                      request.referer.borrow_mut().take(),
+                                                      request.current_url().clone());
+        *request.referer.borrow_mut() = Referer::from_url(referrer_url);
+    }
 
     // Step 9
+    // TODO this step (HSTS)
+
+    // Step 10
+    // this step is obsoleted by fetch_async
+
+    // Step 11
     let response = match response {
         Some(response) => response,
         None => {
@@ -160,14 +192,14 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
                 (current_url.scheme() == "file" && request.same_origin_data.get()) ||
                 current_url.scheme() == "about" ||
                 request.mode == RequestMode::Navigate {
-                basic_fetch(request.clone(), cache)
+                basic_fetch(request.clone(), cache, target, done_chan, context)
 
             } else if request.mode == RequestMode::SameOrigin {
                 Response::network_error()
 
             } else if request.mode == RequestMode::NoCORS {
                 request.response_tainting.set(ResponseTainting::Opaque);
-                basic_fetch(request.clone(), cache)
+                basic_fetch(request.clone(), cache, target, done_chan, context)
 
             } else if !matches!(current_url.scheme(), "http" | "https") {
                 Response::network_error()
@@ -178,7 +210,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
                   request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) {
                 request.response_tainting.set(ResponseTainting::CORSTainting);
                 request.redirect_mode.set(RedirectMode::Error);
-                let response = http_fetch(request.clone(), cache, true, true, false);
+                let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, context);
                 if response.is_network_error() {
                     // TODO clear cache entries using request
                 }
@@ -186,17 +218,17 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
 
             } else {
                 request.response_tainting.set(ResponseTainting::CORSTainting);
-                http_fetch(request.clone(), cache, true, false, false)
+                http_fetch(request.clone(), cache, true, false, false, target, done_chan, context)
             }
         }
     };
 
-    // Step 10
+    // Step 12
     if recursive_flag {
         return response;
     }
 
-    // Step 11
+    // Step 13
     // no need to check if response is a network error, since the type would not be `Default`
     let response = if response.response_type == ResponseType::Default {
         let response_type = match request.response_tainting.get() {
@@ -210,7 +242,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
     };
 
     {
-        // Step 12
+        // Step 14
         let network_error_res = Response::network_error();
         let internal_response = if response.is_network_error() {
             &network_error_res
@@ -218,10 +250,15 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
             response.actual_response()
         };
 
-        // Step 13
+        // Step 15
-        // TODO this step
+        if internal_response.url_list.borrow().is_empty() {
+            *internal_response.url_list.borrow_mut() = request.url_list.borrow().clone();
+        }
 
-        // Step 14
+        // Step 16
+        // TODO this step (CSP/blocking)
+
+        // Step 17
         if !response.is_network_error() && (is_null_body_status(&internal_response.status) ||
             match *request.method.borrow() {
                 Method::Head | Method::Connect => true,
@@ -233,7 +270,7 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
             *body = ResponseBody::Empty;
         }
 
-        // Step 15
+        // Step 18
         // TODO be able to compare response integrity against request integrity metadata
         // if !response.is_network_error() {
 
@@ -248,34 +285,91 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
         // }
     }
 
-    // Step 16
+    // Step 19
     if request.synchronous {
-        response.actual_response().wait_until_done();
+        if let Some(ref mut target) = *target {
+            // process_response is not supposed to be used
+            // by sync fetch, but we overload it here for simplicity
+            target.process_response(&response);
+        }
+
+        if let Some(ref ch) = *done_chan {
+            loop {
+                match ch.1.recv()
+                        .expect("fetch worker should always send Done before terminating") {
+                    Data::Payload(vec) => {
+                        if let Some(ref mut target) = *target {
+                            target.process_response_chunk(vec);
+                        }
+                    }
+                    Data::Done => break,
+                }
+            }
+        } else if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
+            // in case there was no channel to wait for, the body was
+            // obtained synchronously via basic_fetch for data/file/about/etc
+            // We should still send the body across as a chunk
+            if let Some(ref mut target) = *target {
+                target.process_response_chunk(vec.clone());
+            }
+        } else {
+            assert!(*response.body.lock().unwrap() == ResponseBody::Empty)
+        }
+
+        // overloaded similarly to process_response
+        if let Some(ref mut target) = *target {
+            target.process_response_eof(&response);
+        }
         return response;
     }
 
-    // Step 17
+    // Step 20
     if request.body.borrow().is_some() && matches!(request.current_url().scheme(), "http" | "https") {
-        // TODO queue a fetch task on request to process end-of-file
+        if let Some(ref mut target) = *target {
+            // XXXManishearth: We actually should be calling process_request
+            // in http_network_fetch. However, we can't yet follow the request
+            // upload progress, so I'm keeping it here for now and pretending
+            // the body got sent in one chunk
+            target.process_request_body(&request);
+            target.process_request_eof(&request);
+        }
     }
 
-    {
+    // Step 21
-        // Step 12 repeated to use internal_response
+    if let Some(ref mut target) = *target {
-        let network_error_res = Response::network_error();
+        target.process_response(&response);
-        let internal_response = if response.is_network_error() {
+    }
-            &network_error_res
+
+    // Step 22
+    if let Some(ref ch) = *done_chan {
+        loop {
+            match ch.1.recv()
+                    .expect("fetch worker should always send Done before terminating") {
+                Data::Payload(vec) => {
+                    if let Some(ref mut target) = *target {
+                        target.process_response_chunk(vec);
+                    }
+                }
+                Data::Done => break,
+            }
+        }
+    } else if let Some(ref mut target) = *target {
+        if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
+            // in case there was no channel to wait for, the body was
+            // obtained synchronously via basic_fetch for data/file/about/etc
+            // We should still send the body across as a chunk
+            target.process_response_chunk(vec.clone());
         } else {
-            response.actual_response()
+            assert!(*response.body.lock().unwrap() == ResponseBody::Empty)
-        };
+        }
+    }
 
-        // Step 18
+    // Step 23
-        // TODO this step
+    request.done.set(true);
 
-        // Step 19
+    // Step 24
-        internal_response.wait_until_done();
+    if let Some(ref mut target) = *target {
-
+        target.process_response_eof(&response);
-        // Step 20
-        // TODO this step
     }
 
     // TODO remove this line when only asynchronous fetches are used
@@ -283,19 +377,23 @@ fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool, recu
 }
 
 /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch)
-fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response {
+fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache,
+               target: &mut Target, done_chan: &mut DoneChannel,
+               context: &FetchContext) -> Response {
     let url = request.current_url();
 
     match url.scheme() {
         "about" if url.path() == "blank" => {
             let mut response = Response::new();
+            // https://github.com/whatwg/fetch/issues/312
+            response.url = Some(url);
             response.headers.set(ContentType(mime!(Text / Html; Charset = Utf8)));
             *response.body.lock().unwrap() = ResponseBody::Done(vec![]);
             response
         },
 
         "http" | "https" => {
-            http_fetch(request.clone(), cache, false, false, false)
+            http_fetch(request.clone(), cache, false, false, false, target, done_chan, context)
         },
 
         "data" => {
@@ -303,6 +401,8 @@ fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response {
                 match decode(&url) {
                     Ok((mime, bytes)) => {
                         let mut response = Response::new();
+                        // https://github.com/whatwg/fetch/issues/312
+                        response.url = Some(url.clone());
                         *response.body.lock().unwrap() = ResponseBody::Done(bytes);
                         response.headers.set(ContentType(mime));
                         response
@@ -324,6 +424,8 @@ fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response {
                             let mime = guess_mime_type(file_path);
 
                             let mut response = Response::new();
+                            // https://github.com/whatwg/fetch/issues/312
+                            response.url = Some(url.clone());
                             *response.body.lock().unwrap() = ResponseBody::Done(bytes);
                             response.headers.set(ContentType(mime));
                             response
@@ -350,7 +452,12 @@ fn http_fetch(request: Rc<Request>,
               cache: &mut CORSCache,
               cors_flag: bool,
               cors_preflight_flag: bool,
-              authentication_fetch_flag: bool) -> Response {
+              authentication_fetch_flag: bool,
+              target: &mut Target,
+              done_chan: &mut DoneChannel,
+              context: &FetchContext) -> Response {
+    // This is a new async fetch, reset the channel we are waiting on
+    *done_chan = None;
     // Step 1
     let mut response: Option<Response> = None;
 
@@ -378,17 +485,18 @@ fn http_fetch(request: Rc<Request>,
             }
 
             // Substep 4
-            let actual_response = res.actual_response();
-            if actual_response.url_list.borrow().is_empty() {
-                *actual_response.url_list.borrow_mut() = request.url_list.borrow().clone();
-            }
-
-            // Substep 5
             // TODO: set response's CSP list on actual_response
         }
     }
 
     // Step 4
+    let credentials = match request.credentials_mode {
+        CredentialsMode::Include => true,
+        CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic
+            => true,
+        _ => false
+    };
+    // Step 5
     if response.is_none() {
         // Substep 1
         if cors_preflight_flag {
@@ -403,7 +511,7 @@ fn http_fetch(request: Rc<Request>,
 
             // Sub-substep 1
             if method_mismatch || header_mismatch {
-                let preflight_result = cors_preflight_fetch(request.clone(), cache);
+                let preflight_result = cors_preflight_fetch(request.clone(), cache, context);
                 // Sub-substep 2
                 if preflight_result.response_type == ResponseType::Error {
                     return Response::network_error();
@@ -415,17 +523,10 @@ fn http_fetch(request: Rc<Request>,
         request.skip_service_worker.set(true);
 
         // Substep 3
-        let credentials = match request.credentials_mode {
+        let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag,
-            CredentialsMode::Include => true,
+                                                       done_chan, context);
-            CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic
-                => true,
-            _ => false
-        };
 
         // Substep 4
-        let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag);
-
-        // Substep 5
         if cors_flag && cors_check(request.clone(), &fetch_result).is_err() {
             return Response::network_error();
         }
@@ -450,7 +551,8 @@ fn http_fetch(request: Rc<Request>,
                 RedirectMode::Follow => {
                     // set back to default
                     response.return_internal.set(true);
-                    http_redirect_fetch(request, cache, Rc::new(response), cors_flag)
+                    http_redirect_fetch(request, cache, Rc::new(response),
+                                        cors_flag, target, done_chan, context)
                 }
             }
         },
@@ -459,7 +561,7 @@ fn http_fetch(request: Rc<Request>,
         StatusCode::Unauthorized => {
             // Step 1
             // FIXME: Figure out what to do with request window objects
-            if cors_flag || request.credentials_mode != CredentialsMode::Include {
+            if cors_flag || !credentials {
                 return response;
             }
 
@@ -469,10 +571,15 @@ fn http_fetch(request: Rc<Request>,
             // Step 3
             if !request.use_url_credentials || authentication_fetch_flag {
                 // TODO: Prompt the user for username and password from the window
+                // Wrong, but will have to do until we are able to prompt the user
+                // otherwise this creates an infinite loop
+                // We basically pretend that the user declined to enter credentials
+                return response;
             }
 
             // Step 4
-            return http_fetch(request, cache, cors_flag, cors_preflight_flag, true);
+            return http_fetch(request, cache, cors_flag, cors_preflight_flag,
+                              true, target, done_chan, context);
         }
 
         // Code 407
@@ -485,11 +592,16 @@ fn http_fetch(request: Rc<Request>,
 
             // Step 3
             // TODO: Prompt the user for proxy authentication credentials
+            // Wrong, but will have to do until we are able to prompt the user
+            // otherwise this creates an infinite loop
+            // We basically pretend that the user declined to enter credentials
+            return response;
 
             // Step 4
-            return http_fetch(request, cache,
+            // return http_fetch(request, cache,
-                              cors_flag, cors_preflight_flag,
+            //                   cors_flag, cors_preflight_flag,
-                              authentication_fetch_flag);
+            //                   authentication_fetch_flag, target,
+            //                   done_chan, context);
         }
 
         _ => { }
@@ -510,7 +622,10 @@ fn http_fetch(request: Rc<Request>,
 fn http_redirect_fetch(request: Rc<Request>,
                        cache: &mut CORSCache,
                        response: Rc<Response>,
-                       cors_flag: bool) -> Response {
+                       cors_flag: bool,
+                       target: &mut Target,
+                       done_chan: &mut DoneChannel,
+                       context: &FetchContext) -> Response {
     // Step 1
     assert_eq!(response.return_internal.get(), true);
 
@@ -584,20 +699,22 @@ fn http_redirect_fetch(request: Rc<Request>,
     request.url_list.borrow_mut().push(location_url);
 
     // Step 15
-    main_fetch(request, cache, cors_flag, true)
+    main_fetch(request, cache, cors_flag, true, target, done_chan, context)
 }
 
 /// [HTTP network or cache fetch](https://fetch.spec.whatwg.org#http-network-or-cache-fetch)
 fn http_network_or_cache_fetch(request: Rc<Request>,
                                credentials_flag: bool,
-                               authentication_fetch_flag: bool) -> Response {
+                               authentication_fetch_flag: bool,
+                               done_chan: &mut DoneChannel,
+                               context: &FetchContext) -> Response {
     // TODO: Implement Window enum for Request
     let request_has_no_window = true;
 
     // Step 1
     let http_request = if request_has_no_window &&
-        request.redirect_mode.get() != RedirectMode::Follow {
+        request.redirect_mode.get() == RedirectMode::Error {
-        request.clone()
+        request
     } else {
         Rc::new((*request).clone())
     };
@@ -621,9 +738,8 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
     }
 
     // Step 6
-    match http_request.referer {
+    match *http_request.referer.borrow() {
-        Referer::NoReferer =>
+        Referer::NoReferer => (),
-            http_request.headers.borrow_mut().set(RefererHeader("".to_owned())),
         Referer::RefererUrl(ref http_request_referer) =>
             http_request.headers.borrow_mut().set(RefererHeader(http_request_referer.to_string())),
         Referer::Client =>
@@ -640,7 +756,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
     // Step 8
     if !http_request.headers.borrow().has::<UserAgent>() {
-        http_request.headers.borrow_mut().set(UserAgent(global_user_agent().to_owned()));
+        http_request.headers.borrow_mut().set(UserAgent(context.user_agent.clone()));
     }
 
     match http_request.cache_mode.get() {
@@ -670,34 +786,53 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
         _ => {}
     }
 
+    let current_url = http_request.current_url();
     // Step 12
-    // modify_request_headers(http_request.headers.borrow());
+    // todo: pass referrer url and policy
+    // this can only be uncommented when the referer header is set, else it crashes
+    // in the meantime, we manually set the headers in the block below
+    // modify_request_headers(&mut http_request.headers.borrow_mut(), &current_url,
+    //                        None, None, None);
+    {
+        let headers = &mut *http_request.headers.borrow_mut();
+        let host = Host {
+            hostname: current_url.host_str().unwrap().to_owned(),
+            port: current_url.port_or_known_default()
+        };
+        headers.set(host);
+        // unlike http_loader, we should not set the accept header
+        // here, according to the fetch spec
+        set_default_accept_encoding(headers);
+    }
 
     // Step 13
     // TODO some of this step can't be implemented yet
     if credentials_flag {
         // Substep 1
         // TODO http://mxr.mozilla.org/servo/source/components/net/http_loader.rs#504
-
+        // XXXManishearth http_loader has block_cookies: support content blocking here too
+        set_request_cookies(&current_url,
+                            &mut *http_request.headers.borrow_mut(),
+                            &context.state.cookie_jar);
         // Substep 2
         if !http_request.headers.borrow().has::<Authorization<String>>() {
             // Substep 3
             let mut authorization_value = None;
 
             // Substep 4
-            // TODO be able to retrieve https://fetch.spec.whatwg.org/#authentication-entry
+            if let Some(basic) = auth_from_cache(&context.state.auth_cache, &current_url) {
+                if !http_request.use_url_credentials || !has_credentials(&current_url) {
+                    authorization_value = Some(basic);
+                }
+            }
 
             // Substep 5
-            if authentication_fetch_flag {
+            if authentication_fetch_flag && authorization_value.is_none() {
-                let current_url = http_request.current_url();
+                if has_credentials(&current_url) {
-
+                    authorization_value = Some(Basic {
-                authorization_value = if has_credentials(&current_url) {
-                    Some(Basic {
                         username: current_url.username().to_owned(),
                         password: current_url.password().map(str::to_owned)
                     })
-                } else {
-                    None
                 }
             }
 
@@ -753,7 +888,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
     // Step 18
     if response.is_none() {
-        response = Some(http_network_fetch(request.clone(), http_request.clone(), credentials_flag));
+        response = Some(http_network_fetch(http_request.clone(), credentials_flag, done_chan));
     }
     let response = response.unwrap();
 
@@ -788,8 +923,8 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
 /// [HTTP network fetch](https://fetch.spec.whatwg.org/#http-network-fetch)
 fn http_network_fetch(request: Rc<Request>,
-                      _http_request: Rc<Request>,
+                      _credentials_flag: bool,
-                      _credentials_flag: bool) -> Response {
+                      done_chan: &mut DoneChannel) -> Response {
     // TODO: Implement HTTP network fetch spec
 
     // Step 1
@@ -811,42 +946,70 @@ fn http_network_fetch(request: Rc<Request>,
 
     let wrapped_response = obtain_response(&factory, &url, &request.method.borrow(),
                                            &request.headers.borrow(),
-                                           &cancellation_listener, &None, &request.method.borrow(),
+                                           &cancellation_listener, &request.body.borrow(), &request.method.borrow(),
-                                           &None, request.redirect_count.get(), &None, "");
+                                           &None, request.redirect_count.get() + 1, &None, "");
 
     let mut response = Response::new();
     match wrapped_response {
-        Ok((mut res, _)) => {
+        Ok((res, _)) => {
-            response.url = Some(res.response.url.clone());
+            response.url = Some(url.clone());
             response.status = Some(res.response.status);
+            response.raw_status = Some(res.response.status_raw().clone());
             response.headers = res.response.headers.clone();
 
             let res_body = response.body.clone();
-            thread::spawn(move || {
-                *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]);
 
-                loop {
+            // We're about to spawn a thread to be waited on here
-                    match read_block(&mut res.response) {
+            *done_chan = Some(channel());
-                        Ok(ReadResult::Payload(ref mut chunk)) => {
+            let meta = response.metadata().expect("Response metadata should exist at this stage");
-                            if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() {
+            let done_sender = done_chan.as_ref().map(|ch| ch.0.clone());
-                                body.append(chunk);
+            spawn_named(format!("fetch worker thread"), move || {
+                match StreamedResponse::from_http_response(box res, meta) {
+                    Ok(mut res) => {
+                        *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]);
+                        loop {
+                            match read_block(&mut res) {
+                                Ok(ReadResult::Payload(chunk)) => {
+                                    if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() {
+                                        body.extend_from_slice(&chunk);
+                                        if let Some(ref sender) = done_sender {
+                                            let _ = sender.send(Data::Payload(chunk));
+                                        }
+                                    }
+                                },
+                                Ok(ReadResult::EOF) | Err(_) => {
+                                    let mut empty_vec = Vec::new();
+                                    let completed_body = match *res_body.lock().unwrap() {
+                                        ResponseBody::Receiving(ref mut body) => {
+                                            // avoid cloning the body
+                                            swap(body, &mut empty_vec);
+                                            empty_vec
+                                        },
+                                        _ => empty_vec,
+                                    };
+                                    *res_body.lock().unwrap() = ResponseBody::Done(completed_body);
+                                    if let Some(ref sender) = done_sender {
+                                        let _ = sender.send(Data::Done);
+                                    }
+                                    break;
+                                }
                             }
-                        },
+
-                        Ok(ReadResult::EOF) | Err(_) => {
+                        }
-                            let completed_body = match *res_body.lock().unwrap() {
+                    }
-                                ResponseBody::Receiving(ref body) => (*body).clone(),
+                    Err(_) => {
-                                _ => vec![]
+                        // XXXManishearth we should propagate this error somehow
-                            };
+                        *res_body.lock().unwrap() = ResponseBody::Done(vec![]);
-                            *res_body.lock().unwrap() = ResponseBody::Done(completed_body);
+                        if let Some(ref sender) = done_sender {
-                            break;
+                            let _ = sender.send(Data::Done);
                         }
                     }
-
                 }
             });
         },
-        Err(_) =>
+        Err(_) => {
-            response.termination_reason = Some(TerminationReason::Fatal)
+            response.termination_reason = Some(TerminationReason::Fatal);
+        }
     };
 
         // TODO these substeps aren't possible yet
@@ -860,7 +1023,10 @@ fn http_network_fetch(request: Rc<Request>,
 
     // TODO Read request
 
-    // Step 5
+    // Step 5-9
+    // (needs stream bodies)
+
+    // Step 10
     // TODO when https://bugzilla.mozilla.org/show_bug.cgi?id=1030660
     // is resolved, this step will become uneccesary
     // TODO this step
@@ -872,22 +1038,19 @@ fn http_network_fetch(request: Rc<Request>,
         }
     };
 
-    // Step 6
+    // Step 11
-    *response.url_list.borrow_mut() = request.url_list.borrow().clone();
+    // TODO this step isn't possible yet (CSP)
 
-    // Step 7
+    // Step 12
-    // TODO this step isn't possible yet
-
-    // Step 8
     if response.is_network_error() && request.cache_mode.get() == CacheMode::NoStore {
         // TODO update response in the HTTP cache for request
     }
 
     // TODO this step isn't possible yet
-    // Step 9
+    // Step 13
 
     // TODO these steps
-    // Step 10
+    // Step 14
         // Substep 1
         // Substep 2
             // Sub-substep 1
@@ -896,19 +1059,20 @@ fn http_network_fetch(request: Rc<Request>,
             // Sub-substep 4
         // Substep 3
 
-    // Step 11
+    // Step 15
     response
 }
 
 /// [CORS preflight fetch](https://fetch.spec.whatwg.org#cors-preflight-fetch)
-fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response {
+fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache, context: &FetchContext) -> Response {
     // Step 1
     let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()), false);
     *preflight.method.borrow_mut() = Method::Options;
     preflight.initiator = request.initiator.clone();
     preflight.type_ = request.type_.clone();
     preflight.destination = request.destination.clone();
-    preflight.referer = request.referer.clone();
+    *preflight.referer.borrow_mut() = request.referer.borrow().clone();
+    preflight.referrer_policy.set(preflight.referrer_policy.get());
 
     // Step 2
     preflight.headers.borrow_mut().set::<AccessControlRequestMethod>(
@@ -929,7 +1093,7 @@ fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response
 
     // Step 6
     let preflight = Rc::new(preflight);
-    let response = http_network_or_cache_fetch(preflight.clone(), false, false);
+    let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, context);
 
     // Step 7
     if cors_check(request.clone(), &response).is_ok() &&
@@ -962,12 +1126,16 @@ fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache) -> Response
         }
 
         // Substep 5
+        debug!("CORS check: Allowed methods: {:?}, current method: {:?}",
+                methods, request.method.borrow());
         if methods.iter().all(|method| *method != *request.method.borrow()) &&
             !is_simple_method(&*request.method.borrow()) {
             return Response::network_error();
         }
 
         // Substep 6
+        debug!("CORS check: Allowed headers: {:?}, current headers: {:?}",
+                header_names, request.headers.borrow());
         let set: HashSet<&UniCase<String>> = HashSet::from_iter(header_names.iter());
         if request.headers.borrow().iter().any(|ref hv| !set.contains(&UniCase(hv.name().to_owned())) &&
                                                         !is_simple_header(hv)) {
@@ -1040,12 +1208,6 @@ fn cors_check(request: Rc<Request>, response: &Response) -> Result<(), ()> {
     Err(())
 }
 
-fn global_user_agent() -> String {
-    // TODO have a better useragent string
-    const USER_AGENT_STRING: &'static str = "Servo";
-    USER_AGENT_STRING.to_owned()
-}
-
 fn has_credentials(url: &Url) -> bool {
     !url.username().is_empty() || url.password().is_some()
 }
@@ -1056,6 +1218,7 @@ fn is_no_store_cache(headers: &Headers) -> bool {
     headers.has::<IfRange>()
 }
 
+/// https://fetch.spec.whatwg.org/#cors-safelisted-request-header
 fn is_simple_header(h: &HeaderView) -> bool {
     if h.is::<ContentType>() {
         match h.value() {

components/net/http_loader.rs

@@ -370,7 +370,7 @@ impl Error for LoadErrorType {
     }
 }
 
-fn set_default_accept_encoding(headers: &mut Headers) {
+pub fn set_default_accept_encoding(headers: &mut Headers) {
     if headers.has::<AcceptEncoding>() {
         return
     }
@@ -434,10 +434,10 @@ fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option<Url> {
 }
 
 /// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
-fn determine_request_referrer(headers: &mut Headers,
+pub fn determine_request_referrer(headers: &mut Headers,
-                              referrer_policy: Option<ReferrerPolicy>,
+                                  referrer_policy: Option<ReferrerPolicy>,
-                              referrer_url: Option<Url>,
+                                  referrer_url: Option<Url>,
-                              url: Url) -> Option<Url> {
+                                  url: Url) -> Option<Url> {
     //TODO - algorithm step 2 not addressed
     assert!(!headers.has::<Referer>());
     if let Some(ref_url) = referrer_url {
@@ -453,9 +453,9 @@ fn determine_request_referrer(headers: &mut Headers,
     return None;
 }
 
-pub fn set_request_cookies(url: Url, headers: &mut Headers, cookie_jar: &Arc<RwLock<CookieStorage>>) {
+pub fn set_request_cookies(url: &Url, headers: &mut Headers, cookie_jar: &Arc<RwLock<CookieStorage>>) {
     let mut cookie_jar = cookie_jar.write().unwrap();
-    if let Some(cookie_list) = cookie_jar.cookies_for_url(&url, CookieSource::HTTP) {
+    if let Some(cookie_list) = cookie_jar.cookies_for_url(url, CookieSource::HTTP) {
         let mut v = Vec::new();
         v.push(cookie_list.into_bytes());
         headers.set_raw("Cookie".to_owned(), v);
@@ -540,7 +540,7 @@ impl StreamedResponse {
         StreamedResponse { metadata: m, decoder: d }
     }
 
-    fn from_http_response(response: Box<HttpResponse>, m: Metadata) -> Result<StreamedResponse, LoadError> {
+    pub fn from_http_response(response: Box<HttpResponse>, m: Metadata) -> Result<StreamedResponse, LoadError> {
         let decoder = match response.content_encoding() {
             Some(Encoding::Gzip) => {
                 let result = GzDecoder::new(response);
@@ -627,10 +627,7 @@ fn request_must_be_secured(url: &Url, hsts_list: &Arc<RwLock<HstsList>>) -> bool
 pub fn modify_request_headers(headers: &mut Headers,
                               url: &Url,
                               user_agent: &str,
-                              cookie_jar: &Arc<RwLock<CookieStorage>>,
+                              referrer_policy: Option<ReferrerPolicy>,
-                              auth_cache: &Arc<RwLock<AuthCache>>,
-                              load_data: &LoadData,
-                              block_cookies: bool,
                               referrer_url: &mut Option<Url>) {
     // Ensure that the host header is set from the original url
     let host = Host {
@@ -654,23 +651,13 @@ pub fn modify_request_headers(headers: &mut Headers,
     set_default_accept_encoding(headers);
 
     *referrer_url = determine_request_referrer(headers,
-                                               load_data.referrer_policy.clone(),
+                                               referrer_policy.clone(),
                                                referrer_url.clone(),
                                                url.clone());
 
     if let Some(referer_val) = referrer_url.clone() {
         headers.set(Referer(referer_val.into_string()));
     }
-
-    // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
-    if load_data.credentials_flag {
-        if !block_cookies {
-            set_request_cookies(url.clone(), headers, cookie_jar);
-        }
-
-        // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12
-        set_auth_header(headers, url, auth_cache);
-    }
 }
 
 fn set_auth_header(headers: &mut Headers,
@@ -680,18 +667,21 @@ fn set_auth_header(headers: &mut Headers,
         if let Some(auth) = auth_from_url(url) {
             headers.set(auth);
         } else {
-            if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) {
+            if let Some(basic) = auth_from_cache(auth_cache, url) {
-                auth_from_entry(&auth_entry, headers);
+                headers.set(Authorization(basic));
             }
         }
     }
 }
 
-fn auth_from_entry(auth_entry: &AuthCacheEntry, headers: &mut Headers) {
+pub fn auth_from_cache(auth_cache: &Arc<RwLock<AuthCache>>, url: &Url) -> Option<Basic> {
-    let user_name = auth_entry.user_name.clone();
+    if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) {
-    let password  = Some(auth_entry.password.clone());
+        let user_name = auth_entry.user_name.clone();
-
+        let password  = Some(auth_entry.password.clone());
-    headers.set(Authorization(Basic { username: user_name, password: password }));
+        Some(Basic { username: user_name, password: password })
+    } else {
+        None
+    }
 }
 
 fn auth_from_url(doc_url: &Url) -> Option<Authorization<Basic>> {
@@ -956,9 +946,18 @@ pub fn load<A, B>(load_data: &LoadData,
         let request_id = uuid::Uuid::new_v4().simple().to_string();
 
         modify_request_headers(&mut request_headers, &doc_url,
-                               &user_agent, &http_state.cookie_jar,
+                               &user_agent, load_data.referrer_policy,
-                               &http_state.auth_cache, &load_data,
+                               &mut referrer_url);
-                               block_cookies, &mut referrer_url);
+
+        // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
+        if load_data.credentials_flag {
+            if !block_cookies {
+                set_request_cookies(&doc_url, &mut request_headers, &http_state.cookie_jar);
+            }
+
+            // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12
+            set_auth_header(&mut request_headers, &doc_url, &http_state.auth_cache);
+        }
 
         //if there is a new auth header then set the request headers with it
         if let Some(ref auth_header) = new_auth_header {

components/net/resource_thread.rs

@@ -11,6 +11,7 @@ use cookie;
 use cookie_storage::CookieStorage;
 use data_loader;
 use devtools_traits::DevtoolsControlMsg;
+use fetch::methods::{fetch, FetchContext};
 use file_loader;
 use filemanager_thread::FileManagerThreadFactory;
 use hsts::HstsList;
@@ -22,9 +23,11 @@ use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag};
 use net_traits::LoadContext;
 use net_traits::ProgressMsg::Done;
+use net_traits::request::{Request, RequestInit};
 use net_traits::{AsyncResponseTarget, Metadata, ProgressMsg, ResponseAction, CoreResourceThread};
-use net_traits::{CoreResourceMsg, CookieSource, LoadConsumer, LoadData, LoadResponse, ResourceId};
+use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, FetchTaskTarget, LoadConsumer};
-use net_traits::{NetworkError, WebSocketCommunicate, WebSocketConnectData, ResourceThreads};
+use net_traits::{LoadData, LoadResponse, NetworkError, ResourceId};
+use net_traits::{WebSocketCommunicate, WebSocketConnectData, ResourceThreads};
 use profile_traits::time::ProfilerChan;
 use rustc_serialize::json;
 use rustc_serialize::{Decodable, Encodable};
@@ -36,6 +39,7 @@ use std::error::Error;
 use std::fs::File;
 use std::io::prelude::*;
 use std::path::Path;
+use std::rc::Rc;
 use std::sync::mpsc::{Receiver, Sender, channel};
 use std::sync::{Arc, RwLock};
 use storage_thread::StorageThreadFactory;
@@ -193,6 +197,8 @@ impl ResourceChannelManager {
             match self.from_client.recv().unwrap() {
                 CoreResourceMsg::Load(load_data, consumer, id_sender) =>
                     self.resource_manager.load(load_data, consumer, id_sender, control_sender.clone()),
+                CoreResourceMsg::Fetch(init, sender) =>
+                    self.resource_manager.fetch(init, sender),
                 CoreResourceMsg::WebsocketConnect(connect, connect_data) =>
                     self.resource_manager.websocket_connect(connect, connect_data),
                 CoreResourceMsg::SetCookiesForUrl(request, cookie_list, source) =>
@@ -480,6 +486,26 @@ impl CoreResourceManager {
                          cancel_listener));
     }
 
+    fn fetch(&self, init: RequestInit, sender: IpcSender<FetchResponseMsg>) {
+        let http_state = HttpState {
+            hsts_list: self.hsts_list.clone(),
+            cookie_jar: self.cookie_jar.clone(),
+            auth_cache: self.auth_cache.clone(),
+            blocked_content: BLOCKED_CONTENT_RULES.clone(),
+        };
+        let ua = self.user_agent.clone();
+        spawn_named(format!("fetch thread for {}", init.url), move || {
+            let request = Request::from_init(init);
+            // XXXManishearth: Check origin against pipeline id (also ensure that the mode is allowed)
+            // todo load context / mimesniff in fetch
+            // todo referrer policy?
+            // todo service worker stuff
+            let mut target = Some(Box::new(sender) as Box<FetchTaskTarget + Send + 'static>);
+            let context = FetchContext { state: http_state, user_agent: ua };
+            fetch(Rc::new(request), &mut target, context);
+        })
+    }
+
     fn websocket_connect(&self,
                          connect: WebSocketCommunicate,
                          connect_data: WebSocketConnectData) {

components/net/websocket_loader.rs

@@ -43,7 +43,7 @@ fn establish_a_websocket_connection(resource_url: &Url, net_url: (Host, String,
         request.headers.set(WebSocketProtocol(protocols.clone()));
     };
 
-    http_loader::set_request_cookies(resource_url.clone(), &mut request.headers, &cookie_jar);
+    http_loader::set_request_cookies(&resource_url, &mut request.headers, &cookie_jar);
 
     let response = try!(request.send());
     try!(response.validate());

components/net_traits/lib.rs

@@ -36,6 +36,8 @@ use hyper::method::Method;
 use hyper::mime::{Attr, Mime};
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
 use msg::constellation_msg::{PipelineId, ReferrerPolicy};
+use request::{Request, RequestInit};
+use response::{HttpsState, Response};
 use std::io::Error as IOError;
 use std::sync::mpsc::Sender;
 use std::thread;
@@ -112,6 +114,7 @@ pub struct LoadData {
     pub headers: Headers,
     #[ignore_heap_size_of = "Defined in hyper"]
     /// Headers that will apply to the initial request and any redirects
+    /// Unused in fetch
     pub preserved_headers: Headers,
     pub data: Option<Vec<u8>>,
     pub cors: Option<ResourceCORSData>,
@@ -154,9 +157,81 @@ pub trait LoadOrigin {
     fn pipeline_id(&self) -> Option<PipelineId>;
 }
 
-/// Interface for observing the final response for an asynchronous fetch operation.
+#[derive(Deserialize, Serialize)]
-pub trait AsyncFetchListener {
+pub enum FetchResponseMsg {
-    fn response_available(&self, response: response::Response);
+    // todo: should have fields for transmitted/total bytes
+    ProcessRequestBody,
+    ProcessRequestEOF,
+    // todo: send more info about the response (or perhaps the entire Response)
+    ProcessResponse(Result<Metadata, NetworkError>),
+    ProcessResponseChunk(Vec<u8>),
+    ProcessResponseEOF(Result<(), NetworkError>),
+}
+
+pub trait FetchTaskTarget {
+    /// https://fetch.spec.whatwg.org/#process-request-body
+    ///
+    /// Fired when a chunk of the request body is transmitted
+    fn process_request_body(&mut self, request: &Request);
+
+    /// https://fetch.spec.whatwg.org/#process-request-end-of-file
+    ///
+    /// Fired when the entire request finishes being transmitted
+    fn process_request_eof(&mut self, request: &Request);
+
+    /// https://fetch.spec.whatwg.org/#process-response
+    ///
+    /// Fired when headers are received
+    fn process_response(&mut self, response: &Response);
+
+    /// Fired when a chunk of response content is received
+    fn process_response_chunk(&mut self, chunk: Vec<u8>);
+
+    /// https://fetch.spec.whatwg.org/#process-response-end-of-file
+    ///
+    /// Fired when the response is fully fetched
+    fn process_response_eof(&mut self, response: &Response);
+}
+
+pub trait FetchResponseListener {
+    fn process_request_body(&mut self);
+    fn process_request_eof(&mut self);
+    fn process_response(&mut self, metadata: Result<Metadata, NetworkError>);
+    fn process_response_chunk(&mut self, chunk: Vec<u8>);
+    fn process_response_eof(&mut self, response: Result<(), NetworkError>);
+}
+
+impl FetchTaskTarget for IpcSender<FetchResponseMsg> {
+    fn process_request_body(&mut self, _: &Request) {
+        let _ = self.send(FetchResponseMsg::ProcessRequestBody);
+    }
+
+    fn process_request_eof(&mut self, _: &Request) {
+        let _ = self.send(FetchResponseMsg::ProcessRequestEOF);
+    }
+
+    fn process_response(&mut self, response: &Response) {
+        let _ = self.send(FetchResponseMsg::ProcessResponse(response.metadata()));
+    }
+
+    fn process_response_chunk(&mut self, chunk: Vec<u8>) {
+        let _ = self.send(FetchResponseMsg::ProcessResponseChunk(chunk));
+    }
+
+    fn process_response_eof(&mut self, response: &Response) {
+        if response.is_network_error() {
+            // todo: finer grained errors
+            let _ = self.send(FetchResponseMsg::ProcessResponseEOF(
+                              Err(NetworkError::Internal("Network error".into()))));
+        } else {
+            let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(())));
+        }
+    }
+}
+
+
+pub trait Action<Listener> {
+    fn process(self, listener: &mut Listener);
 }
 
 /// A listener for asynchronous network events. Cancelling the underlying request is unsupported.
@@ -183,9 +258,9 @@ pub enum ResponseAction {
     ResponseComplete(Result<(), NetworkError>)
 }
 
-impl ResponseAction {
+impl<T: AsyncResponseListener> Action<T> for ResponseAction {
     /// Execute the default action on a provided listener.
-    pub fn process(self, listener: &mut AsyncResponseListener) {
+    fn process(self, listener: &mut T) {
         match self {
             ResponseAction::HeadersAvailable(m) => listener.headers_available(m),
             ResponseAction::DataAvailable(d) => listener.data_available(d),
@@ -194,6 +269,19 @@ impl ResponseAction {
     }
 }
 
+impl<T: FetchResponseListener> Action<T> for FetchResponseMsg {
+    /// Execute the default action on a provided listener.
+    fn process(self, listener: &mut T) {
+        match self {
+            FetchResponseMsg::ProcessRequestBody => listener.process_request_body(),
+            FetchResponseMsg::ProcessRequestEOF => listener.process_request_eof(),
+            FetchResponseMsg::ProcessResponse(meta) => listener.process_response(meta),
+            FetchResponseMsg::ProcessResponseChunk(data) => listener.process_response_chunk(data),
+            FetchResponseMsg::ProcessResponseEOF(data) => listener.process_response_eof(data),
+        }
+    }
+}
+
 /// A target for async networking events. Commonly used to dispatch a runnable event to another
 /// thread storing the wrapped closure for later execution.
 #[derive(Deserialize, Serialize)]
@@ -331,6 +419,7 @@ pub struct WebSocketConnectData {
 pub enum CoreResourceMsg {
     /// Request the data associated with a particular URL
     Load(LoadData, LoadConsumer, Option<IpcSender<ResourceId>>),
+    Fetch(RequestInit, IpcSender<FetchResponseMsg>),
     /// Try to make a websocket connection to a URL.
     WebsocketConnect(WebSocketCommunicate, WebSocketConnectData),
     /// Store a set of cookies for a given originating URL
@@ -469,7 +558,7 @@ pub struct Metadata {
     pub status: Option<RawStatus>,
 
     /// Is successful HTTPS connection
-    pub https_state: response::HttpsState,
+    pub https_state: HttpsState,
 }
 
 impl Metadata {
@@ -482,7 +571,7 @@ impl Metadata {
             headers: None,
             // https://fetch.spec.whatwg.org/#concept-response-status-message
             status: Some(RawStatus(200, "OK".into())),
-            https_state: response::HttpsState::None,
+            https_state: HttpsState::None,
         }
     }
 

components/net_traits/request.rs

@@ -4,7 +4,9 @@
 
 use hyper::header::Headers;
 use hyper::method::Method;
+use msg::constellation_msg::ReferrerPolicy;
 use std::cell::{Cell, RefCell};
+use std::mem::swap;
 use url::{Origin as UrlOrigin, Url};
 
 /// An [initiator](https://fetch.spec.whatwg.org/#concept-request-initiator)
@@ -25,7 +27,7 @@ pub enum Type {
 }
 
 /// A request [destination](https://fetch.spec.whatwg.org/#concept-request-destination)
-#[derive(Copy, Clone, PartialEq)]
+#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)]
 pub enum Destination {
     None, Document, Embed, Font, Image, Manifest,
     Media, Object, Report, Script, ServiceWorker,
@@ -43,12 +45,13 @@ pub enum Origin {
 #[derive(Clone, PartialEq)]
 pub enum Referer {
     NoReferer,
+    /// Default referer if nothing is specified
     Client,
     RefererUrl(Url)
 }
 
 /// A [request mode](https://fetch.spec.whatwg.org/#concept-request-mode)
-#[derive(Copy, Clone, PartialEq)]
+#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)]
 pub enum RequestMode {
     Navigate,
     SameOrigin,
@@ -57,7 +60,7 @@ pub enum RequestMode {
 }
 
 /// Request [credentials mode](https://fetch.spec.whatwg.org/#concept-request-credentials-mode)
-#[derive(Copy, Clone, PartialEq)]
+#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)]
 pub enum CredentialsMode {
     Omit,
     CredentialsSameOrigin,
@@ -106,6 +109,29 @@ pub enum CORSSettings {
     UseCredentials
 }
 
+#[derive(Serialize, Deserialize, Clone)]
+pub struct RequestInit {
+    pub method: Method,
+    pub url: Url,
+    pub headers: Headers,
+    pub unsafe_request: bool,
+    pub same_origin_data: bool,
+    pub body: Option<Vec<u8>>,
+    // TODO: client object
+    pub destination: Destination,
+    pub synchronous: bool,
+    pub mode: RequestMode,
+    pub use_cors_preflight: bool,
+    pub credentials_mode: CredentialsMode,
+    pub use_url_credentials: bool,
+    // this should actually be set by fetch, but fetch
+    // doesn't have info about the client right now
+    pub origin: Url,
+    // XXXManishearth these should be part of the client object
+    pub referer_url: Option<Url>,
+    pub referrer_policy: Option<ReferrerPolicy>,
+}
+
 /// A [Request](https://fetch.spec.whatwg.org/#requests) as defined by the Fetch spec
 #[derive(Clone)]
 pub struct Request {
@@ -130,8 +156,9 @@ pub struct Request {
     pub origin: RefCell<Origin>,
     pub omit_origin_header: Cell<bool>,
     pub same_origin_data: Cell<bool>,
-    pub referer: Referer,
+    /// https://fetch.spec.whatwg.org/#concept-request-referrer
-    // TODO: referrer policy
+    pub referer: RefCell<Referer>,
+    pub referrer_policy: Cell<Option<ReferrerPolicy>>,
     pub synchronous: bool,
     pub mode: RequestMode,
     pub use_cors_preflight: bool,
@@ -145,7 +172,7 @@ pub struct Request {
     pub url_list: RefCell<Vec<Url>>,
     pub redirect_count: Cell<u32>,
     pub response_tainting: Cell<ResponseTainting>,
-    pub done: Cell<bool>
+    pub done: Cell<bool>,
 }
 
 impl Request {
@@ -169,7 +196,8 @@ impl Request {
             origin: RefCell::new(origin.unwrap_or(Origin::Client)),
             omit_origin_header: Cell::new(false),
             same_origin_data: Cell::new(false),
-            referer: Referer::Client,
+            referer: RefCell::new(Referer::Client),
+            referrer_policy: Cell::new(None),
             synchronous: false,
             mode: RequestMode::NoCORS,
             use_cors_preflight: false,
@@ -185,6 +213,30 @@ impl Request {
         }
     }
 
+    pub fn from_init(init: RequestInit) -> Request {
+        let mut req = Request::new(init.url,
+                                   Some(Origin::Origin(init.origin.origin())),
+                                   false);
+        *req.method.borrow_mut() = init.method;
+        *req.headers.borrow_mut() = init.headers;
+        req.unsafe_request = init.unsafe_request;
+        req.same_origin_data.set(init.same_origin_data);
+        *req.body.borrow_mut() = init.body;
+        req.destination = init.destination;
+        req.synchronous = init.synchronous;
+        req.mode = init.mode;
+        req.use_cors_preflight = init.use_cors_preflight;
+        req.credentials_mode = init.credentials_mode;
+        req.use_url_credentials = init.use_url_credentials;
+        *req.referer.borrow_mut() = if let Some(url) = init.referer_url {
+            Referer::RefererUrl(url)
+        } else {
+            Referer::NoReferer
+        };
+        req.referrer_policy.set(init.referrer_policy);
+        req
+    }
+
     /// https://html.spec.whatwg.org/multipage/#create-a-potential-cors-request
     pub fn potential_cors_request(url: Url,
                                   cors_attribute_state: Option<CORSSettings>,
@@ -207,7 +259,8 @@ impl Request {
             origin: RefCell::new(Origin::Client),
             omit_origin_header: Cell::new(false),
             same_origin_data: Cell::new(false),
-            referer: Referer::Client,
+            referer: RefCell::new(Referer::Client),
+            referrer_policy: Cell::new(None),
             synchronous: false,
             // Step 1-2
             mode: match cors_attribute_state {
@@ -258,3 +311,27 @@ impl Request {
         }
     }
 }
+
+impl Referer {
+    pub fn to_url(&self) -> Option<&Url> {
+        match *self {
+            Referer::NoReferer | Referer::Client => None,
+            Referer::RefererUrl(ref url) => Some(url)
+        }
+    }
+    pub fn from_url(url: Option<Url>) -> Self {
+        if let Some(url) = url {
+            Referer::RefererUrl(url)
+        } else {
+            Referer::NoReferer
+        }
+    }
+    pub fn take(&mut self) -> Option<Url> {
+        let mut new = Referer::Client;
+        swap(self, &mut new);
+        match new {
+            Referer::NoReferer | Referer::Client => None,
+            Referer::RefererUrl(url) => Some(url)
+        }
+    }
+}

components/net_traits/response.rs

@@ -4,15 +4,17 @@
 
 //! The [Response](https://fetch.spec.whatwg.org/#responses) object
 //! resulting from a [fetch operation](https://fetch.spec.whatwg.org/#concept-fetch)
-use hyper::header::{AccessControlExposeHeaders, Headers};
+use hyper::header::{AccessControlExposeHeaders, ContentType, Headers};
+use hyper::http::RawStatus;
 use hyper::status::StatusCode;
 use std::ascii::AsciiExt;
 use std::cell::{Cell, RefCell};
 use std::sync::{Arc, Mutex};
 use url::Url;
+use {Metadata, NetworkError};
 
 /// [Response type](https://fetch.spec.whatwg.org/#concept-response-type)
-#[derive(Clone, PartialEq, Copy, Debug)]
+#[derive(Clone, PartialEq, Copy, Debug, Deserialize, Serialize)]
 pub enum ResponseType {
     Basic,
     CORS,
@@ -23,7 +25,7 @@ pub enum ResponseType {
 }
 
 /// [Response termination reason](https://fetch.spec.whatwg.org/#concept-response-termination-reason)
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Deserialize, Serialize)]
 pub enum TerminationReason {
     EndUserAbort,
     Fatal,
@@ -50,7 +52,7 @@ impl ResponseBody {
 
 
 /// [Cache state](https://fetch.spec.whatwg.org/#concept-response-cache-state)
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Deserialize, Serialize)]
 pub enum CacheState {
     None,
     Local,
@@ -81,6 +83,7 @@ pub struct Response {
     pub url_list: RefCell<Vec<Url>>,
     /// `None` can be considered a StatusCode of `0`.
     pub status: Option<StatusCode>,
+    pub raw_status: Option<RawStatus>,
     pub headers: Headers,
     pub body: Arc<Mutex<ResponseBody>>,
     pub cache_state: CacheState,
@@ -100,6 +103,7 @@ impl Response {
             url: None,
             url_list: RefCell::new(Vec::new()),
             status: Some(StatusCode::Ok),
+            raw_status: Some(RawStatus(200, "OK".into())),
             headers: Headers::new(),
             body: Arc::new(Mutex::new(ResponseBody::Empty)),
             cache_state: CacheState::None,
@@ -116,6 +120,7 @@ impl Response {
             url: None,
             url_list: RefCell::new(vec![]),
             status: None,
+            raw_status: None,
             headers: Headers::new(),
             body: Arc::new(Mutex::new(ResponseBody::Empty)),
             cache_state: CacheState::None,
@@ -132,18 +137,6 @@ impl Response {
         }
     }
 
-    pub fn wait_until_done(&self) {
-        match self.response_type {
-            // since these response types can't hold a body, they should be considered done
-            ResponseType::Error | ResponseType::Opaque | ResponseType::OpaqueRedirect => {},
-            _ => {
-                while !self.body.lock().unwrap().is_done() && !self.is_network_error() {
-                    // loop until done
-                }
-            }
-        }
-    }
-
     pub fn actual_response(&self) -> &Response {
         if self.return_internal.get() && self.internal_response.is_some() {
             &**self.internal_response.as_ref().unwrap()
@@ -228,4 +221,25 @@ impl Response {
 
         response
     }
+
+    pub fn metadata(&self) -> Result<Metadata, NetworkError> {
+        let mut metadata = if let Some(ref url) = self.url {
+            Metadata::default(url.clone())
+        } else {
+            return Err(NetworkError::Internal("No url found in response".to_string()));
+        };
+
+        if self.is_network_error() {
+            return Err(NetworkError::Internal("Cannot extract metadata from network error".to_string()));
+        }
+
+        metadata.set_content_type(match self.headers.get() {
+            Some(&ContentType(ref mime)) => Some(mime),
+            None => None
+        });
+        metadata.headers = Some(self.headers.clone());
+        metadata.status = self.raw_status.clone();
+        metadata.https_state = self.https_state;
+        return Ok(metadata);
+    }
 }

components/script/Cargo.toml

@@ -60,7 +60,6 @@ smallvec = "0.1"
 string_cache = {version = "0.2.18", features = ["heap_size", "unstable"]}
 style = {path = "../style"}
 time = "0.1.12"
-unicase = "1.0"
 url = {version = "1.0.0", features = ["heap_size", "query_encoding"]}
 util = {path = "../util"}
 uuid = {version = "0.2", features = ["v4"]}

components/script/cors.rs

@@ -1,490 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-//! A partial implementation of CORS
-//! For now this library is XHR-specific.
-//! For stuff involving `<img>`, `<iframe>`, `<form>`, etc please check what
-//! the request mode should be and compare with the fetch spec
-//! This library will eventually become the core of the Fetch crate
-//! with CORSRequest being expanded into FetchRequest (etc)
-
-use hyper::client::Request;
-use hyper::header::{AccessControlAllowHeaders, AccessControlRequestHeaders};
-use hyper::header::{AccessControlAllowMethods, AccessControlRequestMethod};
-use hyper::header::{AccessControlAllowOrigin, AccessControlMaxAge};
-use hyper::header::{ContentType, Host};
-use hyper::header::{HeaderView, Headers};
-use hyper::method::Method;
-use hyper::mime::{Mime, SubLevel, TopLevel};
-use hyper::status::StatusClass::Success;
-use net_traits::{AsyncResponseListener, Metadata, NetworkError, ResponseAction};
-use network_listener::{NetworkListener, PreInvoke};
-use script_runtime::ScriptChan;
-use std::ascii::AsciiExt;
-use std::borrow::ToOwned;
-use std::sync::{Arc, Mutex};
-use time::{self, Timespec, now};
-use unicase::UniCase;
-use url::Url;
-use util::thread::spawn_named;
-
-/// Interface for network listeners concerned with CORS checks. Proper network requests
-/// should be initiated from this method, based on the response provided.
-pub trait AsyncCORSResponseListener {
-    fn response_available(&self, response: CORSResponse);
-}
-
-#[derive(Clone, HeapSizeOf)]
-pub struct CORSRequest {
-    pub origin: Url,
-    pub destination: Url,
-    pub mode: RequestMode,
-    #[ignore_heap_size_of = "Defined in hyper"]
-    pub method: Method,
-    #[ignore_heap_size_of = "Defined in hyper"]
-    pub headers: Headers,
-    /// CORS preflight flag (https://fetch.spec.whatwg.org/#concept-http-fetch)
-    /// Indicates that a CORS preflight request and/or cache check is to be performed
-    pub preflight_flag: bool,
-}
-
-/// https://fetch.spec.whatwg.org/#concept-request-mode
-/// This only covers some of the request modes. The
-/// `same-origin` and `no CORS` modes are unnecessary for XHR.
-#[derive(PartialEq, Copy, Clone, HeapSizeOf)]
-pub enum RequestMode {
-    CORS, // CORS
-    ForcedPreflight, // CORS-with-forced-preflight
-}
-
-impl CORSRequest {
-    /// Creates a CORS request if necessary. Will return an error when fetching is forbidden
-    pub fn maybe_new(referer: Url,
-                     destination: Url,
-                     mode: RequestMode,
-                     method: Method,
-                     headers: Headers,
-                     same_origin_data_url_flag: bool)
-                     -> Result<Option<CORSRequest>, ()> {
-        if referer.origin() == destination.origin() {
-            return Ok(None); // Not cross-origin, proceed with a normal fetch
-        }
-        match destination.scheme() {
-            // As per (https://fetch.spec.whatwg.org/#main-fetch 5.1.9), about URLs can be fetched
-            // the same as a basic request.
-            "about" if destination.path() == "blank" => Ok(None),
-            // As per (https://fetch.spec.whatwg.org/#main-fetch 5.1.9), data URLs can be fetched
-            // the same as a basic request if the request's method is GET and the
-            // same-origin data-URL flag is set.
-            "data" if same_origin_data_url_flag && method == Method::Get => Ok(None),
-            "http" | "https" => {
-                let mut req = CORSRequest::new(referer, destination, mode, method, headers);
-                req.preflight_flag = !is_simple_method(&req.method) ||
-                                     mode == RequestMode::ForcedPreflight;
-                if req.headers.iter().any(|h| !is_simple_header(&h)) {
-                    req.preflight_flag = true;
-                }
-                Ok(Some(req))
-            },
-            _ => Err(()),
-        }
-    }
-
-    fn new(mut referer: Url,
-           destination: Url,
-           mode: RequestMode,
-           method: Method,
-           headers: Headers)
-           -> CORSRequest {
-        referer.set_fragment(None);
-        referer.set_query(None);
-        referer.set_path("");
-        CORSRequest {
-            origin: referer,
-            destination: destination,
-            mode: mode,
-            method: method,
-            headers: headers,
-            preflight_flag: false,
-        }
-    }
-
-    pub fn http_fetch_async(&self,
-                            listener: Box<AsyncCORSResponseListener + Send>,
-                            script_chan: Box<ScriptChan + Send>) {
-        struct CORSContext {
-            listener: Box<AsyncCORSResponseListener + Send>,
-            response: Option<CORSResponse>,
-        }
-
-        // This is shoe-horning the CORSReponse stuff into the rest of the async network
-        // framework right now. It would be worth redesigning http_fetch to do this properly.
-        impl AsyncResponseListener for CORSContext {
-            fn headers_available(&mut self, _metadata: Result<Metadata, NetworkError>) {
-            }
-
-            fn data_available(&mut self, _payload: Vec<u8>) {
-            }
-
-            fn response_complete(&mut self, _status: Result<(), NetworkError>) {
-                let response = self.response.take().unwrap();
-                self.listener.response_available(response);
-            }
-        }
-        impl PreInvoke for CORSContext {}
-
-        let context = CORSContext {
-            listener: listener,
-            response: None,
-        };
-        let listener = NetworkListener {
-            context: Arc::new(Mutex::new(context)),
-            script_chan: script_chan,
-        };
-
-        // TODO: this exists only to make preflight check non-blocking
-        // perhaps should be handled by the resource thread?
-        let req = self.clone();
-        spawn_named("cors".to_owned(), move || {
-            let response = req.http_fetch();
-            let mut context = listener.context.lock();
-            let context = context.as_mut().unwrap();
-            context.response = Some(response);
-            listener.notify(ResponseAction::ResponseComplete(Ok(())));
-        });
-    }
-
-    /// http://fetch.spec.whatwg.org/#concept-http-fetch
-    /// This method assumes that the CORS flag is set
-    /// This does not perform the full HTTP fetch, rather it handles part of the CORS filtering
-    /// if self.mode is ForcedPreflight, then the CORS-with-forced-preflight
-    /// fetch flag is set as well
-    pub fn http_fetch(&self) -> CORSResponse {
-        let response = CORSResponse::new();
-        // Step 2: Handle service workers (unimplemented)
-        // Step 3
-        // Substep 1: Service workers (unimplemented )
-        // Substep 2
-        let cache = &mut CORSCache(vec!()); // XXXManishearth Should come from user agent
-        if self.preflight_flag &&
-           !cache.match_method(self, &self.method) &&
-           !self.headers.iter().all(|h| is_simple_header(&h) && cache.match_header(self, h.name())) &&
-           (!is_simple_method(&self.method) || self.mode == RequestMode::ForcedPreflight) {
-            return self.preflight_fetch();
-            // Everything after this is part of XHR::fetch()
-            // Expect the organization of code to improve once we have a fetch crate
-        }
-        response
-    }
-
-    /// https://fetch.spec.whatwg.org/#cors-preflight-fetch
-    fn preflight_fetch(&self) -> CORSResponse {
-        let error = CORSResponse::new_error();
-        let mut cors_response = CORSResponse::new();
-
-        // Step 1
-        let mut preflight = self.clone();
-        preflight.method = Method::Options;
-        preflight.headers = Headers::new();
-        // Step 2
-        preflight.headers.set(AccessControlRequestMethod(self.method.clone()));
-
-        // Steps 3-5
-        let mut header_names = vec![];
-        for header in self.headers.iter() {
-            header_names.push(header.name().to_owned());
-        }
-        header_names.sort();
-        preflight.headers
-                 .set(AccessControlRequestHeaders(header_names.into_iter().map(UniCase).collect()));
-
-        let preflight_request = Request::new(preflight.method, preflight.destination);
-        let mut req = match preflight_request {
-            Ok(req) => req,
-            Err(_) => return error,
-        };
-
-        let host = req.headers().get::<Host>().unwrap().clone();
-        *req.headers_mut() = preflight.headers.clone();
-        req.headers_mut().set(host);
-        let stream = match req.start() {
-            Ok(s) => s,
-            Err(_) => return error,
-        };
-        // Step 6
-        let response = match stream.send() {
-            Ok(r) => r,
-            Err(_) => return error,
-        };
-
-        // Step 7: We don't perform a CORS check here
-        // FYI, fn allow_cross_origin_request() performs the CORS check
-        match response.status.class() {
-            Success => {}
-            _ => return error,
-        }
-        cors_response.headers = response.headers.clone();
-        // Substeps 1-3 (parsing rules: https://fetch.spec.whatwg.org/#http-new-header-syntax)
-        let methods_substep4 = [self.method.clone()];
-        let mut methods = match response.headers.get() {
-            Some(&AccessControlAllowMethods(ref v)) => &**v,
-            _ => return error,
-        };
-        let headers = match response.headers.get() {
-            Some(&AccessControlAllowHeaders(ref h)) => h,
-            _ => return error,
-        };
-        // Substep 4
-        if methods.is_empty() && preflight.mode == RequestMode::ForcedPreflight {
-            methods = &methods_substep4;
-        }
-        // Substep 5
-        if !is_simple_method(&self.method) && !methods.iter().any(|m| m == &self.method) {
-            return error;
-        }
-        // Substep 6
-        for h in self.headers.iter() {
-            if is_simple_header(&h) {
-                continue;
-            }
-            if !headers.iter().any(|ref h2| h.name().eq_ignore_ascii_case(h2)) {
-                return error;
-            }
-        }
-        // Substeps 7-8
-        let max_age = match response.headers.get() {
-            Some(&AccessControlMaxAge(num)) => num,
-            None => 0,
-        };
-        // Substep 9: Impose restrictions on max-age, if any (unimplemented)
-        // Substeps 10-12: Add a cache (partially implemented, XXXManishearth)
-        // This cache should come from the user agent, creating a new one here to check
-        // for compile time errors
-        let cache = &mut CORSCache(vec![]);
-        for m in methods {
-            let cache_match = cache.match_method_and_update(self, m, max_age);
-            if !cache_match {
-                cache.insert(CORSCacheEntry::new(self.origin.clone(),
-                                                 self.destination.clone(),
-                                                 max_age,
-                                                 false,
-                                                 HeaderOrMethod::MethodData(m.clone())));
-            }
-        }
-        // Substeps 13-14
-        for h in response.headers.iter() {
-            let cache_match = cache.match_header_and_update(self, h.name(), max_age);
-            if !cache_match {
-                cache.insert(CORSCacheEntry::new(self.origin.clone(),
-                                                 self.destination.clone(),
-                                                 max_age,
-                                                 false,
-                                                 HeaderOrMethod::HeaderData(h.to_string())));
-            }
-        }
-        // Substep 15
-        cors_response
-    }
-}
-
-
-pub struct CORSResponse {
-    pub network_error: bool,
-    pub headers: Headers,
-}
-
-impl CORSResponse {
-    fn new() -> CORSResponse {
-        CORSResponse {
-            network_error: false,
-            headers: Headers::new(),
-        }
-    }
-
-    fn new_error() -> CORSResponse {
-        CORSResponse {
-            network_error: true,
-            headers: Headers::new(),
-        }
-    }
-}
-
-// CORS Cache stuff
-
-/// A CORS cache object. Anchor it somewhere to the user agent.
-#[derive(Clone)]
-pub struct CORSCache(Vec<CORSCacheEntry>);
-
-/// Union type for CORS cache entries
-/// Each entry might pertain to a header or method
-#[derive(Clone)]
-pub enum HeaderOrMethod {
-    HeaderData(String),
-    MethodData(Method),
-}
-
-impl HeaderOrMethod {
-    fn match_header(&self, header_name: &str) -> bool {
-        match *self {
-            HeaderOrMethod::HeaderData(ref s) => (&**s).eq_ignore_ascii_case(header_name),
-            _ => false,
-        }
-    }
-
-    fn match_method(&self, method: &Method) -> bool {
-        match *self {
-            HeaderOrMethod::MethodData(ref m) => m == method,
-            _ => false,
-        }
-    }
-}
-
-// An entry in the CORS cache
-#[derive(Clone)]
-pub struct CORSCacheEntry {
-    pub origin: Url,
-    pub url: Url,
-    pub max_age: u32,
-    pub credentials: bool,
-    pub header_or_method: HeaderOrMethod,
-    created: Timespec,
-}
-
-impl CORSCacheEntry {
-    fn new(origin: Url,
-           url: Url,
-           max_age: u32,
-           credentials: bool,
-           header_or_method: HeaderOrMethod)
-           -> CORSCacheEntry {
-        CORSCacheEntry {
-            origin: origin,
-            url: url,
-            max_age: max_age,
-            credentials: credentials,
-            header_or_method: header_or_method,
-            created: time::now().to_timespec(),
-        }
-    }
-}
-
-impl CORSCache {
-    /// https://fetch.spec.whatwg.org/#concept-cache-clear
-    #[allow(dead_code)]
-    fn clear(&mut self, request: &CORSRequest) {
-        let CORSCache(buf) = self.clone();
-        let new_buf: Vec<CORSCacheEntry> =
-            buf.into_iter()
-               .filter(|e| e.origin == request.origin && request.destination == e.url)
-               .collect();
-        *self = CORSCache(new_buf);
-    }
-
-    // Remove old entries
-    fn cleanup(&mut self) {
-        let CORSCache(buf) = self.clone();
-        let now = time::now().to_timespec();
-        let new_buf: Vec<CORSCacheEntry> = buf.into_iter()
-                                              .filter(|e| now.sec > e.created.sec + e.max_age as i64)
-                                              .collect();
-        *self = CORSCache(new_buf);
-    }
-
-    /// https://fetch.spec.whatwg.org/#concept-cache-match-header
-    fn find_entry_by_header<'a>(&'a mut self,
-                                request: &CORSRequest,
-                                header_name: &str)
-                                -> Option<&'a mut CORSCacheEntry> {
-        self.cleanup();
-        // Credentials are not yet implemented here
-        self.0.iter_mut().find(|e| {
-            e.origin.scheme() == request.origin.scheme() &&
-            e.origin.host_str() == request.origin.host_str() &&
-            e.origin.port() == request.origin.port() &&
-            e.url == request.destination &&
-            e.header_or_method.match_header(header_name)
-        })
-    }
-
-    fn match_header(&mut self, request: &CORSRequest, header_name: &str) -> bool {
-        self.find_entry_by_header(request, header_name).is_some()
-    }
-
-    fn match_header_and_update(&mut self,
-                               request: &CORSRequest,
-                               header_name: &str,
-                               new_max_age: u32)
-                               -> bool {
-        self.find_entry_by_header(request, header_name).map(|e| e.max_age = new_max_age).is_some()
-    }
-
-    fn find_entry_by_method<'a>(&'a mut self,
-                                request: &CORSRequest,
-                                method: &Method)
-                                -> Option<&'a mut CORSCacheEntry> {
-        // we can take the method from CORSRequest itself
-        self.cleanup();
-        // Credentials are not yet implemented here
-        self.0.iter_mut().find(|e| {
-            e.origin.scheme() == request.origin.scheme() &&
-            e.origin.host_str() == request.origin.host_str() &&
-            e.origin.port() == request.origin.port() &&
-            e.url == request.destination &&
-            e.header_or_method.match_method(method)
-        })
-    }
-
-    /// https://fetch.spec.whatwg.org/#concept-cache-match-method
-    fn match_method(&mut self, request: &CORSRequest, method: &Method) -> bool {
-        self.find_entry_by_method(request, method).is_some()
-    }
-
-    fn match_method_and_update(&mut self,
-                               request: &CORSRequest,
-                               method: &Method,
-                               new_max_age: u32)
-                               -> bool {
-        self.find_entry_by_method(request, method).map(|e| e.max_age = new_max_age).is_some()
-    }
-
-    fn insert(&mut self, entry: CORSCacheEntry) {
-        self.cleanup();
-        self.0.push(entry);
-    }
-}
-
-fn is_simple_header(h: &HeaderView) -> bool {
-    // FIXME: use h.is::<HeaderType>() when AcceptLanguage and
-    // ContentLanguage headers exist
-    match &*h.name().to_ascii_lowercase() {
-        "accept" | "accept-language" | "content-language" => true,
-        "content-type" => match h.value() {
-            Some(&ContentType(Mime(TopLevel::Text, SubLevel::Plain, _))) |
-            Some(&ContentType(Mime(TopLevel::Application, SubLevel::WwwFormUrlEncoded, _))) |
-            Some(&ContentType(Mime(TopLevel::Multipart, SubLevel::FormData, _))) => true,
-
-            _ => false,
-
-        },
-        _ => false,
-    }
-}
-
-fn is_simple_method(m: &Method) -> bool {
-    match *m {
-        Method::Get | Method::Head | Method::Post => true,
-        _ => false,
-    }
-}
-
-/// Perform a CORS check on a header list and CORS request
-/// https://fetch.spec.whatwg.org/#cors-check
-pub fn allow_cross_origin_request(req: &CORSRequest, headers: &Headers) -> bool {
-    match headers.get::<AccessControlAllowOrigin>() {
-        Some(&AccessControlAllowOrigin::Any) => true, // Not always true, depends on credentials mode
-        Some(&AccessControlAllowOrigin::Value(ref url)) => req.origin.as_str() == *url,
-        Some(&AccessControlAllowOrigin::Null) |
-        None => false,
-    }
-}

components/script/dom/htmllinkelement.rs

@@ -227,7 +227,7 @@ impl HTMLLinkElement {
                     sender: action_sender,
                 };
                 ROUTER.add_route(action_receiver.to_opaque(), box move |message| {
-                    listener.notify(message.to().unwrap());
+                    listener.notify_action(message.to().unwrap());
                 });
 
                 if self.parser_inserted.get() {

components/script/dom/htmlmediaelement.rs

@@ -484,7 +484,7 @@ impl HTMLMediaElement {
                 sender: action_sender,
             };
             ROUTER.add_route(action_receiver.to_opaque(), box move |message| {
-                listener.notify(message.to().unwrap());
+                listener.notify_action(message.to().unwrap());
             });
 
             // FIXME: we're supposed to block the load event much earlier than now

components/script/dom/htmlscriptelement.rs

@@ -314,7 +314,7 @@ impl HTMLScriptElement {
                     sender: action_sender,
                 };
                 ROUTER.add_route(action_receiver.to_opaque(), box move |message| {
-                    listener.notify(message.to().unwrap());
+                    listener.notify_action(message.to().unwrap());
                 });
 
                 doc.load_async(LoadType::Script(url), response_target);

components/script/dom/xmlhttprequest.rs

@@ -2,8 +2,6 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use cors::CORSResponse;
-use cors::{AsyncCORSResponseListener, CORSRequest, RequestMode, allow_cross_origin_request};
 use document_loader::DocumentLoader;
 use dom::bindings::cell::DOMRefCell;
 use dom::bindings::codegen::Bindings::BlobBinding::BlobMethods;
@@ -35,27 +33,28 @@ use encoding::label::encoding_from_whatwg_label;
 use encoding::types::{DecoderTrap, EncoderTrap, Encoding, EncodingRef};
 use euclid::length::Length;
 use hyper::header::Headers;
-use hyper::header::{Accept, ContentLength, ContentType, qitem};
+use hyper::header::{ContentLength, ContentType};
 use hyper::http::RawStatus;
 use hyper::method::Method;
-use hyper::mime::{self, Mime};
+use hyper::mime::{self, Mime, Attr as MimeAttr, Value as MimeValue};
 use ipc_channel::ipc;
 use ipc_channel::router::ROUTER;
 use js::jsapi::JS_ClearPendingException;
 use js::jsapi::{JSContext, JS_ParseJSON, RootedValue};
 use js::jsval::{JSVal, NullValue, UndefinedValue};
 use msg::constellation_msg::{PipelineId, ReferrerPolicy};
-use net_traits::CoreResourceMsg::Load;
+use net_traits::CoreResourceMsg::Fetch;
+use net_traits::request::{CredentialsMode, Destination, RequestInit, RequestMode};
 use net_traits::trim_http_whitespace;
-use net_traits::{AsyncResponseListener, AsyncResponseTarget, Metadata, NetworkError, RequestSource};
+use net_traits::{CoreResourceThread, LoadOrigin};
-use net_traits::{LoadConsumer, LoadContext, LoadData, ResourceCORSData, CoreResourceThread, LoadOrigin};
+use net_traits::{FetchResponseListener, Metadata, NetworkError, RequestSource};
 use network_listener::{NetworkListener, PreInvoke};
 use parse::html::{ParseContext, parse_html};
 use parse::xml::{self, parse_xml};
 use script_runtime::ScriptChan;
 use std::ascii::AsciiExt;
 use std::borrow::ToOwned;
-use std::cell::{Cell, RefCell};
+use std::cell::Cell;
 use std::default::Default;
 use std::str;
 use std::sync::{Arc, Mutex};
@@ -82,7 +81,6 @@ pub struct GenerationId(u32);
 struct XHRContext {
     xhr: TrustedXHRAddress,
     gen_id: GenerationId,
-    cors_request: Option<CORSRequest>,
     buf: DOMRefCell<Vec<u8>>,
     sync_status: DOMRefCell<Option<ErrorResult>>,
 }
@@ -142,7 +140,6 @@ pub struct XMLHttpRequest {
     request_body_len: Cell<usize>,
     sync: Cell<bool>,
     upload_complete: Cell<bool>,
-    upload_events: Cell<bool>,
     send_flag: Cell<bool>,
 
     timeout_cancel: DOMRefCell<Option<OneshotTimerHandle>>,
@@ -187,7 +184,6 @@ impl XMLHttpRequest {
             request_body_len: Cell::new(0),
             sync: Cell::new(false),
             upload_complete: Cell::new(false),
-            upload_events: Cell::new(false),
             send_flag: Cell::new(false),
 
             timeout_cancel: DOMRefCell::new(None),
@@ -216,75 +212,40 @@ impl XMLHttpRequest {
         }
     }
 
-    fn check_cors(context: Arc<Mutex<XHRContext>>,
-                  load_data: LoadData,
-                  req: CORSRequest,
-                  script_chan: Box<ScriptChan + Send>,
-                  core_resource_thread: CoreResourceThread) {
-        struct CORSContext {
-            xhr: Arc<Mutex<XHRContext>>,
-            load_data: RefCell<Option<LoadData>>,
-            req: CORSRequest,
-            script_chan: Box<ScriptChan + Send>,
-            core_resource_thread: CoreResourceThread,
-        }
-
-        impl AsyncCORSResponseListener for CORSContext {
-            fn response_available(&self, response: CORSResponse) {
-                if response.network_error {
-                    let mut context = self.xhr.lock().unwrap();
-                    let xhr = context.xhr.root();
-                    xhr.process_partial_response(XHRProgress::Errored(context.gen_id, Error::Network));
-                    *context.sync_status.borrow_mut() = Some(Err(Error::Network));
-                    return;
-                }
-
-                let mut load_data = self.load_data.borrow_mut().take().unwrap();
-                load_data.cors = Some(ResourceCORSData {
-                    preflight: self.req.preflight_flag,
-                    origin: self.req.origin.clone()
-                });
-
-                XMLHttpRequest::initiate_async_xhr(self.xhr.clone(), self.script_chan.clone(),
-                                                   self.core_resource_thread.clone(), load_data);
-            }
-        }
-
-        let cors_context = CORSContext {
-            xhr: context,
-            load_data: RefCell::new(Some(load_data)),
-            req: req.clone(),
-            script_chan: script_chan.clone(),
-            core_resource_thread: core_resource_thread,
-        };
-
-        req.http_fetch_async(box cors_context, script_chan);
-    }
-
     fn initiate_async_xhr(context: Arc<Mutex<XHRContext>>,
                           script_chan: Box<ScriptChan + Send>,
                           core_resource_thread: CoreResourceThread,
-                          load_data: LoadData) {
+                          init: RequestInit) {
-        impl AsyncResponseListener for XHRContext {
+        impl FetchResponseListener for XHRContext {
-            fn headers_available(&mut self, metadata: Result<Metadata, NetworkError>) {
+                fn process_request_body(&mut self) {
-                let xhr = self.xhr.root();
+                    // todo
-                let rv = xhr.process_headers_available(self.cors_request.clone(),
+                }
-                                                       self.gen_id,
+                fn process_request_eof(&mut self) {
-                                                       metadata);
+                    // todo
-                if rv.is_err() {
+                }
+                fn process_response(&mut self, metadata: Result<Metadata, NetworkError>) {
+                    let xhr = self.xhr.root();
+                    let rv = xhr.process_headers_available(self.gen_id,
+                                                           metadata);
+                    if rv.is_err() {
+                        *self.sync_status.borrow_mut() = Some(rv);
+                    }
+                }
+                fn process_response_chunk(&mut self, mut chunk: Vec<u8>) {
+                    self.buf.borrow_mut().append(&mut chunk);
+                    self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone());
+                }
+                fn process_response_eof(&mut self, response: Result<(), NetworkError>) {
+                    let rv = match response {
+                        Ok(()) => {
+                            self.xhr.root().process_response_complete(self.gen_id, Ok(()))
+                        }
+                        Err(e) => {
+                            self.xhr.root().process_response_complete(self.gen_id, Err(e))
+                        }
+                    };
                     *self.sync_status.borrow_mut() = Some(rv);
                 }
-            }
-
-            fn data_available(&mut self, payload: Vec<u8>) {
-                self.buf.borrow_mut().extend_from_slice(&payload);
-                self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone());
-            }
-
-            fn response_complete(&mut self, status: Result<(), NetworkError>) {
-                let rv = self.xhr.root().process_response_complete(self.gen_id, status);
-                *self.sync_status.borrow_mut() = Some(rv);
-            }
         }
 
         impl PreInvoke for XHRContext {
@@ -298,13 +259,10 @@ impl XMLHttpRequest {
             context: context,
             script_chan: script_chan,
         };
-        let response_target = AsyncResponseTarget {
-            sender: action_sender,
-        };
         ROUTER.add_route(action_receiver.to_opaque(), box move |message| {
-            listener.notify(message.to().unwrap());
+            listener.notify_fetch(message.to().unwrap());
         });
-        core_resource_thread.send(Load(load_data, LoadConsumer::Listener(response_target), None)).unwrap();
+        core_resource_thread.send(Fetch(init, action_sender)).unwrap();
     }
 }
 
@@ -563,12 +521,15 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
             Method::Get | Method::Head => None,
             _ => data
         };
-        // Step 4
+        // Step 4 (first half)
         let extracted = data.as_ref().map(|d| d.extract());
+
         self.request_body_len.set(extracted.as_ref().map_or(0, |e| e.0.len()));
 
+        // todo preserved headers?
+
         // Step 6
-        self.upload_events.set(false);
+        self.upload_complete.set(false);
         // Step 7
         self.upload_complete.set(match extracted {
             None => true,
@@ -580,11 +541,6 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
 
         // Step 9
         if !self.sync.get() {
-            let event_target = self.upload.upcast::<EventTarget>();
-            if event_target.has_handlers() {
-                self.upload_events.set(true);
-            }
-
             // If one of the event handlers below aborts the fetch by calling
             // abort or open we will need the current generation id to detect it.
             // Substep 1
@@ -604,67 +560,108 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
         }
 
         // Step 5
-        let global = self.global();
+        //TODO - set referrer_policy/referrer_url in request
+        let has_handlers = self.upload.upcast::<EventTarget>().has_handlers();
+        let credentials_mode = if self.with_credentials.get() {
+            CredentialsMode::Include
+        } else {
+            CredentialsMode::CredentialsSameOrigin
+        };
+        let use_url_credentials = if let Some(ref url) = *self.request_url.borrow() {
+            !url.username().is_empty() || url.password().is_some()
+        } else {
+            unreachable!()
+        };
 
-        let mut load_data =
+        let bypass_cross_origin_check = {
-            LoadData::new(LoadContext::Browsing,
+            // We want to be able to do cross-origin requests in browser.html.
-                          self.request_url.borrow().clone().unwrap(),
+            // If the XHR happens in a top level window and the mozbrowser
-                          self);
+            // preference is enabled, we allow bypassing the CORS check.
+            // This is a temporary measure until we figure out Servo privilege
+            // story. See https://github.com/servo/servo/issues/9582
+            if let GlobalRoot::Window(win) = self.global() {
+                let is_root_pipeline = win.parent_info().is_none();
+                let is_mozbrowser_enabled = mozbrowser_enabled();
+                is_root_pipeline && is_mozbrowser_enabled
+            } else {
+                false
+            }
+        };
 
-        if load_data.url.origin().ne(&global.r().get_url().origin()) {
+        let mut request = RequestInit {
-            load_data.credentials_flag = self.WithCredentials();
+            method: self.request_method.borrow().clone(),
+            url: self.request_url.borrow().clone().unwrap(),
+            headers: (*self.request_headers.borrow()).clone(),
+            unsafe_request: true,
+            same_origin_data: true,
+            // XXXManishearth figure out how to avoid this clone
+            body: extracted.as_ref().map(|e| e.0.clone()),
+            // XXXManishearth actually "subresource", but it doesn't exist
+            // https://github.com/whatwg/xhr/issues/71
+            destination: Destination::None,
+            synchronous: self.sync.get(),
+            mode: RequestMode::CORSMode,
+            use_cors_preflight: has_handlers,
+            credentials_mode: credentials_mode,
+            use_url_credentials: use_url_credentials,
+            origin: self.global().r().get_url(),
+            referer_url: self.referrer_url.clone(),
+            referrer_policy: self.referrer_policy.clone(),
+        };
+
+        if bypass_cross_origin_check {
+            request.mode = RequestMode::Navigate;
         }
-        load_data.data = extracted.as_ref().map(|e| e.0.clone());
 
-        // XHR spec differs from http, and says UTF-8 should be in capitals,
+        // step 4 (second half)
-        // instead of "utf-8", which is what Hyper defaults to. So not
-        // using content types provided by Hyper.
-        let n = "content-type";
         match extracted {
-            Some((_, Some(ref content_type))) =>
+            Some((_, ref content_type)) => {
-                load_data.headers.set_raw(n.to_owned(), vec![content_type.bytes().collect()]),
+                // this should handle Document bodies too, not just BodyInit
+                let encoding = if let Some(BodyInit::String(_)) = data {
+                    // XHR spec differs from http, and says UTF-8 should be in capitals,
+                    // instead of "utf-8", which is what Hyper defaults to. So not
+                    // using content types provided by Hyper.
+                    Some(MimeValue::Ext("UTF-8".to_string()))
+                } else {
+                    None
+                };
+
+                let mut content_type_set = false;
+                if let Some(ref ct) = *content_type {
+                    if !request.headers.has::<ContentType>() {
+                        request.headers.set_raw("content-type", vec![ct.bytes().collect()]);
+                        content_type_set = true;
+                    }
+                }
+
+                if !content_type_set {
+                    let ct = request.headers.get::<ContentType>().map(|x| x.clone());
+                    if let Some(mut ct) = ct {
+                        if let Some(encoding) = encoding {
+                            for param in &mut (ct.0).2 {
+                                if param.0 == MimeAttr::Charset {
+                                    if !param.0.as_str().eq_ignore_ascii_case(encoding.as_str()) {
+                                        *param = (MimeAttr::Charset, encoding.clone());
+                                    }
+                                }
+                            }
+                        }
+                        // remove instead of mutate in place
+                        // https://github.com/hyperium/hyper/issues/821
+                        request.headers.remove_raw("content-type");
+                        request.headers.set(ct);
+                    }
+                }
+
+            }
             _ => (),
         }
 
-        load_data.preserved_headers = (*self.request_headers.borrow()).clone();
+        debug!("request.headers = {:?}", request.headers);
-
-        if !load_data.preserved_headers.has::<Accept>() {
-            let mime = Mime(mime::TopLevel::Star, mime::SubLevel::Star, vec![]);
-            load_data.preserved_headers.set(Accept(vec![qitem(mime)]));
-        }
-
-        load_data.method = (*self.request_method.borrow()).clone();
-
-        // CORS stuff
-        let global = self.global();
-        let referer_url = self.global().r().get_url();
-        let mode = if self.upload_events.get() {
-            RequestMode::ForcedPreflight
-        } else {
-            RequestMode::CORS
-        };
-        let mut combined_headers = load_data.headers.clone();
-        combined_headers.extend(load_data.preserved_headers.iter());
-        let cors_request = CORSRequest::maybe_new(referer_url.clone(),
-                                                  load_data.url.clone(),
-                                                  mode,
-                                                  load_data.method.clone(),
-                                                  combined_headers,
-                                                  true);
-        match cors_request {
-            Ok(None) => {
-                let bytes = referer_url[..Position::AfterPath].as_bytes().to_vec();
-                self.request_headers.borrow_mut().set_raw("Referer".to_owned(), vec![bytes]);
-            },
-            Ok(Some(ref req)) => self.insert_trusted_header("origin".to_owned(),
-                                                            req.origin.to_string()),
-            _ => {}
-        }
-
-        debug!("request_headers = {:?}", *self.request_headers.borrow());
 
         self.fetch_time.set(time::now().to_timespec().sec);
-        let rv = self.fetch(load_data, cors_request, global.r());
+
+        let rv = self.fetch(request, self.global().r());
         // Step 10
         if self.sync.get() {
             return rv;
@@ -879,7 +876,7 @@ impl XMLHttpRequest {
         event.fire(self.upcast());
     }
 
-    fn process_headers_available(&self, cors_request: Option<CORSRequest>,
+    fn process_headers_available(&self,
                                  gen_id: GenerationId, metadata: Result<Metadata, NetworkError>)
                                  -> Result<(), Error> {
         let metadata = match metadata {
@@ -890,35 +887,6 @@ impl XMLHttpRequest {
             },
         };
 
-        let bypass_cross_origin_check = {
-            // We want to be able to do cross-origin requests in browser.html.
-            // If the XHR happens in a top level window and the mozbrowser
-            // preference is enabled, we allow bypassing the CORS check.
-            // This is a temporary measure until we figure out Servo privilege
-            // story. See https://github.com/servo/servo/issues/9582
-            if let GlobalRoot::Window(win) = self.global() {
-                let is_root_pipeline = win.parent_info().is_none();
-                let is_mozbrowser_enabled = mozbrowser_enabled();
-                is_root_pipeline && is_mozbrowser_enabled
-            } else {
-                false
-            }
-        };
-
-        if !bypass_cross_origin_check {
-            if let Some(ref req) = cors_request {
-                match metadata.headers {
-                    Some(ref h) if allow_cross_origin_request(req, h) => {},
-                    _ => {
-                        self.process_partial_response(XHRProgress::Errored(gen_id, Error::Network));
-                        return Err(Error::Network);
-                    }
-                }
-            }
-        } else {
-            debug!("Bypassing cross origin check");
-        }
-
         *self.response_url.borrow_mut() = metadata.final_url[..Position::AfterQuery].to_owned();
 
         // XXXManishearth Clear cache entries in case of a network error
@@ -1318,25 +1286,12 @@ impl XMLHttpRequest {
     }
 
     fn fetch(&self,
-              load_data: LoadData,
+              init: RequestInit,
-              cors_request: Result<Option<CORSRequest>, ()>,
               global: GlobalRef) -> ErrorResult {
-        let cors_request = match cors_request {
-            Err(_) => {
-                // Happens in case of unsupported cross-origin URI schemes.
-                // Supported schemes are http, https, data, and about.
-                self.process_partial_response(XHRProgress::Errored(
-                    self.generation_id.get(), Error::Network));
-                return Err(Error::Network);
-            }
-            Ok(req) => req,
-        };
-
         let xhr = Trusted::new(self);
 
         let context = Arc::new(Mutex::new(XHRContext {
             xhr: xhr,
-            cors_request: cors_request.clone(),
             gen_id: self.generation_id.get(),
             buf: DOMRefCell::new(vec!()),
             sync_status: DOMRefCell::new(None),
@@ -1350,13 +1305,8 @@ impl XMLHttpRequest {
         };
 
         let core_resource_thread = global.core_resource_thread();
-        if let Some(req) = cors_request {
+        XMLHttpRequest::initiate_async_xhr(context.clone(), script_chan,
-            XMLHttpRequest::check_cors(context.clone(), load_data, req.clone(),
+                                           core_resource_thread, init);
-                                       script_chan.clone(), core_resource_thread);
-        } else {
-            XMLHttpRequest::initiate_async_xhr(context.clone(), script_chan,
-                                               core_resource_thread, load_data);
-        }
 
         if let Some(script_port) = script_port {
             loop {

components/script/lib.rs

@@ -79,7 +79,6 @@ extern crate style;
 extern crate time;
 #[cfg(any(target_os = "macos", target_os = "linux"))]
 extern crate tinyfiledialogs;
-extern crate unicase;
 extern crate url;
 #[macro_use]
 extern crate util;
@@ -91,7 +90,6 @@ extern crate xml5ever;
 mod blob_url_store;
 pub mod bluetooth_blacklist;
 pub mod clipboard_provider;
-pub mod cors;
 mod devtools;
 pub mod document_loader;
 #[macro_use]

components/script/network_listener.rs

@@ -2,7 +2,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use net_traits::{AsyncResponseListener, ResponseAction};
+use net_traits::{Action, AsyncResponseListener, FetchResponseListener};
+use net_traits::{FetchResponseMsg, ResponseAction};
 use script_runtime::ScriptThreadEventCategory::NetworkEvent;
 use script_runtime::{CommonScriptMsg, ScriptChan};
 use script_thread::Runnable;
@@ -10,13 +11,13 @@ use std::sync::{Arc, Mutex};
 
 /// An off-thread sink for async network event runnables. All such events are forwarded to
 /// a target thread, where they are invoked on the provided context object.
-pub struct NetworkListener<T: AsyncResponseListener + PreInvoke + Send + 'static> {
+pub struct NetworkListener<Listener: PreInvoke + Send + 'static> {
-    pub context: Arc<Mutex<T>>,
+    pub context: Arc<Mutex<Listener>>,
     pub script_chan: Box<ScriptChan + Send>,
 }
 
-impl<T: AsyncResponseListener + PreInvoke + Send + 'static> NetworkListener<T> {
+impl<Listener: PreInvoke + Send + 'static> NetworkListener<Listener> {
-    pub fn notify(&self, action: ResponseAction) {
+    pub fn notify<A: Action<Listener> + Send + 'static>(&self, action: A) {
         if let Err(err) = self.script_chan.send(CommonScriptMsg::RunnableMsg(NetworkEvent, box ListenerRunnable {
             context: self.context.clone(),
             action: action,
@@ -26,6 +27,20 @@ impl<T: AsyncResponseListener + PreInvoke + Send + 'static> NetworkListener<T> {
     }
 }
 
+// helps type inference
+impl<Listener: AsyncResponseListener + PreInvoke + Send + 'static> NetworkListener<Listener> {
+    pub fn notify_action(&self, action: ResponseAction) {
+        self.notify(action);
+    }
+}
+
+// helps type inference
+impl<Listener: FetchResponseListener + PreInvoke + Send + 'static> NetworkListener<Listener> {
+    pub fn notify_fetch(&self, action: FetchResponseMsg) {
+        self.notify(action);
+    }
+}
+
 /// A gating mechanism that runs before invoking the runnable on the target thread.
 /// If the `should_invoke` method returns false, the runnable is discarded without
 /// being invoked.
@@ -36,13 +51,13 @@ pub trait PreInvoke {
 }
 
 /// A runnable for moving the async network events between threads.
-struct ListenerRunnable<T: AsyncResponseListener + PreInvoke + Send> {
+struct ListenerRunnable<A: Action<Listener> + Send + 'static, Listener: PreInvoke + Send> {
-    context: Arc<Mutex<T>>,
+    context: Arc<Mutex<Listener>>,
-    action: ResponseAction,
+    action: A,
 }
 
-impl<T: AsyncResponseListener + PreInvoke + Send> Runnable for ListenerRunnable<T> {
+impl<A: Action<Listener> + Send + 'static, Listener: PreInvoke + Send> Runnable for ListenerRunnable<A, Listener> {
-    fn handler(self: Box<ListenerRunnable<T>>) {
+    fn handler(self: Box<ListenerRunnable<A, Listener>>) {
         let this = *self;
         let mut context = this.context.lock().unwrap();
         if context.should_invoke() {

components/script/script_thread.rs

@@ -1962,7 +1962,7 @@ impl ScriptThread {
             script_chan: self.chan.clone(),
         };
         ROUTER.add_route(action_receiver.to_opaque(), box move |message| {
-            listener.notify(message.to().unwrap());
+            listener.notify_action(message.to().unwrap());
         });
         let response_target = AsyncResponseTarget {
             sender: action_sender,

components/servo/Cargo.lock

@@ -1916,7 +1916,6 @@ dependencies = [
  "style 0.0.1",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "tinyfiledialogs 0.1.0 (git+https://github.com/jdm/tinyfiledialogs)",
- "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "util 0.0.1",
  "uuid 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",

ports/cef/Cargo.lock

@@ -1774,7 +1774,6 @@ dependencies = [
  "style 0.0.1",
  "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
  "tinyfiledialogs 0.1.0 (git+https://github.com/jdm/tinyfiledialogs)",
- "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "util 0.0.1",
  "uuid 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",

tests/unit/net/fetch.rs

@@ -14,8 +14,9 @@ use hyper::server::{Request as HyperRequest, Response as HyperResponse};
 use hyper::status::StatusCode;
 use hyper::uri::RequestUri;
 use net::fetch::cors_cache::CORSCache;
-use net::fetch::methods::{fetch, fetch_async, fetch_with_cors_cache};
+use net::fetch::methods::{FetchContext, fetch, fetch_with_cors_cache};
-use net_traits::AsyncFetchListener;
+use net::http_loader::HttpState;
+use net_traits::FetchTaskTarget;
 use net_traits::request::{Origin, RedirectMode, Referer, Request, RequestMode};
 use net_traits::response::{CacheState, Response, ResponseBody, ResponseType};
 use std::fs::File;
@@ -24,6 +25,7 @@ use std::rc::Rc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::mpsc::{Sender, channel};
 use std::sync::{Arc, Mutex};
+use std::thread;
 use time::{self, Duration};
 use unicase::UniCase;
 use url::{Origin as UrlOrigin, Url};
@@ -35,11 +37,32 @@ struct FetchResponseCollector {
     sender: Sender<Response>,
 }
 
-impl AsyncFetchListener for FetchResponseCollector {
+fn new_fetch_context() -> FetchContext {
-    fn response_available(&self, response: Response) {
+    FetchContext {
-        let _ = self.sender.send(response);
+        state: HttpState::new(),
+        user_agent: "Such Browser. Very Layout. Wow.".into(),
     }
 }
+impl FetchTaskTarget for FetchResponseCollector {
+    fn process_request_body(&mut self, _: &Request) {}
+    fn process_request_eof(&mut self, _: &Request) {}
+    fn process_response(&mut self, _: &Response) {}
+    fn process_response_chunk(&mut self, _: Vec<u8>) {}
+    /// Fired when the response is fully fetched
+    fn process_response_eof(&mut self, response: &Response) {
+        self.sender.send(response.clone());
+    }
+}
+
+fn fetch_async(request: Request, target: Box<FetchTaskTarget + Send>) {
+    thread::spawn(move || {
+        fetch(Rc::new(request), &mut Some(target), new_fetch_context());
+    });
+}
+
+fn fetch_sync(request: Request) -> Response {
+    fetch(Rc::new(request), &mut None, new_fetch_context())
+}
 
 fn make_server<H: Handler + 'static>(handler: H) -> (Listening, Url) {
     // this is a Listening server because of handle_threads()
@@ -61,10 +84,8 @@ fn test_fetch_response_is_not_network_error() {
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     if fetch_response.is_network_error() {
@@ -82,10 +103,8 @@ fn test_fetch_response_body_matches_const_message() {
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     assert!(!fetch_response.is_network_error());
@@ -104,10 +123,8 @@ fn test_fetch_aboutblank() {
     let url = Url::parse("about:blank").unwrap();
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     assert!(!fetch_response.is_network_error());
     assert!(*fetch_response.body.lock().unwrap() == ResponseBody::Done(vec![]));
 }
@@ -119,7 +136,7 @@ fn test_fetch_data() {
     let request = Request::new(url, Some(origin), false);
     request.same_origin_data.set(true);
     let expected_resp_body = "<p>Servo</p>".to_owned();
-    let fetch_response = fetch(Rc::new(request));
+    let fetch_response = fetch_sync(request);
 
     assert!(!fetch_response.is_network_error());
     assert_eq!(fetch_response.headers.len(), 1);
@@ -148,7 +165,7 @@ fn test_fetch_file() {
     let request = Request::new(url, Some(origin), false);
     request.same_origin_data.set(true);
 
-    let fetch_response = fetch(Rc::new(request));
+    let fetch_response = fetch_sync(request);
     assert!(!fetch_response.is_network_error());
     assert_eq!(fetch_response.headers.len(), 1);
     let content_type: &ContentType = fetch_response.headers.get().unwrap();
@@ -187,12 +204,10 @@ fn test_cors_preflight_fetch() {
 
     let origin = Origin::Origin(UrlOrigin::new_opaque());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
     request.use_cors_preflight = true;
     request.mode = RequestMode::CORSMode;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     assert!(!fetch_response.is_network_error());
@@ -226,14 +241,14 @@ fn test_cors_preflight_cache_fetch() {
 
     let origin = Origin::Origin(UrlOrigin::new_opaque());
     let mut request = Request::new(url.clone(), Some(origin.clone()), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
     request.use_cors_preflight = true;
     request.mode = RequestMode::CORSMode;
     let wrapped_request0 = Rc::new(request.clone());
     let wrapped_request1 = Rc::new(request);
 
-    let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache);
+    let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache, &mut None, new_fetch_context());
-    let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache);
+    let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache, &mut None, new_fetch_context());
     let _ = server.close();
 
     assert!(!fetch_response0.is_network_error() && !fetch_response1.is_network_error());
@@ -276,12 +291,10 @@ fn test_cors_preflight_fetch_network_error() {
     let origin = Origin::Origin(UrlOrigin::new_opaque());
     let mut request = Request::new(url, Some(origin), false);
     *request.method.borrow_mut() = Method::Extension("CHICKEN".to_owned());
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
     request.use_cors_preflight = true;
     request.mode = RequestMode::CORSMode;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     assert!(fetch_response.is_network_error());
@@ -301,10 +314,8 @@ fn test_fetch_response_is_basic_filtered() {
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     assert!(!fetch_response.is_network_error());
@@ -348,11 +359,9 @@ fn test_fetch_response_is_cors_filtered() {
     // an origin mis-match will stop it from defaulting to a basic filtered response
     let origin = Origin::Origin(UrlOrigin::new_opaque());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
     request.mode = RequestMode::CORSMode;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     assert!(!fetch_response.is_network_error());
@@ -382,10 +391,8 @@ fn test_fetch_response_is_opaque_filtered() {
     // an origin mis-match will fall through to an Opaque filtered response
     let origin = Origin::Origin(UrlOrigin::new_opaque());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     assert!(!fetch_response.is_network_error());
@@ -431,11 +438,9 @@ fn test_fetch_response_is_opaque_redirect_filtered() {
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
     request.redirect_mode.set(RedirectMode::Manual);
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
 
     assert!(!fetch_response.is_network_error());
@@ -467,13 +472,12 @@ fn test_fetch_with_local_urls_only() {
     let do_fetch = |url: Url| {
         let origin = Origin::Origin(url.origin());
         let mut request = Request::new(url, Some(origin), false);
-        request.referer = Referer::NoReferer;
+        *request.referer.borrow_mut() = Referer::NoReferer;
 
         // Set the flag.
         request.local_urls_only = true;
 
-        let wrapped_request = Rc::new(request);
+        fetch_sync(request)
-        fetch(wrapped_request)
     };
 
     let local_url = Url::parse("about:blank").unwrap();
@@ -509,10 +513,8 @@ fn setup_server_and_fetch(message: &'static [u8], redirect_cap: u32) -> Response
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
-    let wrapped_request = Rc::new(request);
+    let fetch_response = fetch_sync(request);
-
-    let fetch_response = fetch(wrapped_request);
     let _ = server.close();
     fetch_response
 }
@@ -594,11 +596,10 @@ fn test_fetch_redirect_updates_method_runner(tx: Sender<bool>, status_code: Stat
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
     *request.method.borrow_mut() = method;
-    let wrapped_request = Rc::new(request);
 
-    let _ = fetch(wrapped_request);
+    let _ = fetch_sync(request);
     let _ = server.close();
 }
 
@@ -670,7 +671,7 @@ fn test_fetch_async_returns_complete_response() {
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
 
     let (tx, rx) = channel();
     let listener = Box::new(FetchResponseCollector {
@@ -695,7 +696,7 @@ fn test_opaque_filtered_fetch_async_returns_complete_response() {
     // an origin mis-match will fall through to an Opaque filtered response
     let origin = Origin::Origin(UrlOrigin::new_opaque());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
 
     let (tx, rx) = channel();
     let listener = Box::new(FetchResponseCollector {
@@ -735,7 +736,7 @@ fn test_opaque_redirect_filtered_fetch_async_returns_complete_response() {
 
     let origin = Origin::Origin(url.origin());
     let mut request = Request::new(url, Some(origin), false);
-    request.referer = Referer::NoReferer;
+    *request.referer.borrow_mut() = Referer::NoReferer;
     request.redirect_mode.set(RedirectMode::Manual);
 
     let (tx, rx) = channel();

tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini

@@ -1,5 +0,0 @@
-[send-authentication-basic-setrequestheader-existing-session.htm]
-  type: testharness
-  [XMLHttpRequest: send() - "Basic" authenticated request using setRequestHeader() when there is an existing session]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini

@@ -1,5 +0,0 @@
-[send-authentication-basic.htm]
-  type: testharness
-  [XMLHttpRequest: send() - "Basic" authenticated requests with user name and password passed to open()]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini

@@ -0,0 +1,8 @@
+[send-conditional.htm]
+  type: testharness
+  [XMLHttpRequest: send() - conditional requests (tag)]
+    expected: FAIL
+
+  [XMLHttpRequest: send() - conditional requests (date)]
+    expected: FAIL
+

tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini

@@ -15,9 +15,12 @@
   [charset given but wrong, fix it (known MIME, bogus charset)]
     expected: FAIL
 
-  [charset given but wrong, fix it (known MIME, actual charset)]
-    expected: FAIL
-
   [If multiple charset parameters are given, all should be rewritten]
     expected: FAIL
 
+  [Correct text/plain MIME with charset]
+    expected: FAIL
+
+  [charset given but wrong, fix it (known MIME, actual charset)]
+    expected: FAIL
+

tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini

@@ -1,5 +0,0 @@
-[send-entity-body-get-head.htm]
-  type: testharness
-  [XMLHttpRequest: send() - non-empty data argument and GET/HEAD (HEAD)]
-    expected: FAIL
-

tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini

@@ -0,0 +1,14 @@
+[send-redirect-to-cors.htm]
+  type: testharness
+  [XMLHttpRequest: send() - Redirect to CORS-enabled resource (301)]
+    expected: FAIL
+
+  [XMLHttpRequest: send() - Redirect to CORS-enabled resource (302)]
+    expected: FAIL
+
+  [XMLHttpRequest: send() - Redirect to CORS-enabled resource (303)]
+    expected: FAIL
+
+  [XMLHttpRequest: send() - Redirect to CORS-enabled resource (307)]
+    expected: FAIL
+

tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini

@@ -1,14 +0,0 @@
-[send-redirect-to-non-cors.htm]
-  type: testharness
-  [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (301)]
-    expected: FAIL
-
-  [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (302)]
-    expected: FAIL
-
-  [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (303)]
-    expected: FAIL
-
-  [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (307)]
-    expected: FAIL
-

tests/wpt/metadata/cors/allow-headers.htm.ini

@@ -33,6 +33,3 @@
   [Allow origin: [tab\]http://web-platform.test:8000]
     expected: FAIL
 
-  [Disallow origin: http://web-platform.test:8000/]
-    expected: FAIL
-

tests/wpt/metadata/cors/origin.htm.ini

@@ -33,12 +33,3 @@
   [Allow origin: [tab\]http://web-platform.test:8000]
     expected: FAIL
 
-  [Disallow origin: http://web-platform.test:8000/]
-    expected: FAIL
-
-  [Disallow multiple headers (, *)]
-    expected: FAIL
-
-  [Disallow multiple headers (*, )]
-    expected: FAIL
-

tests/wpt/metadata/cors/preflight-cache.htm.ini

@@ -1,17 +1,8 @@
 [preflight-cache.htm]
   type: testharness
-  [Test preflight]
-    expected: FAIL
-
   [preflight for x-print should be cached]
     expected: FAIL
 
-  [age = 0, should not be cached]
-    expected: FAIL
-
-  [age = -1, should not be cached]
-    expected: FAIL
-
   [preflight first request, second from cache, wait, third should preflight again]
     expected: FAIL
 

tests/wpt/metadata/cors/redirect-origin.htm.ini

@@ -62,24 +62,12 @@
   [local (*) to remote (http://web-platform.test:8000), expect origin=http://web-platform.test:8000]
     expected: FAIL
 
-  [local (*) to remote (null), expect to fail]
-    expected: FAIL
-
-  [local (*) to remote (none), expect to fail]
-    expected: FAIL
-
   [local (http://web-platform.test:8000) to remote (*), expect origin=http://web-platform.test:8000]
     expected: FAIL
 
   [local (http://web-platform.test:8000) to remote (http://web-platform.test:8000), expect origin=http://web-platform.test:8000]
     expected: FAIL
 
-  [local (http://web-platform.test:8000) to remote (null), expect to fail]
-    expected: FAIL
-
-  [local (http://web-platform.test:8000) to remote (none), expect to fail]
-    expected: FAIL
-
   [local (null) to remote (*), expect origin=http://web-platform.test:8000]
     expected: FAIL
 
@@ -110,27 +98,3 @@
   [remote (http://web-platform.test:8000) to remote2 (null), expect origin=null]
     expected: FAIL
 
-  [remote (http://www1.web-platform.test:8000) to remote (*), expect to fail]
-    expected: FAIL
-
-  [remote (null) to remote2 (*), expect to fail]
-    expected: FAIL
-
-  [remote (none) to remote2 (*), expect to fail]
-    expected: FAIL
-
-  [remote (none) to remote2 (*), expect to fail]
-    expected: FAIL
-
-  [remote (null) to remote (*), expect to fail]
-    expected: FAIL
-
-  [remote (none) to remote (*), expect to fail]
-    expected: FAIL
-
-  [remote (none) to local (*), expect to fail]
-    expected: FAIL
-
-  [remote (null) to local (*), expect to fail]
-    expected: FAIL
-

tests/wpt/metadata/cors/redirect-preflight-2.htm.ini

@@ -1,8 +0,0 @@
-[redirect-preflight-2.htm]
-  type: testharness
-  [Same-origin custom-header request, redirect to cross-origin succeeds after doing a preflight]
-    expected: FAIL
-
-  [Same-origin custom-header request, redirect to cross-origin fails after doing a non-successful preflight]
-    expected: FAIL
-

tests/wpt/metadata/cors/response-headers.htm.ini

@@ -9,6 +9,3 @@
   [getResponse: don't expose x-nonexposed]
     expected: FAIL
 
-  [getAllResponseHeaders: don't expose x-nonexposed]
-    expected: FAIL
-

tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js

@@ -1,3 +1,5 @@
+const NUM_TESTS = 128;
+
 function encode(n) {
   if (n === 0x20) {
     return "\x2B";
@@ -13,27 +15,34 @@ function encode(n) {
   return "%" + (s.length === 2 ? s : '0' + s);
 }
 
-function do_test(n) {
-  async_test(function() {
-    var x = new XMLHttpRequest();
-    x.onload = this.step_func_done(function(e) {
-      assert_equals(x.response, "a=" + encode(n))
-    });
-    x.onerror = this.unreached_func();
-    x.open("POST", "resources/content.py");
-    var usp = new URLSearchParams();
-    usp.append("a", String.fromCharCode(n));
-    x.send(usp)
-  }, "XMLHttpRequest.send(URLSearchParams) (" + n + ")");
-}
-
 function run_test() {
-  var i = 0;
+  var tests = [];
-  add_result_callback(function() {
+  var overall_test = async_test("Overall fetch with URLSearchParams");
-    if (++i === 128) {
+  for (var i = 0; i < NUM_TESTS; i++) {
-      return;
+    // Multiple subtests so that failures can be fine-grained
+    tests[i] = async_test("XMLHttpRequest.send(URLSearchParams) (" + i + ")");
+  }
+
+  // We use a single XHR since this test tends to time out
+  // with 128 consecutive fetches when run in parallel
+  // with many other WPT tests.
+  var x = new XMLHttpRequest();
+  x.onload = overall_test.step_func(function() {
+    var response_split = x.response.split("&");
+    overall_test.done();
+    for (var i = 0; i < NUM_TESTS; i++) {
+      tests[i].step(function() {
+        assert_equals(response_split[i], "a" + i + "="+encode(i));
+        tests[i].done();
+      });
     }
-    do_test(i);
   });
-  do_test(i);
+  x.onerror = overall_test.unreached_func();
+
+  x.open("POST", "resources/content.py");
+  var usp = new URLSearchParams();
+  for (var i = 0; i < NUM_TESTS; i++) {
+    usp.append("a" + i, String.fromCharCode(i));
+  }
+  x.send(usp)
 }