Partial implementation of Main Fetch step, including appropiate updates to tests

components/net/fetch/cors_cache.rs

@@ -14,7 +14,7 @@ use std::ascii::AsciiExt;
 use std::sync::mpsc::{Sender, Receiver, channel};
 use time;
 use time::{now, Timespec};
-use url::Url;
+use url::{Origin, Url};
 
 /// Union type for CORS cache entries
 ///
@@ -44,7 +44,7 @@ impl HeaderOrMethod {
 /// An entry in the CORS cache
 #[derive(Clone)]
 pub struct CORSCacheEntry {
-    pub origin: Url,
+    pub origin: Origin,
     pub url: Url,
     pub max_age: u32,
     pub credentials: bool,
@@ -53,7 +53,7 @@ pub struct CORSCacheEntry {
 }
 
 impl CORSCacheEntry {
-    fn new(origin: Url, url: Url, max_age: u32, credentials: bool,
+    fn new(origin: Origin, url: Url, max_age: u32, credentials: bool,
             header_or_method: HeaderOrMethod) -> CORSCacheEntry {
         CORSCacheEntry {
             origin: origin,
@@ -68,7 +68,7 @@ impl CORSCacheEntry {
 
 /// Properties of Request required to cache match.
 pub struct CacheRequestDetails {
-    pub origin: Url,
+    pub origin: Origin,
     pub destination: Url,
     pub credentials: bool
 }
@@ -109,9 +109,7 @@ pub trait CORSCache {
 pub struct BasicCORSCache(Vec<CORSCacheEntry>);
 
 fn match_headers(cors_cache: &CORSCacheEntry, cors_req: &CacheRequestDetails) -> bool {
-    cors_cache.origin.scheme == cors_req.origin.scheme &&
+    cors_cache.origin == cors_req.origin &&
-        cors_cache.origin.host() == cors_req.origin.host() &&
-        cors_cache.origin.port() == cors_req.origin.port() &&
         cors_cache.url == cors_req.destination &&
         cors_cache.credentials == cors_req.credentials
 }

components/net/fetch/methods.rs

@@ -22,6 +22,7 @@ use net_traits::response::{Response, ResponseBody, ResponseType};
 use net_traits::{AsyncFetchListener, Metadata};
 use resource_thread::CancellationListener;
 use std::ascii::AsciiExt;
+use std::cell::RefCell;
 use std::io::Read;
 use std::rc::Rc;
 use std::str::FromStr;
@@ -90,14 +91,177 @@ pub fn fetch(request: Rc<Request>, cors_flag: bool) -> Response {
     // TODO: Figure out what a Priority object is
     // Step 3
     // Step 4
-    main_fetch(request, cors_flag)
+    main_fetch(request, cors_flag, false)
 }
 
 /// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch)
-fn main_fetch(request: Rc<Request>, _cors_flag: bool) -> Response {
+fn main_fetch(request: Rc<Request>, cors_flag: bool, recursive_flag: bool) -> Response {
     // TODO: Implement main fetch spec
-    let response = basic_fetch(request);
+
-    response
+    // Step 1
+    let mut response = None;
+
+    // Step 2
+    if request.local_urls_only {
+        match &*request.current_url().scheme {
+            "about" | "blob" | "data" | "filesystem" => response = Some(Response::network_error()),
+            _ => { }
+        };
+    }
+
+    // Step 3
+    // TODO be able to execute report CSP
+
+    // Step 4
+    // TODO this step, based off of http_loader.rs
+
+    // Step 5
+    // TODO this step
+
+    // Step 6
+    if request.referer != Referer::NoReferer {
+        // TODO be able to invoke "determine request's referer"
+    }
+
+    // Step 7
+    // TODO this step
+
+    // Step 8
+    if !request.synchronous && !recursive_flag {
+        // TODO run the remaining steps in parallel
+    }
+
+    // Step 9
+    let mut response = if response.is_none() {
+
+        let current_url = request.current_url();
+        let origin_match = request.origin == current_url.origin();
+
+        if (!cors_flag && origin_match) ||
+            (current_url.scheme == "data" && request.same_origin_data.get()) ||
+            current_url.scheme == "about" ||
+            request.mode == RequestMode::Navigate {
+
+            basic_fetch(request.clone())
+
+        } else if request.mode == RequestMode::SameOrigin {
+            Response::network_error()
+
+        } else if request.mode == RequestMode::NoCORS {
+            request.response_tainting.set(ResponseTainting::Opaque);
+            basic_fetch(request.clone())
+
+        } else if current_url.scheme != "http" && current_url.scheme != "https" {
+            Response::network_error()
+
+        } else if request.use_cors_preflight ||
+            (request.unsafe_request &&
+            (!is_simple_method(&request.method.borrow()) ||
+            request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) {
+
+            request.response_tainting.set(ResponseTainting::CORSTainting);
+            request.redirect_mode.set(RedirectMode::Error);
+            let response = http_fetch(request.clone(), BasicCORSCache::new(), true, true, false);
+            if Response::is_network_error(&response) {
+                // TODO clear cache entries using request
+            }
+            response
+
+        } else {
+            request.response_tainting.set(ResponseTainting::CORSTainting);
+            http_fetch(request.clone(), BasicCORSCache::new(), true, false, false)
+        }
+    } else {
+        response.unwrap()
+    };
+
+    // Step 10
+    if recursive_flag {
+        return response;
+    }
+
+    // Step 11
+    // no need to check if response is a network error, since the type would not be `Default`
+    let mut response = if response.response_type == ResponseType::Default {
+        let old_response = Rc::new(response);
+        let response_type = match request.response_tainting.get() {
+            ResponseTainting::Basic => ResponseType::Basic,
+            ResponseTainting::CORSTainting => ResponseType::CORS,
+            ResponseTainting::Opaque => ResponseType::Opaque,
+        };
+        Response::to_filtered(old_response, response_type)
+    } else {
+        response
+    };
+
+    // Step 12
+    let mut internal_response = if Response::is_network_error(&response) {
+        Rc::new(Response::network_error())
+    } else {
+        response.internal_response.clone().unwrap()
+    };
+
+    // Step 13
+    // TODO this step
+
+    // Step 14
+    if !Response::is_network_error(&response) && (is_null_body_status(&internal_response.status) ||
+        match *request.method.borrow() {
+            Method::Head | Method::Connect => true,
+            _ => false })
+        {
+        // when the Fetch implementation does asynchronous retrieval of the body,
+        // we will need to make sure nothing tries to write to the body at this point
+        *internal_response.body.borrow_mut() = ResponseBody::Empty;
+    }
+
+    // Step 15
+    // TODO be able to compare response integrity against request integrity metadata
+    // if !Response::is_network_error(&response) {
+
+    //     // Substep 1
+    //     // TODO wait for response
+
+    //     // Substep 2
+    //     if response.termination_reason.is_none() {
+    //         response = Response::network_error();
+    //         internal_response = Response::network_error();
+    //     }
+    // }
+
+    // Step 16
+    if request.synchronous {
+        // TODO wait for internal_response
+        return response;
+    }
+
+    // Step 17
+    if request.body.is_some() && match &*request.current_url().scheme {
+        "http" | "https" => true,
+        _ => false }
+        {
+        // TODO queue a fetch task on request to process end-of-file
+    }
+
+    // Step 18
+    // TODO this step
+
+    match *internal_response.body.borrow() {
+        // Step 20
+        ResponseBody::Empty => {
+            // Substep 1
+            // Substep 2
+        },
+
+        // Step 19
+        _ => {
+            // Substep 1
+            // Substep 2
+        }
+    };
+
+    // TODO remove this line when asynchronous fetches are supported
+    return response;
 }
 
 /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch)
@@ -180,7 +344,7 @@ fn http_fetch(request: Rc<Request>,
             if (res.response_type == ResponseType::Opaque &&
                 request.mode != RequestMode::NoCORS) ||
                (res.response_type == ResponseType::OpaqueRedirect &&
-                request.redirect_mode != RedirectMode::Manual) ||
+                request.redirect_mode.get() != RedirectMode::Manual) ||
                res.response_type == ResponseType::Error {
                 return Response::network_error();
             }
@@ -205,9 +369,7 @@ fn http_fetch(request: Rc<Request>,
             let mut method_mismatch = false;
             let mut header_mismatch = false;
 
-            // FIXME: Once Url::Origin is available, rewrite origin to
+            let origin = request.origin.clone();
-            // take an Origin instead of a Url
-            let origin = request.origin.clone().unwrap_or(Url::parse("").unwrap());
             let url = request.current_url();
             let credentials = request.credentials_mode == CredentialsMode::Include;
             let method_cache_match = cache.match_method(CacheRequestDetails {
@@ -217,7 +379,7 @@ fn http_fetch(request: Rc<Request>,
             }, request.method.borrow().clone());
 
             method_mismatch = !method_cache_match && (!is_simple_method(&request.method.borrow()) ||
-                request.mode == RequestMode::ForcedPreflightMode);
+                request.use_cors_preflight);
             header_mismatch = request.headers.borrow().iter().any(|view|
                 !cache.match_header(CacheRequestDetails {
                     origin: origin.clone(),
@@ -226,12 +388,13 @@ fn http_fetch(request: Rc<Request>,
                 }, view.name()) && !is_simple_header(&view)
             );
 
+            // Sub-substep 1
             if method_mismatch || header_mismatch {
                 let preflight_result = preflight_fetch(request.clone());
+                // Sub-substep 2
                 if preflight_result.response_type == ResponseType::Error {
                     return Response::network_error();
                 }
-                response = Some(Rc::new(preflight_result));
             }
         }
 
@@ -242,7 +405,7 @@ fn http_fetch(request: Rc<Request>,
         let credentials = match request.credentials_mode {
             CredentialsMode::Include => true,
             CredentialsMode::CredentialsSameOrigin if (!cors_flag ||
-                request.response_tainting == ResponseTainting::Opaque)
+                request.response_tainting.get() == ResponseTainting::Opaque)
                 => true,
             _ => false
         };
@@ -271,7 +434,7 @@ fn http_fetch(request: Rc<Request>,
         StatusCode::TemporaryRedirect | StatusCode::PermanentRedirect => {
 
             // Step 1
-            if request.redirect_mode == RedirectMode::Error {
+            if request.redirect_mode.get() == RedirectMode::Error {
                 return Response::network_error();
             }
 
@@ -308,11 +471,11 @@ fn http_fetch(request: Rc<Request>,
             // Step 9
             request.same_origin_data.set(false);
 
-            match request.redirect_mode {
+            match request.redirect_mode.get() {
 
                 // Step 10
                 RedirectMode::Manual => {
-                    response = Rc::new(Response::to_filtered(actual_response, ResponseType::Opaque));
+                    response = Rc::new(Response::to_filtered(actual_response, ResponseType::OpaqueRedirect));
                 }
 
                 // Step 11
@@ -350,7 +513,7 @@ fn http_fetch(request: Rc<Request>,
                     request.url_list.borrow_mut().push(location_url);
 
                     // Substep 6
-                    return main_fetch(request.clone(), cors_flag);
+                    return main_fetch(request.clone(), cors_flag, true);
                 }
                 RedirectMode::Error => { panic!("RedirectMode is Error after step 8") }
             }
@@ -418,7 +581,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
 
     // Step 1
     let http_request = if request_has_no_window &&
-        request.redirect_mode != RedirectMode::Follow {
+        request.redirect_mode.get() != RedirectMode::Follow {
         request.clone()
     } else {
         Rc::new((*request).clone())
@@ -457,9 +620,7 @@ fn http_network_or_cache_fetch(request: Rc<Request>,
     // Step 7
     if http_request.omit_origin_header == false {
         // TODO update this when https://github.com/hyperium/hyper/pull/691 is finished
-        if let Some(ref _origin) = http_request.origin {
+        // http_request.headers.borrow_mut().set_raw("origin", origin);
-            // http_request.headers.borrow_mut().set_raw("origin", origin);
-        }
     }
 
     // Step 8
@@ -628,7 +789,7 @@ fn http_network_fetch(request: Rc<Request>,
 
             let mut body = vec![];
             res.response.read_to_end(&mut body);
-            response.body = ResponseBody::Done(body);
+            *response.body.borrow_mut() = ResponseBody::Done(body);
         },
         Err(e) =>
             response.termination_reason = Some(TerminationReason::Fatal)
@@ -677,6 +838,7 @@ fn http_network_fetch(request: Rc<Request>,
         // Substep 3
         // Substep 4
 
+    // TODO these steps
     // Step 10
         // Substep 1
         // Substep 2
@@ -761,3 +923,14 @@ fn response_needs_revalidation(response: &Response) -> bool {
 //     // TODO this function
 
 // }
+
+fn is_null_body_status(status: &Option<StatusCode>) -> bool {
+    match *status {
+        Some(status) => match status {
+            StatusCode::SwitchingProtocols | StatusCode::NoContent |
+                StatusCode::ResetContent | StatusCode::NotModified => true,
+            _ => false
+        },
+        _ => false
+    }
+}

components/net/fetch/response.rs

@@ -2,7 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use hyper::header::Headers;
+use hyper::header::{AccessControlExposeHeaders, Headers};
 use hyper::status::StatusCode;
 use net_traits::response::{CacheState, HttpsState, Response, ResponseBody, ResponseType};
 use std::ascii::AsciiExt;
@@ -25,7 +25,7 @@ impl ResponseMethods for Response {
             url_list: RefCell::new(Vec::new()),
             status: Some(StatusCode::Ok),
             headers: Headers::new(),
-            body: ResponseBody::Empty,
+            body: RefCell::new(ResponseBody::Empty),
             cache_state: CacheState::None,
             https_state: HttpsState::None,
             internal_response: None
@@ -46,6 +46,7 @@ impl ResponseMethods for Response {
         let old_headers = old_response.headers.clone();
         let mut response = (*old_response).clone();
         response.internal_response = Some(old_response);
+        response.response_type = filter_type;
 
         match filter_type {
 
@@ -59,26 +60,40 @@ impl ResponseMethods for Response {
                     }
                 }).collect();
                 response.headers = headers;
-                response.response_type = filter_type;
             },
 
             ResponseType::CORS => {
+
+                let access = old_headers.get::<AccessControlExposeHeaders>();
+                let allowed_headers = access.as_ref().map(|v| &v[..]).unwrap_or(&[]);
+
                 let headers = old_headers.iter().filter(|header| {
                     match &*header.name().to_ascii_lowercase() {
                         "cache-control" | "content-language" |
-                        "content-type" | "expires" | "last-modified" | "Pragma" => false,
+                        "content-type" | "expires" | "last-modified" | "Pragma" => true,
-                        // XXXManishearth handle Access-Control-Expose-Headers
+                        "set-cookie" | "set-cookie2" => false,
-                        _ => true
+                        header => {
+                            let result =
+                                allowed_headers.iter().find(|h| *header == *h.to_ascii_lowercase());
+                            result.is_some()
+                        }
                     }
                 }).collect();
                 response.headers = headers;
-                response.response_type = filter_type;
             },
 
-            ResponseType::Opaque | ResponseType::OpaqueRedirect => {
+            ResponseType::Opaque => {
+                response.url_list = RefCell::new(vec![]);
+                response.url = None;
                 response.headers = Headers::new();
                 response.status = None;
-                response.body = ResponseBody::Empty;
+                response.body = RefCell::new(ResponseBody::Empty);
+            },
+
+            ResponseType::OpaqueRedirect => {
+                response.headers = Headers::new();
+                response.status = None;
+                response.body = RefCell::new(ResponseBody::Empty);
             }
         }
 

components/net_traits/request.rs

@@ -5,7 +5,7 @@
 use hyper::header::Headers;
 use hyper::method::Method;
 use std::cell::{Cell, RefCell};
-use url::Url;
+use url::{Origin, Url};
 
 /// A [request context](https://fetch.spec.whatwg.org/#concept-request-context)
 #[derive(Copy, Clone, PartialEq)]
@@ -37,10 +37,10 @@ pub enum Referer {
 /// A [request mode](https://fetch.spec.whatwg.org/#concept-request-mode)
 #[derive(Copy, Clone, PartialEq)]
 pub enum RequestMode {
+    Navigate,
     SameOrigin,
     NoCORS,
-    CORSMode,
+    CORSMode
-    ForcedPreflightMode
 }
 
 /// Request [credentials mode](https://fetch.spec.whatwg.org/#concept-request-credentials-mode)
@@ -82,6 +82,7 @@ pub enum ResponseTainting {
 #[derive(Clone)]
 pub struct Request {
     pub method: RefCell<Method>,
+    pub local_urls_only: bool,
     // Use the last method on url_list to act as spec url field
     pub url_list: RefCell<Vec<Url>>,
     pub headers: RefCell<Headers>,
@@ -94,26 +95,28 @@ pub struct Request {
     pub skip_service_worker: Cell<bool>,
     pub context: Context,
     pub context_frame_type: ContextFrameType,
-    pub origin: Option<Url>, // FIXME: Use Url::Origin
+    pub origin: Origin,
     pub force_origin_header: bool,
     pub omit_origin_header: bool,
     pub same_origin_data: Cell<bool>,
     pub referer: Referer,
     pub authentication: bool,
-    pub sync: bool,
+    pub synchronous: bool,
+    pub use_cors_preflight: bool,
     pub mode: RequestMode,
     pub credentials_mode: CredentialsMode,
     pub use_url_credentials: bool,
     pub cache_mode: Cell<CacheMode>,
-    pub redirect_mode: RedirectMode,
+    pub redirect_mode: Cell<RedirectMode>,
     pub redirect_count: Cell<u32>,
-    pub response_tainting: ResponseTainting
+    pub response_tainting: Cell<ResponseTainting>
 }
 
 impl Request {
-    pub fn new(url: Url, context: Context, is_service_worker_global_scope: bool) -> Request {
+    pub fn new(url: Url, context: Context, origin: Origin, is_service_worker_global_scope: bool) -> Request {
          Request {
             method: RefCell::new(Method::Get),
+            local_urls_only: false,
             url_list: RefCell::new(vec![url]),
             headers: RefCell::new(Headers::new()),
             unsafe_request: false,
@@ -123,20 +126,21 @@ impl Request {
             skip_service_worker: Cell::new(false),
             context: context,
             context_frame_type: ContextFrameType::ContextNone,
-            origin: None,
+            origin: origin,
             force_origin_header: false,
             omit_origin_header: false,
             same_origin_data: Cell::new(false),
             referer: Referer::Client,
             authentication: false,
-            sync: false,
+            synchronous: false,
+            use_cors_preflight: false,
             mode: RequestMode::NoCORS,
             credentials_mode: CredentialsMode::Omit,
             use_url_credentials: false,
             cache_mode: Cell::new(CacheMode::Default),
-            redirect_mode: RedirectMode::Follow,
+            redirect_mode: Cell::new(RedirectMode::Follow),
             redirect_count: Cell::new(0),
-            response_tainting: ResponseTainting::Basic,
+            response_tainting: Cell::new(ResponseTainting::Basic)
         }
     }
 

components/net_traits/response.rs

@@ -11,7 +11,7 @@ use std::rc::Rc;
 use url::Url;
 
 /// [Response type](https://fetch.spec.whatwg.org/#concept-response-type)
-#[derive(Clone, PartialEq, Copy)]
+#[derive(Clone, PartialEq, Copy, Debug)]
 pub enum ResponseType {
     Basic,
     CORS,
@@ -71,7 +71,7 @@ pub struct Response {
     /// `None` can be considered a StatusCode of `0`.
     pub status: Option<StatusCode>,
     pub headers: Headers,
-    pub body: ResponseBody,
+    pub body: RefCell<ResponseBody>,
     pub cache_state: CacheState,
     pub https_state: HttpsState,
     /// [Internal response](https://fetch.spec.whatwg.org/#concept-internal-response), only used if the Response
@@ -88,7 +88,7 @@ impl Response {
             url_list: RefCell::new(vec![]),
             status: None,
             headers: Headers::new(),
-            body: ResponseBody::Empty,
+            body: RefCell::new(ResponseBody::Empty),
             cache_state: CacheState::None,
             https_state: HttpsState::None,
             internal_response: None

tests/unit/net/fetch.rs

@@ -9,7 +9,7 @@ use hyper::status::StatusCode;
 use hyper::uri::RequestUri;
 use net::fetch::methods::fetch;
 use net_traits::request::{Context, Referer, Request};
-use net_traits::response::{Response, ResponseBody};
+use net_traits::response::{Response, ResponseBody, ResponseType};
 use std::rc::Rc;
 use url::Url;
 
@@ -35,7 +35,8 @@ fn test_fetch_response_is_not_network_error() {
     };
     let (mut server, url) = make_server(handler);
 
-    let mut request = Request::new(url, Context::Fetch, false);
+    let origin = url.origin();
+    let mut request = Request::new(url, Context::Fetch, origin, false);
     request.referer = Referer::NoReferer;
     let wrapped_request = Rc::new(request);
 
@@ -56,16 +57,20 @@ fn test_fetch_response_body_matches_const_message() {
     };
     let (mut server, url) = make_server(handler);
 
-    let mut request = Request::new(url, Context::Fetch, false);
+    let origin = url.origin();
+    let mut request = Request::new(url, Context::Fetch, origin, false);
     request.referer = Referer::NoReferer;
     let wrapped_request = Rc::new(request);
 
     let fetch_response = fetch(wrapped_request, false);
     let _ = server.close();
 
-    match fetch_response.body {
+    assert!(!Response::is_network_error(&fetch_response));
-        ResponseBody::Done(body) => {
+    assert_eq!(fetch_response.response_type, ResponseType::Basic);
-            assert_eq!(body, MESSAGE);
+
+    match *fetch_response.body.borrow() {
+        ResponseBody::Done(ref body) => {
+            assert_eq!(&**body, MESSAGE);
         },
         _ => panic!()
     };
@@ -94,7 +99,8 @@ fn test_fetch_redirect_count(message: &'static [u8], redirect_cap: u32) -> Respo
 
     let (mut server, url) = make_server(handler);
 
-    let mut request = Request::new(url, Context::Fetch, false);
+    let origin = url.origin();
+    let mut request = Request::new(url, Context::Fetch, origin, false);
     request.referer = Referer::NoReferer;
     let wrapped_request = Rc::new(request);
 
@@ -112,10 +118,12 @@ fn test_fetch_redirect_count_ceiling() {
 
     let fetch_response = test_fetch_redirect_count(MESSAGE, redirect_cap);
 
-    assert_eq!(Response::is_network_error(&fetch_response), false);
+    assert!(!Response::is_network_error(&fetch_response));
-    match fetch_response.body {
+    assert_eq!(fetch_response.response_type, ResponseType::Basic);
-        ResponseBody::Done(body) => {
+
-            assert_eq!(body, MESSAGE);
+    match *fetch_response.body.borrow() {
+        ResponseBody::Done(ref body) => {
+            assert_eq!(&**body, MESSAGE);
         },
         _ => panic!()
     };
@@ -130,9 +138,10 @@ fn test_fetch_redirect_count_failure() {
 
     let fetch_response = test_fetch_redirect_count(MESSAGE, redirect_cap);
 
-    assert_eq!(Response::is_network_error(&fetch_response), true);
+    assert!(Response::is_network_error(&fetch_response));
-    match fetch_response.body {
+
-        ResponseBody::Done(_) => panic!(),
+    match *fetch_response.body.borrow() {
+        ResponseBody::Done(_) | ResponseBody::Receiving(_) => panic!(),
         _ => { }
     };
 }