From b5255f011e66615e9c6c6a461fe200b11d62ccf2 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Thu, 26 May 2016 16:38:22 +0530 Subject: [PATCH 01/12] Revamp Fetch async handling to use a fetch target and done channels --- components/net/fetch/methods.rs | 174 ++++++++++++++++++------------ components/net_traits/lib.rs | 23 +++- components/net_traits/request.rs | 2 +- components/net_traits/response.rs | 12 --- tests/unit/net/fetch.rs | 53 +++++---- 5 files changed, 158 insertions(+), 106 deletions(-) diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 16d64a86a7c..92b9aec1b00 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -16,9 +16,10 @@ use hyper::method::Method; use hyper::mime::{Mime, SubLevel, TopLevel}; use hyper::status::StatusCode; use mime_guess::guess_mime_type; -use net_traits::AsyncFetchListener; -use net_traits::request::{CacheMode, CredentialsMode, Type, Origin, Window}; +use net_traits::FetchTaskTarget; +use net_traits::request::{CacheMode, CredentialsMode}; use net_traits::request::{RedirectMode, Referer, Request, RequestMode, ResponseTainting}; +use net_traits::request::{Type, Origin, Window}; use net_traits::response::{HttpsState, TerminationReason}; use net_traits::response::{Response, ResponseBody, ResponseType}; use resource_thread::CancellationListener; @@ -27,26 +28,23 @@ use std::fs::File; use std::io::Read; use std::iter::FromIterator; use std::rc::Rc; -use std::thread; +use std::sync::mpsc::{channel, Sender, Receiver}; use unicase::UniCase; use url::{Origin as UrlOrigin, Url}; use util::thread::spawn_named; -pub fn fetch_async(request: Request, listener: Box) { - spawn_named(format!("fetch for {:?}", request.current_url_string()), move || { - let request = Rc::new(request); - let fetch_response = fetch(request); - fetch_response.wait_until_done(); - listener.response_available(fetch_response); - }) -} +pub type Target = Option>; + +type DoneChannel = Option<(Sender<()>, Receiver<()>)>; /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch) -pub fn fetch(request: Rc) -> Response { - fetch_with_cors_cache(request, &mut CORSCache::new()) +pub fn fetch(request: Rc, target: Target) -> Response { + fetch_with_cors_cache(request, &mut CORSCache::new(), target) } -pub fn fetch_with_cors_cache(request: Rc, cache: &mut CORSCache) -> Response { +pub fn fetch_with_cors_cache(request: Rc, + cache: &mut CORSCache, + mut target: Target) -> Response { // Step 1 if request.window.get() == Window::Client { // TODO: Set window to request's client object if client is a Window object @@ -105,12 +103,14 @@ pub fn fetch_with_cors_cache(request: Rc, cache: &mut CORSCache) -> Res if request.is_subresource_request() { // TODO: create a fetch record and append it to request's client's fetch group list } + // Step 7 - main_fetch(request, cache, false, false) + main_fetch(request, cache, false, false, &mut target, &mut None) } /// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch) -fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recursive_flag: bool) -> Response { +fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, + recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel) -> Response { // TODO: Implement main fetch spec // Step 1 @@ -128,23 +128,26 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu // TODO be able to execute report CSP // Step 4 - // TODO this step, based off of http_loader.rs + // TODO this step, based off of http_loader.rs (upgrade) // Step 5 - // TODO this step + // TODO this step (CSP port/content blocking) - // Step 6 + // Step 6-7 + // TODO this step (referer policy) + + // Step 8 if request.referer != Referer::NoReferer { // TODO be able to invoke "determine request's referer" } - // Step 7 - // TODO this step + // Step 9 + // TODO this step (HSTS) - // Step 8 + // Step 10 // this step is obsoleted by fetch_async - // Step 9 + // Step 11 let response = match response { Some(response) => response, None => { @@ -160,14 +163,14 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu (current_url.scheme() == "file" && request.same_origin_data.get()) || current_url.scheme() == "about" || request.mode == RequestMode::Navigate { - basic_fetch(request.clone(), cache) + basic_fetch(request.clone(), cache, target, done_chan) } else if request.mode == RequestMode::SameOrigin { Response::network_error() } else if request.mode == RequestMode::NoCORS { request.response_tainting.set(ResponseTainting::Opaque); - basic_fetch(request.clone(), cache) + basic_fetch(request.clone(), cache, target, done_chan) } else if !matches!(current_url.scheme(), "http" | "https") { Response::network_error() @@ -178,7 +181,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) { request.response_tainting.set(ResponseTainting::CORSTainting); request.redirect_mode.set(RedirectMode::Error); - let response = http_fetch(request.clone(), cache, true, true, false); + let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan); if response.is_network_error() { // TODO clear cache entries using request } @@ -186,17 +189,17 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu } else { request.response_tainting.set(ResponseTainting::CORSTainting); - http_fetch(request.clone(), cache, true, false, false) + http_fetch(request.clone(), cache, true, false, false, target, done_chan) } } }; - // Step 10 + // Step 12 if recursive_flag { return response; } - // Step 11 + // Step 13 // no need to check if response is a network error, since the type would not be `Default` let response = if response.response_type == ResponseType::Default { let response_type = match request.response_tainting.get() { @@ -210,7 +213,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu }; { - // Step 12 + // Step 14 let network_error_res = Response::network_error(); let internal_response = if response.is_network_error() { &network_error_res @@ -218,10 +221,10 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu response.actual_response() }; - // Step 13 - // TODO this step + // Step 15 + // TODO this step (CSP/blocking) - // Step 14 + // Step 16 if !response.is_network_error() && (is_null_body_status(&internal_response.status) || match *request.method.borrow() { Method::Head | Method::Connect => true, @@ -233,7 +236,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu *body = ResponseBody::Empty; } - // Step 15 + // Step 17 // TODO be able to compare response integrity against request integrity metadata // if !response.is_network_error() { @@ -248,34 +251,45 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu // } } - // Step 16 + // Step 18 if request.synchronous { - response.actual_response().wait_until_done(); + if !response.is_network_error() { + if let Some(ref ch) = *done_chan { + let _ = ch.1.recv(); + } + } return response; } - // Step 17 + // Step 19 if request.body.borrow().is_some() && matches!(request.current_url().scheme(), "http" | "https") { - // TODO queue a fetch task on request to process end-of-file + if let Some(ref mut target) = *target { + // XXXManishearth: We actually should be calling process_request + // in http_network_fetch. However, we can't yet follow the request + // upload progress, so I'm keeping it here for now and pretending + // the body got sent in one chunk + target.process_request_body(&request); + target.process_request_eof(&request); + } } { - // Step 12 repeated to use internal_response - let network_error_res = Response::network_error(); - let internal_response = if response.is_network_error() { - &network_error_res - } else { - response.actual_response() - }; - - // Step 18 - // TODO this step - - // Step 19 - internal_response.wait_until_done(); - // Step 20 - // TODO this step + if let Some(ref mut target) = *target { + target.process_response(&response); + } + + // Step 21 + if !response.is_network_error() { + if let Some(ref ch) = *done_chan { + let _ = ch.1.recv(); + } + } + + // Step 22 + if let Some(ref mut target) = *target { + target.process_response_eof(&response); + } } // TODO remove this line when only asynchronous fetches are used @@ -283,7 +297,8 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recu } /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch) -fn basic_fetch(request: Rc, cache: &mut CORSCache) -> Response { +fn basic_fetch(request: Rc, cache: &mut CORSCache, + target: &mut Target, done_chan: &mut DoneChannel) -> Response { let url = request.current_url(); match url.scheme() { @@ -295,7 +310,7 @@ fn basic_fetch(request: Rc, cache: &mut CORSCache) -> Response { }, "http" | "https" => { - http_fetch(request.clone(), cache, false, false, false) + http_fetch(request.clone(), cache, false, false, false, target, done_chan) }, "data" => { @@ -350,7 +365,12 @@ fn http_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, cors_preflight_flag: bool, - authentication_fetch_flag: bool) -> Response { + authentication_fetch_flag: bool, + target: &mut Target, + done_chan: &mut DoneChannel) -> Response { + + // This is a new async fetch, reset the channel we are waiting on + *done_chan = None; // Step 1 let mut response: Option = None; @@ -423,7 +443,8 @@ fn http_fetch(request: Rc, }; // Substep 4 - let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag); + let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag, + done_chan); // Substep 5 if cors_flag && cors_check(request.clone(), &fetch_result).is_err() { @@ -450,7 +471,8 @@ fn http_fetch(request: Rc, RedirectMode::Follow => { // set back to default response.return_internal.set(true); - http_redirect_fetch(request, cache, Rc::new(response), cors_flag) + http_redirect_fetch(request, cache, Rc::new(response), + cors_flag, target, done_chan) } } }, @@ -472,7 +494,8 @@ fn http_fetch(request: Rc, } // Step 4 - return http_fetch(request, cache, cors_flag, cors_preflight_flag, true); + return http_fetch(request, cache, cors_flag, cors_preflight_flag, + true, target, done_chan); } // Code 407 @@ -489,7 +512,8 @@ fn http_fetch(request: Rc, // Step 4 return http_fetch(request, cache, cors_flag, cors_preflight_flag, - authentication_fetch_flag); + authentication_fetch_flag, target, + done_chan); } _ => { } @@ -510,7 +534,9 @@ fn http_fetch(request: Rc, fn http_redirect_fetch(request: Rc, cache: &mut CORSCache, response: Rc, - cors_flag: bool) -> Response { + cors_flag: bool, + target: &mut Target, + done_chan: &mut DoneChannel) -> Response { // Step 1 assert_eq!(response.return_internal.get(), true); @@ -584,13 +610,14 @@ fn http_redirect_fetch(request: Rc, request.url_list.borrow_mut().push(location_url); // Step 15 - main_fetch(request, cache, cors_flag, true) + main_fetch(request, cache, cors_flag, true, target, done_chan) } /// [HTTP network or cache fetch](https://fetch.spec.whatwg.org#http-network-or-cache-fetch) fn http_network_or_cache_fetch(request: Rc, credentials_flag: bool, - authentication_fetch_flag: bool) -> Response { + authentication_fetch_flag: bool, + done_chan: &mut DoneChannel) -> Response { // TODO: Implement Window enum for Request let request_has_no_window = true; @@ -753,7 +780,7 @@ fn http_network_or_cache_fetch(request: Rc, // Step 18 if response.is_none() { - response = Some(http_network_fetch(request.clone(), http_request.clone(), credentials_flag)); + response = Some(http_network_fetch(request.clone(), http_request.clone(), credentials_flag, done_chan)); } let response = response.unwrap(); @@ -789,7 +816,8 @@ fn http_network_or_cache_fetch(request: Rc, /// [HTTP network fetch](https://fetch.spec.whatwg.org/#http-network-fetch) fn http_network_fetch(request: Rc, _http_request: Rc, - _credentials_flag: bool) -> Response { + _credentials_flag: bool, + done_chan: &mut DoneChannel) -> Response { // TODO: Implement HTTP network fetch spec // Step 1 @@ -822,7 +850,11 @@ fn http_network_fetch(request: Rc, response.headers = res.response.headers.clone(); let res_body = response.body.clone(); - thread::spawn(move || { + + // We're about to spawn a thread to be waited on here + *done_chan = Some(channel()); + let done_sender = done_chan.as_ref().map(|ch| ch.0.clone()); + spawn_named(format!("fetch worker thread"), move || { *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); loop { @@ -838,6 +870,9 @@ fn http_network_fetch(request: Rc, _ => vec![] }; *res_body.lock().unwrap() = ResponseBody::Done(completed_body); + if let Some(sender) = done_sender { + let _ = sender.send(()); + } break; } } @@ -845,8 +880,9 @@ fn http_network_fetch(request: Rc, } }); }, - Err(_) => - response.termination_reason = Some(TerminationReason::Fatal) + Err(_) => { + response.termination_reason = Some(TerminationReason::Fatal); + } }; // TODO these substeps aren't possible yet @@ -929,7 +965,7 @@ fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache) -> Response // Step 6 let preflight = Rc::new(preflight); - let response = http_network_or_cache_fetch(preflight.clone(), false, false); + let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None); // Step 7 if cors_check(request.clone(), &response).is_ok() && diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index 4af820b0ebd..ed17e52c876 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -154,9 +154,26 @@ pub trait LoadOrigin { fn pipeline_id(&self) -> Option; } -/// Interface for observing the final response for an asynchronous fetch operation. -pub trait AsyncFetchListener { - fn response_available(&self, response: response::Response); +pub trait FetchTaskTarget { + /// https://fetch.spec.whatwg.org/#process-request-body + /// + /// Fired when a chunk of the request body is transmitted + fn process_request_body(&mut self, request: &request::Request); + + /// https://fetch.spec.whatwg.org/#process-request-end-of-file + /// + /// Fired when the entire request finishes being transmitted + fn process_request_eof(&mut self, request: &request::Request); + + /// https://fetch.spec.whatwg.org/#process-response + /// + /// Fired when headers are received + fn process_response(&mut self, response: &response::Response); + + /// https://fetch.spec.whatwg.org/#process-response-end-of-file + /// + /// Fired when the response is fully fetched + fn process_response_eof(&mut self, response: &response::Response); } /// A listener for asynchronous network events. Cancelling the underlying request is unsupported. diff --git a/components/net_traits/request.rs b/components/net_traits/request.rs index 675a78fd788..be7a09e4b25 100644 --- a/components/net_traits/request.rs +++ b/components/net_traits/request.rs @@ -145,7 +145,7 @@ pub struct Request { pub url_list: RefCell>, pub redirect_count: Cell, pub response_tainting: Cell, - pub done: Cell + pub done: Cell, } impl Request { diff --git a/components/net_traits/response.rs b/components/net_traits/response.rs index d107e219988..84826fed2b1 100644 --- a/components/net_traits/response.rs +++ b/components/net_traits/response.rs @@ -132,18 +132,6 @@ impl Response { } } - pub fn wait_until_done(&self) { - match self.response_type { - // since these response types can't hold a body, they should be considered done - ResponseType::Error | ResponseType::Opaque | ResponseType::OpaqueRedirect => {}, - _ => { - while !self.body.lock().unwrap().is_done() && !self.is_network_error() { - // loop until done - } - } - } - } - pub fn actual_response(&self) -> &Response { if self.return_internal.get() && self.internal_response.is_some() { &**self.internal_response.as_ref().unwrap() diff --git a/tests/unit/net/fetch.rs b/tests/unit/net/fetch.rs index 4803eee5178..16dc664ffff 100644 --- a/tests/unit/net/fetch.rs +++ b/tests/unit/net/fetch.rs @@ -14,13 +14,14 @@ use hyper::server::{Request as HyperRequest, Response as HyperResponse}; use hyper::status::StatusCode; use hyper::uri::RequestUri; use net::fetch::cors_cache::CORSCache; -use net::fetch::methods::{fetch, fetch_async, fetch_with_cors_cache}; -use net_traits::AsyncFetchListener; +use net::fetch::methods::{fetch, fetch_with_cors_cache}; +use net_traits::FetchTaskTarget; use net_traits::request::{Origin, RedirectMode, Referer, Request, RequestMode}; use net_traits::response::{CacheState, Response, ResponseBody, ResponseType}; use std::fs::File; use std::io::Read; use std::rc::Rc; +use std::thread; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::{Sender, channel}; use std::sync::{Arc, Mutex}; @@ -35,12 +36,22 @@ struct FetchResponseCollector { sender: Sender, } -impl AsyncFetchListener for FetchResponseCollector { - fn response_available(&self, response: Response) { - let _ = self.sender.send(response); +impl FetchTaskTarget for FetchResponseCollector { + fn process_request_body(&mut self, _: &Request) {} + fn process_request_eof(&mut self, _: &Request) {} + fn process_response(&mut self, _: &Response) {} + /// Fired when the response is fully fetched + fn process_response_eof(&mut self, response: &Response) { + self.sender.send(response.clone()); } } +fn fetch_async(request: Request, target: Box) { + thread::spawn(move || { + fetch(Rc::new(request), Some(target)); + }); +} + fn make_server(handler: H) -> (Listening, Url) { // this is a Listening server because of handle_threads() let server = Server::http("0.0.0.0:0").unwrap().handle_threads(handler, 1).unwrap(); @@ -64,7 +75,7 @@ fn test_fetch_response_is_not_network_error() { request.referer = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); if fetch_response.is_network_error() { @@ -85,7 +96,7 @@ fn test_fetch_response_body_matches_const_message() { request.referer = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -107,7 +118,7 @@ fn test_fetch_aboutblank() { request.referer = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); assert!(!fetch_response.is_network_error()); assert!(*fetch_response.body.lock().unwrap() == ResponseBody::Done(vec![])); } @@ -119,7 +130,7 @@ fn test_fetch_data() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); let expected_resp_body = "

Servo

".to_owned(); - let fetch_response = fetch(Rc::new(request)); + let fetch_response = fetch(Rc::new(request), None); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); @@ -148,7 +159,7 @@ fn test_fetch_file() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); - let fetch_response = fetch(Rc::new(request)); + let fetch_response = fetch(Rc::new(request), None); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); let content_type: &ContentType = fetch_response.headers.get().unwrap(); @@ -192,7 +203,7 @@ fn test_cors_preflight_fetch() { request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -232,8 +243,8 @@ fn test_cors_preflight_cache_fetch() { let wrapped_request0 = Rc::new(request.clone()); let wrapped_request1 = Rc::new(request); - let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache); - let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache); + let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache, None); + let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache, None); let _ = server.close(); assert!(!fetch_response0.is_network_error() && !fetch_response1.is_network_error()); @@ -281,7 +292,7 @@ fn test_cors_preflight_fetch_network_error() { request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); assert!(fetch_response.is_network_error()); @@ -304,7 +315,7 @@ fn test_fetch_response_is_basic_filtered() { request.referer = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -352,7 +363,7 @@ fn test_fetch_response_is_cors_filtered() { request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -385,7 +396,7 @@ fn test_fetch_response_is_opaque_filtered() { request.referer = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -435,7 +446,7 @@ fn test_fetch_response_is_opaque_redirect_filtered() { request.redirect_mode.set(RedirectMode::Manual); let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -473,7 +484,7 @@ fn test_fetch_with_local_urls_only() { request.local_urls_only = true; let wrapped_request = Rc::new(request); - fetch(wrapped_request) + fetch(wrapped_request, None) }; let local_url = Url::parse("about:blank").unwrap(); @@ -512,7 +523,7 @@ fn setup_server_and_fetch(message: &'static [u8], redirect_cap: u32) -> Response request.referer = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request); + let fetch_response = fetch(wrapped_request, None); let _ = server.close(); fetch_response } @@ -598,7 +609,7 @@ fn test_fetch_redirect_updates_method_runner(tx: Sender, status_code: Stat *request.method.borrow_mut() = method; let wrapped_request = Rc::new(request); - let _ = fetch(wrapped_request); + let _ = fetch(wrapped_request, None); let _ = server.close(); } From 2cbc8dee25b42c5b0f3c9e34e008ae03a959d247 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 30 May 2016 17:01:12 +0530 Subject: [PATCH 02/12] Net side of XHR fetch integration --- components/net/fetch/methods.rs | 1 + components/net/resource_thread.rs | 39 ++++++++++++++++++++++++++-- components/net_traits/lib.rs | 42 +++++++++++++++++++++++++++++++ components/net_traits/request.rs | 1 + components/net_traits/response.rs | 30 +++++++++++++++++++--- 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 92b9aec1b00..79469469e87 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -847,6 +847,7 @@ fn http_network_fetch(request: Rc, Ok((mut res, _)) => { response.url = Some(res.response.url.clone()); response.status = Some(res.response.status); + response.raw_status = Some(res.response.status_raw().clone()); response.headers = res.response.headers.clone(); let res_body = response.body.clone(); diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index fecbff3e077..b85532b22f1 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -11,6 +11,7 @@ use cookie; use cookie_storage::CookieStorage; use data_loader; use devtools_traits::DevtoolsControlMsg; +use fetch::methods::fetch; use file_loader; use filemanager_thread::FileManagerThreadFactory; use hsts::HstsList; @@ -23,8 +24,10 @@ use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag}; use net_traits::LoadContext; use net_traits::ProgressMsg::Done; use net_traits::{AsyncResponseTarget, Metadata, ProgressMsg, ResponseAction, CoreResourceThread}; -use net_traits::{CoreResourceMsg, CookieSource, LoadConsumer, LoadData, LoadResponse, ResourceId}; -use net_traits::{NetworkError, WebSocketCommunicate, WebSocketConnectData, ResourceThreads}; +use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, LoadConsumer}; +use net_traits::{LoadData, LoadResponse, NetworkError, ResourceId}; +use net_traits::{WebSocketCommunicate, WebSocketConnectData, ResourceThreads}; +use net_traits::request::{Referer, Request}; use profile_traits::time::ProfilerChan; use rustc_serialize::json; use rustc_serialize::{Decodable, Encodable}; @@ -36,6 +39,7 @@ use std::error::Error; use std::fs::File; use std::io::prelude::*; use std::path::Path; +use std::rc::Rc; use std::sync::mpsc::{Receiver, Sender, channel}; use std::sync::{Arc, RwLock}; use storage_thread::StorageThreadFactory; @@ -193,6 +197,8 @@ impl ResourceChannelManager { match self.from_client.recv().unwrap() { CoreResourceMsg::Load(load_data, consumer, id_sender) => self.resource_manager.load(load_data, consumer, id_sender, control_sender.clone()), + CoreResourceMsg::Fetch(load_data, sender) => + self.resource_manager.fetch(load_data, sender), CoreResourceMsg::WebsocketConnect(connect, connect_data) => self.resource_manager.websocket_connect(connect, connect_data), CoreResourceMsg::SetCookiesForUrl(request, cookie_list, source) => @@ -480,6 +486,35 @@ impl CoreResourceManager { cancel_listener)); } + fn fetch(&self, load_data: LoadData, sender: IpcSender) { + spawn_named(format!("fetch thread for {}", load_data.url), move || { + let mut request = Request::new(load_data.url, + None, false); + // todo handle origin + // todo consider replacing LoadData with a direct mapping + // to a subset of Request + // todo set is_service_worker_global_scope + *request.method.borrow_mut() = load_data.method; + *request.headers.borrow_mut() = load_data.headers; + *request.body.borrow_mut() = load_data.data.clone(); + if let Some(cors) = load_data.cors { + request.use_cors_preflight = cors.preflight; + } + // XXXManishearth: Check origin against pipeline id + request.use_url_credentials = load_data.credentials_flag; + // todo load context / mimesniff in fetch + // todo referrer policy? + if let Some(referer) = load_data.referrer_url { + request.referer = Referer::RefererUrl(referer); + } + // todo worker stuff + + + fetch(Rc::new(request), Some(Box::new(sender))); + }) + + } + fn websocket_connect(&self, connect: WebSocketCommunicate, connect_data: WebSocketConnectData) { diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index ed17e52c876..127c85f13b0 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -112,6 +112,7 @@ pub struct LoadData { pub headers: Headers, #[ignore_heap_size_of = "Defined in hyper"] /// Headers that will apply to the initial request and any redirects + /// Unused in fetch pub preserved_headers: Headers, pub data: Option>, pub cors: Option, @@ -154,6 +155,16 @@ pub trait LoadOrigin { fn pipeline_id(&self) -> Option; } +#[derive(Deserialize, Serialize)] +pub enum FetchResponseMsg { + // todo: should have fields for transmitted/total bytes + ProcessRequestBody, + ProcessRequestEOF, + // todo: send more info about the response (or perhaps the entire Response) + ProcessResponse(Result), + ProcessResponseEOF(Result, NetworkError>), +} + pub trait FetchTaskTarget { /// https://fetch.spec.whatwg.org/#process-request-body /// @@ -176,6 +187,36 @@ pub trait FetchTaskTarget { fn process_response_eof(&mut self, response: &response::Response); } +impl FetchTaskTarget for IpcSender { + fn process_request_body(&mut self, _: &request::Request) { + let _ = self.send(FetchResponseMsg::ProcessRequestBody); + } + + fn process_request_eof(&mut self, _: &request::Request) { + let _ = self.send(FetchResponseMsg::ProcessRequestEOF); + } + + fn process_response(&mut self, response: &response::Response) { + let _ = self.send(FetchResponseMsg::ProcessResponse(response.metadata())); + } + + fn process_response_eof(&mut self, response: &response::Response) { + if response.is_network_error() { + // todo: finer grained errors + let _ = self.send(FetchResponseMsg::ProcessResponse(Err(NetworkError::Internal("Network error".into())))); + } + if let Ok(ref guard) = response.body.lock() { + if let response::ResponseBody::Done(ref vec) = **guard { + let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(vec.clone()))); + return; + } + } + + // If something goes wrong, log it instead of crashing the resource thread + let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Err(NetworkError::Internal("Incomplete body".into())))); + } +} + /// A listener for asynchronous network events. Cancelling the underlying request is unsupported. pub trait AsyncResponseListener { /// The response headers for a request have been received. @@ -348,6 +389,7 @@ pub struct WebSocketConnectData { pub enum CoreResourceMsg { /// Request the data associated with a particular URL Load(LoadData, LoadConsumer, Option>), + Fetch(LoadData, IpcSender), /// Try to make a websocket connection to a URL. WebsocketConnect(WebSocketCommunicate, WebSocketConnectData), /// Store a set of cookies for a given originating URL diff --git a/components/net_traits/request.rs b/components/net_traits/request.rs index be7a09e4b25..0aa5417b3ba 100644 --- a/components/net_traits/request.rs +++ b/components/net_traits/request.rs @@ -43,6 +43,7 @@ pub enum Origin { #[derive(Clone, PartialEq)] pub enum Referer { NoReferer, + /// Default referer if nothing is specified Client, RefererUrl(Url) } diff --git a/components/net_traits/response.rs b/components/net_traits/response.rs index 84826fed2b1..6db2228b4fe 100644 --- a/components/net_traits/response.rs +++ b/components/net_traits/response.rs @@ -4,15 +4,17 @@ //! The [Response](https://fetch.spec.whatwg.org/#responses) object //! resulting from a [fetch operation](https://fetch.spec.whatwg.org/#concept-fetch) -use hyper::header::{AccessControlExposeHeaders, Headers}; +use hyper::header::{AccessControlExposeHeaders, ContentType, Headers}; +use hyper::http::RawStatus; use hyper::status::StatusCode; +use {Metadata, NetworkError}; use std::ascii::AsciiExt; use std::cell::{Cell, RefCell}; use std::sync::{Arc, Mutex}; use url::Url; /// [Response type](https://fetch.spec.whatwg.org/#concept-response-type) -#[derive(Clone, PartialEq, Copy, Debug)] +#[derive(Clone, PartialEq, Copy, Debug, Deserialize, Serialize)] pub enum ResponseType { Basic, CORS, @@ -23,7 +25,7 @@ pub enum ResponseType { } /// [Response termination reason](https://fetch.spec.whatwg.org/#concept-response-termination-reason) -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Deserialize, Serialize)] pub enum TerminationReason { EndUserAbort, Fatal, @@ -50,7 +52,7 @@ impl ResponseBody { /// [Cache state](https://fetch.spec.whatwg.org/#concept-response-cache-state) -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub enum CacheState { None, Local, @@ -81,6 +83,7 @@ pub struct Response { pub url_list: RefCell>, /// `None` can be considered a StatusCode of `0`. pub status: Option, + pub raw_status: Option, pub headers: Headers, pub body: Arc>, pub cache_state: CacheState, @@ -100,6 +103,7 @@ impl Response { url: None, url_list: RefCell::new(Vec::new()), status: Some(StatusCode::Ok), + raw_status: Some(RawStatus(200, "OK".into())), headers: Headers::new(), body: Arc::new(Mutex::new(ResponseBody::Empty)), cache_state: CacheState::None, @@ -116,6 +120,7 @@ impl Response { url: None, url_list: RefCell::new(vec![]), status: None, + raw_status: None, headers: Headers::new(), body: Arc::new(Mutex::new(ResponseBody::Empty)), cache_state: CacheState::None, @@ -216,4 +221,21 @@ impl Response { response } + + pub fn metadata(&self) -> Result { + let mut metadata = if let Some(ref url) = self.url { + Metadata::default(url.clone()) + } else { + return Err(NetworkError::Internal("No url found".to_string())); + }; + + metadata.set_content_type(match self.headers.get() { + Some(&ContentType(ref mime)) => Some(mime), + None => None + }); + metadata.headers = Some(self.headers.clone()); + metadata.status = self.raw_status.clone(); + metadata.https_state = self.https_state; + return Ok(metadata); + } } From d4f428ad9559f7d5f338e565e7f5d5efab70f67f Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 1 Jun 2016 14:59:55 +0530 Subject: [PATCH 03/12] Abstractify the Listener abstractions further --- components/net_traits/lib.rs | 27 ++++++++++++++++++++-- components/script/dom/htmllinkelement.rs | 2 +- components/script/dom/htmlmediaelement.rs | 2 +- components/script/dom/htmlscriptelement.rs | 2 +- components/script/dom/xmlhttprequest.rs | 2 +- components/script/network_listener.rs | 27 ++++++++++++++-------- components/script/script_thread.rs | 2 +- 7 files changed, 47 insertions(+), 17 deletions(-) diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index 127c85f13b0..4760797cc2f 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -187,6 +187,13 @@ pub trait FetchTaskTarget { fn process_response_eof(&mut self, response: &response::Response); } +pub trait FetchResponseListener { + fn process_request_body(&mut self); + fn process_request_eof(&mut self); + fn process_response(&mut self, metadata: Result); + fn process_response_eof(&mut self, response: Result, NetworkError>); +} + impl FetchTaskTarget for IpcSender { fn process_request_body(&mut self, _: &request::Request) { let _ = self.send(FetchResponseMsg::ProcessRequestBody); @@ -217,6 +224,10 @@ impl FetchTaskTarget for IpcSender { } } +pub trait Action { + fn process(self, listener: &mut Listener); +} + /// A listener for asynchronous network events. Cancelling the underlying request is unsupported. pub trait AsyncResponseListener { /// The response headers for a request have been received. @@ -241,9 +252,9 @@ pub enum ResponseAction { ResponseComplete(Result<(), NetworkError>) } -impl ResponseAction { +impl Action for ResponseAction { /// Execute the default action on a provided listener. - pub fn process(self, listener: &mut AsyncResponseListener) { + fn process(self, listener: &mut T) { match self { ResponseAction::HeadersAvailable(m) => listener.headers_available(m), ResponseAction::DataAvailable(d) => listener.data_available(d), @@ -252,6 +263,18 @@ impl ResponseAction { } } +impl Action for FetchResponseMsg { + /// Execute the default action on a provided listener. + fn process(self, listener: &mut T) { + match self { + FetchResponseMsg::ProcessRequestBody => listener.process_request_body(), + FetchResponseMsg::ProcessRequestEOF => listener.process_request_eof(), + FetchResponseMsg::ProcessResponse(meta) => listener.process_response(meta), + FetchResponseMsg::ProcessResponseEOF(data) => listener.process_response_eof(data), + } + } +} + /// A target for async networking events. Commonly used to dispatch a runnable event to another /// thread storing the wrapped closure for later execution. #[derive(Deserialize, Serialize)] diff --git a/components/script/dom/htmllinkelement.rs b/components/script/dom/htmllinkelement.rs index 2b0056a8d65..79b6ed32f0b 100644 --- a/components/script/dom/htmllinkelement.rs +++ b/components/script/dom/htmllinkelement.rs @@ -227,7 +227,7 @@ impl HTMLLinkElement { sender: action_sender, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); if self.parser_inserted.get() { diff --git a/components/script/dom/htmlmediaelement.rs b/components/script/dom/htmlmediaelement.rs index 7256ae1b131..70563ec95c7 100644 --- a/components/script/dom/htmlmediaelement.rs +++ b/components/script/dom/htmlmediaelement.rs @@ -484,7 +484,7 @@ impl HTMLMediaElement { sender: action_sender, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); // FIXME: we're supposed to block the load event much earlier than now diff --git a/components/script/dom/htmlscriptelement.rs b/components/script/dom/htmlscriptelement.rs index 1f9f49516dd..f280f670e4b 100644 --- a/components/script/dom/htmlscriptelement.rs +++ b/components/script/dom/htmlscriptelement.rs @@ -314,7 +314,7 @@ impl HTMLScriptElement { sender: action_sender, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); doc.load_async(LoadType::Script(url), response_target); diff --git a/components/script/dom/xmlhttprequest.rs b/components/script/dom/xmlhttprequest.rs index 318d3bbc1ec..3cb74ec282c 100644 --- a/components/script/dom/xmlhttprequest.rs +++ b/components/script/dom/xmlhttprequest.rs @@ -302,7 +302,7 @@ impl XMLHttpRequest { sender: action_sender, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); core_resource_thread.send(Load(load_data, LoadConsumer::Listener(response_target), None)).unwrap(); } diff --git a/components/script/network_listener.rs b/components/script/network_listener.rs index cff0723d277..84561667f4f 100644 --- a/components/script/network_listener.rs +++ b/components/script/network_listener.rs @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use net_traits::{AsyncResponseListener, ResponseAction}; +use net_traits::{Action, AsyncResponseListener, ResponseAction}; use script_runtime::ScriptThreadEventCategory::NetworkEvent; use script_runtime::{CommonScriptMsg, ScriptChan}; use script_thread::Runnable; @@ -10,13 +10,13 @@ use std::sync::{Arc, Mutex}; /// An off-thread sink for async network event runnables. All such events are forwarded to /// a target thread, where they are invoked on the provided context object. -pub struct NetworkListener { - pub context: Arc>, +pub struct NetworkListener { + pub context: Arc>, pub script_chan: Box, } -impl NetworkListener { - pub fn notify(&self, action: ResponseAction) { +impl NetworkListener { + pub fn notify+Send+'static>(&self, action: A) { if let Err(err) = self.script_chan.send(CommonScriptMsg::RunnableMsg(NetworkEvent, box ListenerRunnable { context: self.context.clone(), action: action, @@ -26,6 +26,13 @@ impl NetworkListener { } } +// helps type inference +impl NetworkListener { + pub fn notify_action(&self, action: ResponseAction) { + self.notify(action); + } +} + /// A gating mechanism that runs before invoking the runnable on the target thread. /// If the `should_invoke` method returns false, the runnable is discarded without /// being invoked. @@ -36,13 +43,13 @@ pub trait PreInvoke { } /// A runnable for moving the async network events between threads. -struct ListenerRunnable { - context: Arc>, - action: ResponseAction, +struct ListenerRunnable+Send+'static, Listener: PreInvoke + Send> { + context: Arc>, + action: A, } -impl Runnable for ListenerRunnable { - fn handler(self: Box>) { +impl+Send+'static, Listener: PreInvoke + Send> Runnable for ListenerRunnable { + fn handler(self: Box>) { let this = *self; let mut context = this.context.lock().unwrap(); if context.should_invoke() { diff --git a/components/script/script_thread.rs b/components/script/script_thread.rs index eec1853239e..c4254af7eee 100644 --- a/components/script/script_thread.rs +++ b/components/script/script_thread.rs @@ -1947,7 +1947,7 @@ impl ScriptThread { script_chan: self.chan.clone(), }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify(message.to().unwrap()); + listener.notify_action(message.to().unwrap()); }); let response_target = AsyncResponseTarget { sender: action_sender, From 5e49873af7bfe7190afa98a8d6bee08582213fb1 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 1 Jun 2016 17:18:11 +0530 Subject: [PATCH 04/12] Hook XHR up to fetch --- components/script/dom/xmlhttprequest.rs | 189 ++++++------------------ components/script/network_listener.rs | 10 +- 2 files changed, 51 insertions(+), 148 deletions(-) diff --git a/components/script/dom/xmlhttprequest.rs b/components/script/dom/xmlhttprequest.rs index 3cb74ec282c..400549dc714 100644 --- a/components/script/dom/xmlhttprequest.rs +++ b/components/script/dom/xmlhttprequest.rs @@ -2,8 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use cors::CORSResponse; -use cors::{AsyncCORSResponseListener, CORSRequest, RequestMode, allow_cross_origin_request}; use document_loader::DocumentLoader; use dom::bindings::cell::DOMRefCell; use dom::bindings::codegen::Bindings::BlobBinding::BlobMethods; @@ -45,17 +43,17 @@ use js::jsapi::JS_ClearPendingException; use js::jsapi::{JSContext, JS_ParseJSON, RootedValue}; use js::jsval::{JSVal, NullValue, UndefinedValue}; use msg::constellation_msg::{PipelineId, ReferrerPolicy}; -use net_traits::CoreResourceMsg::Load; +use net_traits::CoreResourceMsg::Fetch; use net_traits::trim_http_whitespace; -use net_traits::{AsyncResponseListener, AsyncResponseTarget, Metadata, NetworkError, RequestSource}; -use net_traits::{LoadConsumer, LoadContext, LoadData, ResourceCORSData, CoreResourceThread, LoadOrigin}; +use net_traits::{FetchResponseListener, Metadata, NetworkError, RequestSource}; +use net_traits::{LoadContext, LoadData, CoreResourceThread, LoadOrigin}; use network_listener::{NetworkListener, PreInvoke}; use parse::html::{ParseContext, parse_html}; use parse::xml::{self, parse_xml}; use script_runtime::ScriptChan; use std::ascii::AsciiExt; use std::borrow::ToOwned; -use std::cell::{Cell, RefCell}; +use std::cell::Cell; use std::default::Default; use std::str; use std::sync::{Arc, Mutex}; @@ -63,7 +61,6 @@ use string_cache::Atom; use time; use timers::{OneshotTimerCallback, OneshotTimerHandle}; use url::{Url, Position}; -use util::prefs::mozbrowser_enabled; #[derive(JSTraceable, PartialEq, Copy, Clone, HeapSizeOf)] enum XMLHttpRequestState { @@ -82,7 +79,6 @@ pub struct GenerationId(u32); struct XHRContext { xhr: TrustedXHRAddress, gen_id: GenerationId, - cors_request: Option, buf: DOMRefCell>, sync_status: DOMRefCell>, } @@ -216,75 +212,42 @@ impl XMLHttpRequest { } } - fn check_cors(context: Arc>, - load_data: LoadData, - req: CORSRequest, - script_chan: Box, - core_resource_thread: CoreResourceThread) { - struct CORSContext { - xhr: Arc>, - load_data: RefCell>, - req: CORSRequest, - script_chan: Box, - core_resource_thread: CoreResourceThread, - } - - impl AsyncCORSResponseListener for CORSContext { - fn response_available(&self, response: CORSResponse) { - if response.network_error { - let mut context = self.xhr.lock().unwrap(); - let xhr = context.xhr.root(); - xhr.process_partial_response(XHRProgress::Errored(context.gen_id, Error::Network)); - *context.sync_status.borrow_mut() = Some(Err(Error::Network)); - return; - } - - let mut load_data = self.load_data.borrow_mut().take().unwrap(); - load_data.cors = Some(ResourceCORSData { - preflight: self.req.preflight_flag, - origin: self.req.origin.clone() - }); - - XMLHttpRequest::initiate_async_xhr(self.xhr.clone(), self.script_chan.clone(), - self.core_resource_thread.clone(), load_data); - } - } - - let cors_context = CORSContext { - xhr: context, - load_data: RefCell::new(Some(load_data)), - req: req.clone(), - script_chan: script_chan.clone(), - core_resource_thread: core_resource_thread, - }; - - req.http_fetch_async(box cors_context, script_chan); - } - fn initiate_async_xhr(context: Arc>, script_chan: Box, core_resource_thread: CoreResourceThread, load_data: LoadData) { - impl AsyncResponseListener for XHRContext { - fn headers_available(&mut self, metadata: Result) { - let xhr = self.xhr.root(); - let rv = xhr.process_headers_available(self.cors_request.clone(), - self.gen_id, - metadata); - if rv.is_err() { - *self.sync_status.borrow_mut() = Some(rv); + impl FetchResponseListener for XHRContext { + fn process_request_body(&mut self) { + // todo + } + fn process_request_eof(&mut self) { + // todo + } + fn process_response(&mut self, metadata: Result) { + let xhr = self.xhr.root(); + let rv = xhr.process_headers_available(self.gen_id, + metadata); + if rv.is_err() { + *self.sync_status.borrow_mut() = Some(rv); + } + } + fn process_response_eof(&mut self, response: Result, NetworkError>) { + match response { + Ok(buf) => { + *self.buf.borrow_mut() = buf; + // todo move to a process_chunk + self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); + let rv = self.xhr.root().process_response_complete(self.gen_id, Ok(())); + *self.sync_status.borrow_mut() = Some(rv); + } + Err(e) => { + let rv = self.xhr.root().process_response_complete(self.gen_id, Err(e)); + *self.sync_status.borrow_mut() = Some(rv); + } + } + + } - } - - fn data_available(&mut self, payload: Vec) { - self.buf.borrow_mut().extend_from_slice(&payload); - self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); - } - - fn response_complete(&mut self, status: Result<(), NetworkError>) { - let rv = self.xhr.root().process_response_complete(self.gen_id, status); - *self.sync_status.borrow_mut() = Some(rv); - } } impl PreInvoke for XHRContext { @@ -298,13 +261,10 @@ impl XMLHttpRequest { context: context, script_chan: script_chan, }; - let response_target = AsyncResponseTarget { - sender: action_sender, - }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - listener.notify_action(message.to().unwrap()); + listener.notify_fetch(message.to().unwrap()); }); - core_resource_thread.send(Load(load_data, LoadConsumer::Listener(response_target), None)).unwrap(); + core_resource_thread.send(Fetch(load_data, action_sender)).unwrap(); } } @@ -637,34 +597,13 @@ impl XMLHttpRequestMethods for XMLHttpRequest { // CORS stuff let global = self.global(); - let referer_url = self.global().r().get_url(); - let mode = if self.upload_events.get() { - RequestMode::ForcedPreflight - } else { - RequestMode::CORS - }; let mut combined_headers = load_data.headers.clone(); combined_headers.extend(load_data.preserved_headers.iter()); - let cors_request = CORSRequest::maybe_new(referer_url.clone(), - load_data.url.clone(), - mode, - load_data.method.clone(), - combined_headers, - true); - match cors_request { - Ok(None) => { - let bytes = referer_url[..Position::AfterPath].as_bytes().to_vec(); - self.request_headers.borrow_mut().set_raw("Referer".to_owned(), vec![bytes]); - }, - Ok(Some(ref req)) => self.insert_trusted_header("origin".to_owned(), - req.origin.to_string()), - _ => {} - } debug!("request_headers = {:?}", *self.request_headers.borrow()); self.fetch_time.set(time::now().to_timespec().sec); - let rv = self.fetch(load_data, cors_request, global.r()); + let rv = self.fetch(load_data, global.r()); // Step 10 if self.sync.get() { return rv; @@ -879,7 +818,7 @@ impl XMLHttpRequest { event.fire(self.upcast()); } - fn process_headers_available(&self, cors_request: Option, + fn process_headers_available(&self, gen_id: GenerationId, metadata: Result) -> Result<(), Error> { let metadata = match metadata { @@ -890,34 +829,7 @@ impl XMLHttpRequest { }, }; - let bypass_cross_origin_check = { - // We want to be able to do cross-origin requests in browser.html. - // If the XHR happens in a top level window and the mozbrowser - // preference is enabled, we allow bypassing the CORS check. - // This is a temporary measure until we figure out Servo privilege - // story. See https://github.com/servo/servo/issues/9582 - if let GlobalRoot::Window(win) = self.global() { - let is_root_pipeline = win.parent_info().is_none(); - let is_mozbrowser_enabled = mozbrowser_enabled(); - is_root_pipeline && is_mozbrowser_enabled - } else { - false - } - }; - - if !bypass_cross_origin_check { - if let Some(ref req) = cors_request { - match metadata.headers { - Some(ref h) if allow_cross_origin_request(req, h) => {}, - _ => { - self.process_partial_response(XHRProgress::Errored(gen_id, Error::Network)); - return Err(Error::Network); - } - } - } - } else { - debug!("Bypassing cross origin check"); - } + // todo allow cors in mozbrowser *self.response_url.borrow_mut() = metadata.final_url[..Position::AfterQuery].to_owned(); @@ -1319,24 +1231,12 @@ impl XMLHttpRequest { fn fetch(&self, load_data: LoadData, - cors_request: Result, ()>, global: GlobalRef) -> ErrorResult { - let cors_request = match cors_request { - Err(_) => { - // Happens in case of unsupported cross-origin URI schemes. - // Supported schemes are http, https, data, and about. - self.process_partial_response(XHRProgress::Errored( - self.generation_id.get(), Error::Network)); - return Err(Error::Network); - } - Ok(req) => req, - }; let xhr = Trusted::new(self); let context = Arc::new(Mutex::new(XHRContext { xhr: xhr, - cors_request: cors_request.clone(), gen_id: self.generation_id.get(), buf: DOMRefCell::new(vec!()), sync_status: DOMRefCell::new(None), @@ -1350,13 +1250,8 @@ impl XMLHttpRequest { }; let core_resource_thread = global.core_resource_thread(); - if let Some(req) = cors_request { - XMLHttpRequest::check_cors(context.clone(), load_data, req.clone(), - script_chan.clone(), core_resource_thread); - } else { - XMLHttpRequest::initiate_async_xhr(context.clone(), script_chan, - core_resource_thread, load_data); - } + XMLHttpRequest::initiate_async_xhr(context.clone(), script_chan, + core_resource_thread, load_data); if let Some(script_port) = script_port { loop { diff --git a/components/script/network_listener.rs b/components/script/network_listener.rs index 84561667f4f..cbaa073af25 100644 --- a/components/script/network_listener.rs +++ b/components/script/network_listener.rs @@ -2,7 +2,8 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use net_traits::{Action, AsyncResponseListener, ResponseAction}; +use net_traits::{Action, AsyncResponseListener, FetchResponseListener}; +use net_traits::{FetchResponseMsg, ResponseAction}; use script_runtime::ScriptThreadEventCategory::NetworkEvent; use script_runtime::{CommonScriptMsg, ScriptChan}; use script_thread::Runnable; @@ -33,6 +34,13 @@ impl NetworkListen } } +// helps type inference +impl NetworkListener { + pub fn notify_fetch(&self, action: FetchResponseMsg) { + self.notify(action); + } +} + /// A gating mechanism that runs before invoking the runnable on the target thread. /// If the `should_invoke` method returns false, the runnable is discarded without /// being invoked. From 8bcf54deb521f09ebbc14e75ca311d826a8ea559 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 1 Jun 2016 18:19:20 +0530 Subject: [PATCH 05/12] Use RequestInit instead of LoadData. Make code look like the spec. --- components/net/resource_thread.rs | 29 ++------- components/net_traits/lib.rs | 6 +- components/net_traits/request.rs | 43 ++++++++++++- components/script/dom/xmlhttprequest.rs | 85 +++++++++++++------------ 4 files changed, 94 insertions(+), 69 deletions(-) diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index b85532b22f1..95d360c62a6 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -27,7 +27,7 @@ use net_traits::{AsyncResponseTarget, Metadata, ProgressMsg, ResponseAction, Cor use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, LoadConsumer}; use net_traits::{LoadData, LoadResponse, NetworkError, ResourceId}; use net_traits::{WebSocketCommunicate, WebSocketConnectData, ResourceThreads}; -use net_traits::request::{Referer, Request}; +use net_traits::request::{Request, RequestInit}; use profile_traits::time::ProfilerChan; use rustc_serialize::json; use rustc_serialize::{Decodable, Encodable}; @@ -197,8 +197,8 @@ impl ResourceChannelManager { match self.from_client.recv().unwrap() { CoreResourceMsg::Load(load_data, consumer, id_sender) => self.resource_manager.load(load_data, consumer, id_sender, control_sender.clone()), - CoreResourceMsg::Fetch(load_data, sender) => - self.resource_manager.fetch(load_data, sender), + CoreResourceMsg::Fetch(init, sender) => + self.resource_manager.fetch(init, sender), CoreResourceMsg::WebsocketConnect(connect, connect_data) => self.resource_manager.websocket_connect(connect, connect_data), CoreResourceMsg::SetCookiesForUrl(request, cookie_list, source) => @@ -486,30 +486,13 @@ impl CoreResourceManager { cancel_listener)); } - fn fetch(&self, load_data: LoadData, sender: IpcSender) { - spawn_named(format!("fetch thread for {}", load_data.url), move || { - let mut request = Request::new(load_data.url, - None, false); - // todo handle origin - // todo consider replacing LoadData with a direct mapping - // to a subset of Request - // todo set is_service_worker_global_scope - *request.method.borrow_mut() = load_data.method; - *request.headers.borrow_mut() = load_data.headers; - *request.body.borrow_mut() = load_data.data.clone(); - if let Some(cors) = load_data.cors { - request.use_cors_preflight = cors.preflight; - } + fn fetch(&self, init: RequestInit, sender: IpcSender) { + spawn_named(format!("fetch thread for {}", init.url), move || { + let request = Request::from_init(init); // XXXManishearth: Check origin against pipeline id - request.use_url_credentials = load_data.credentials_flag; // todo load context / mimesniff in fetch // todo referrer policy? - if let Some(referer) = load_data.referrer_url { - request.referer = Referer::RefererUrl(referer); - } // todo worker stuff - - fetch(Rc::new(request), Some(Box::new(sender))); }) diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index 4760797cc2f..92251bf77d5 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -196,18 +196,22 @@ pub trait FetchResponseListener { impl FetchTaskTarget for IpcSender { fn process_request_body(&mut self, _: &request::Request) { + println!("PRqB"); let _ = self.send(FetchResponseMsg::ProcessRequestBody); } fn process_request_eof(&mut self, _: &request::Request) { + println!("PRqE"); let _ = self.send(FetchResponseMsg::ProcessRequestEOF); } fn process_response(&mut self, response: &response::Response) { + println!("PR"); let _ = self.send(FetchResponseMsg::ProcessResponse(response.metadata())); } fn process_response_eof(&mut self, response: &response::Response) { + println!("PRE"); if response.is_network_error() { // todo: finer grained errors let _ = self.send(FetchResponseMsg::ProcessResponse(Err(NetworkError::Internal("Network error".into())))); @@ -412,7 +416,7 @@ pub struct WebSocketConnectData { pub enum CoreResourceMsg { /// Request the data associated with a particular URL Load(LoadData, LoadConsumer, Option>), - Fetch(LoadData, IpcSender), + Fetch(request::RequestInit, IpcSender), /// Try to make a websocket connection to a URL. WebsocketConnect(WebSocketCommunicate, WebSocketConnectData), /// Store a set of cookies for a given originating URL diff --git a/components/net_traits/request.rs b/components/net_traits/request.rs index 0aa5417b3ba..70ac600bb25 100644 --- a/components/net_traits/request.rs +++ b/components/net_traits/request.rs @@ -25,7 +25,7 @@ pub enum Type { } /// A request [destination](https://fetch.spec.whatwg.org/#concept-request-destination) -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] pub enum Destination { None, Document, Embed, Font, Image, Manifest, Media, Object, Report, Script, ServiceWorker, @@ -49,7 +49,7 @@ pub enum Referer { } /// A [request mode](https://fetch.spec.whatwg.org/#concept-request-mode) -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] pub enum RequestMode { Navigate, SameOrigin, @@ -58,7 +58,7 @@ pub enum RequestMode { } /// Request [credentials mode](https://fetch.spec.whatwg.org/#concept-request-credentials-mode) -#[derive(Copy, Clone, PartialEq)] +#[derive(Copy, Clone, PartialEq, Serialize, Deserialize)] pub enum CredentialsMode { Omit, CredentialsSameOrigin, @@ -107,6 +107,26 @@ pub enum CORSSettings { UseCredentials } +#[derive(Serialize, Deserialize, Clone)] +pub struct RequestInit { + pub method: Method, + pub url: Url, + pub headers: Headers, + pub unsafe_request: bool, + pub same_origin_data: bool, + pub body: Option>, + // TODO: cleint object + pub destination: Destination, + pub synchronous: bool, + pub mode: RequestMode, + pub use_cors_preflight: bool, + pub credentials_mode: CredentialsMode, + pub use_url_credentials: bool, + // this should actually be set by fetch, but fetch + // doesn't have info about the client right now + pub origin: Url, +} + /// A [Request](https://fetch.spec.whatwg.org/#requests) as defined by the Fetch spec #[derive(Clone)] pub struct Request { @@ -186,6 +206,23 @@ impl Request { } } + pub fn from_init(init: RequestInit) -> Request { + let mut req = Request::new(init.url, None, false); + *req.method.borrow_mut() = init.method; + *req.headers.borrow_mut() = init.headers; + req.unsafe_request = init.unsafe_request; + req.same_origin_data.set(init.same_origin_data); + *req.body.borrow_mut() = init.body; + req.destination = init.destination; + req.synchronous = init.synchronous; + req.mode = init.mode; + req.use_cors_preflight = init.use_cors_preflight; + req.credentials_mode = init.credentials_mode; + req.use_url_credentials = init.use_url_credentials; + *req.origin.borrow_mut() = Origin::Origin(init.origin.origin()); + req + } + /// https://html.spec.whatwg.org/multipage/#create-a-potential-cors-request pub fn potential_cors_request(url: Url, cors_attribute_state: Option, diff --git a/components/script/dom/xmlhttprequest.rs b/components/script/dom/xmlhttprequest.rs index 400549dc714..7239ae5a745 100644 --- a/components/script/dom/xmlhttprequest.rs +++ b/components/script/dom/xmlhttprequest.rs @@ -33,7 +33,7 @@ use encoding::label::encoding_from_whatwg_label; use encoding::types::{DecoderTrap, EncoderTrap, Encoding, EncodingRef}; use euclid::length::Length; use hyper::header::Headers; -use hyper::header::{Accept, ContentLength, ContentType, qitem}; +use hyper::header::{ContentLength, ContentType}; use hyper::http::RawStatus; use hyper::method::Method; use hyper::mime::{self, Mime}; @@ -46,7 +46,8 @@ use msg::constellation_msg::{PipelineId, ReferrerPolicy}; use net_traits::CoreResourceMsg::Fetch; use net_traits::trim_http_whitespace; use net_traits::{FetchResponseListener, Metadata, NetworkError, RequestSource}; -use net_traits::{LoadContext, LoadData, CoreResourceThread, LoadOrigin}; +use net_traits::{CoreResourceThread, LoadOrigin}; +use net_traits::request::{CredentialsMode, Destination, RequestInit, RequestMode, Origin}; use network_listener::{NetworkListener, PreInvoke}; use parse::html::{ParseContext, parse_html}; use parse::xml::{self, parse_xml}; @@ -138,7 +139,6 @@ pub struct XMLHttpRequest { request_body_len: Cell, sync: Cell, upload_complete: Cell, - upload_events: Cell, send_flag: Cell, timeout_cancel: DOMRefCell>, @@ -183,7 +183,6 @@ impl XMLHttpRequest { request_body_len: Cell::new(0), sync: Cell::new(false), upload_complete: Cell::new(false), - upload_events: Cell::new(false), send_flag: Cell::new(false), timeout_cancel: DOMRefCell::new(None), @@ -215,7 +214,7 @@ impl XMLHttpRequest { fn initiate_async_xhr(context: Arc>, script_chan: Box, core_resource_thread: CoreResourceThread, - load_data: LoadData) { + init: RequestInit) { impl FetchResponseListener for XHRContext { fn process_request_body(&mut self) { // todo @@ -262,9 +261,10 @@ impl XMLHttpRequest { script_chan: script_chan, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { + println!("routing"); listener.notify_fetch(message.to().unwrap()); }); - core_resource_thread.send(Fetch(load_data, action_sender)).unwrap(); + core_resource_thread.send(Fetch(init, action_sender)).unwrap(); } } @@ -527,8 +527,10 @@ impl XMLHttpRequestMethods for XMLHttpRequest { let extracted = data.as_ref().map(|d| d.extract()); self.request_body_len.set(extracted.as_ref().map_or(0, |e| e.0.len())); + // todo preserved headers? + // Step 6 - self.upload_events.set(false); + self.upload_complete.set(false); // Step 7 self.upload_complete.set(match extracted { None => true, @@ -540,11 +542,6 @@ impl XMLHttpRequestMethods for XMLHttpRequest { // Step 9 if !self.sync.get() { - let event_target = self.upload.upcast::(); - if event_target.has_handlers() { - self.upload_events.set(true); - } - // If one of the event handlers below aborts the fetch by calling // abort or open we will need the current generation id to detect it. // Substep 1 @@ -564,46 +561,50 @@ impl XMLHttpRequestMethods for XMLHttpRequest { } // Step 5 - let global = self.global(); - - let mut load_data = - LoadData::new(LoadContext::Browsing, - self.request_url.borrow().clone().unwrap(), - self); - - if load_data.url.origin().ne(&global.r().get_url().origin()) { - load_data.credentials_flag = self.WithCredentials(); - } - load_data.data = extracted.as_ref().map(|e| e.0.clone()); - + //TODO - set referrer_policy/referrer_url in request + let has_handlers = self.upload.upcast::().has_handlers(); + let credentials_mode = if self.with_credentials.get() { + CredentialsMode::Include + } else { + CredentialsMode::CredentialsSameOrigin + }; + let use_url_credentials = if let Some(ref url) = *self.request_url.borrow() { + url.username().len() != 0 || url.password().is_some() + } else { + unreachable!() + }; + let mut request = RequestInit { + method: self.request_method.borrow().clone(), + url: self.request_url.borrow().clone().unwrap(), + headers: (*self.request_headers.borrow()).clone(), + unsafe_request: true, + same_origin_data: true, + // XXXManishearth figure out how to avoid this clone + body: extracted.as_ref().map(|e| e.0.clone()), + // XXXManishearth actually "subresource", but it doesn't exist + // https://github.com/whatwg/xhr/issues/71 + destination: Destination::None, + synchronous: self.sync.get(), + mode: RequestMode::CORSMode, + use_cors_preflight: has_handlers, + credentials_mode: credentials_mode, + use_url_credentials: use_url_credentials, + origin: self.global().r().get_url(), + }; // XHR spec differs from http, and says UTF-8 should be in capitals, // instead of "utf-8", which is what Hyper defaults to. So not // using content types provided by Hyper. let n = "content-type"; match extracted { Some((_, Some(ref content_type))) => - load_data.headers.set_raw(n.to_owned(), vec![content_type.bytes().collect()]), + request.headers.set_raw(n.to_owned(), vec![content_type.bytes().collect()]), _ => (), } - load_data.preserved_headers = (*self.request_headers.borrow()).clone(); - - if !load_data.preserved_headers.has::() { - let mime = Mime(mime::TopLevel::Star, mime::SubLevel::Star, vec![]); - load_data.preserved_headers.set(Accept(vec![qitem(mime)])); - } - - load_data.method = (*self.request_method.borrow()).clone(); - - // CORS stuff - let global = self.global(); - let mut combined_headers = load_data.headers.clone(); - combined_headers.extend(load_data.preserved_headers.iter()); - debug!("request_headers = {:?}", *self.request_headers.borrow()); self.fetch_time.set(time::now().to_timespec().sec); - let rv = self.fetch(load_data, global.r()); + let rv = self.fetch(request, self.global().r()); // Step 10 if self.sync.get() { return rv; @@ -1230,7 +1231,7 @@ impl XMLHttpRequest { } fn fetch(&self, - load_data: LoadData, + init: RequestInit, global: GlobalRef) -> ErrorResult { let xhr = Trusted::new(self); @@ -1251,7 +1252,7 @@ impl XMLHttpRequest { let core_resource_thread = global.core_resource_thread(); XMLHttpRequest::initiate_async_xhr(context.clone(), script_chan, - core_resource_thread, load_data); + core_resource_thread, init); if let Some(script_port) = script_port { loop { From 6e29b872d7fb8556ea7dd610e82868b6f719f24e Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Wed, 1 Jun 2016 20:32:40 +0530 Subject: [PATCH 06/12] Test fixes: - Hack to stop hitting unreachable on referer - add fetch_done to make sync work - Make datauris work by setting the response URI, spec bug - Allow for empty bodies - Make request bodies work (pass to http, fix fencepost in iter count) --- components/net/fetch/methods.rs | 122 ++++++++++++++---------- components/net/http_loader.rs | 6 +- components/net/resource_thread.rs | 18 +++- components/net/websocket_loader.rs | 2 +- components/net_traits/lib.rs | 84 ++++++++++++---- components/net_traits/request.rs | 6 +- components/script/dom/xmlhttprequest.rs | 26 +++-- tests/unit/net/fetch.rs | 68 ++++++------- 8 files changed, 206 insertions(+), 126 deletions(-) diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 79469469e87..d2bab7b16ad 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -5,6 +5,7 @@ use connector::create_http_connector; use data_loader::decode; use fetch::cors_cache::CORSCache; +use http_loader::{HttpState, set_request_cookies}; use http_loader::{NetworkHttpRequestFactory, ReadResult, obtain_response, read_block}; use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials}; use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods}; @@ -38,13 +39,14 @@ pub type Target = Option>; type DoneChannel = Option<(Sender<()>, Receiver<()>)>; /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch) -pub fn fetch(request: Rc, target: Target) -> Response { - fetch_with_cors_cache(request, &mut CORSCache::new(), target) +pub fn fetch(request: Rc, target: &mut Target, state: HttpState) -> Response { + fetch_with_cors_cache(request, &mut CORSCache::new(), target, state) } pub fn fetch_with_cors_cache(request: Rc, cache: &mut CORSCache, - mut target: Target) -> Response { + target: &mut Target, + state: HttpState) -> Response { // Step 1 if request.window.get() == Window::Client { // TODO: Set window to request's client object if client is a Window object @@ -105,12 +107,13 @@ pub fn fetch_with_cors_cache(request: Rc, } // Step 7 - main_fetch(request, cache, false, false, &mut target, &mut None) + main_fetch(request, cache, false, false, target, &mut None, &state) } /// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch) fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, - recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel) -> Response { + recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel, + state: &HttpState) -> Response { // TODO: Implement main fetch spec // Step 1 @@ -137,8 +140,10 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, // TODO this step (referer policy) // Step 8 - if request.referer != Referer::NoReferer { + if *request.referer.borrow() != Referer::NoReferer { // TODO be able to invoke "determine request's referer" + // once this is filled in be sure to update the match + // referer below to have an unreachable branch for client } // Step 9 @@ -163,14 +168,14 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, (current_url.scheme() == "file" && request.same_origin_data.get()) || current_url.scheme() == "about" || request.mode == RequestMode::Navigate { - basic_fetch(request.clone(), cache, target, done_chan) + basic_fetch(request.clone(), cache, target, done_chan, state) } else if request.mode == RequestMode::SameOrigin { Response::network_error() } else if request.mode == RequestMode::NoCORS { request.response_tainting.set(ResponseTainting::Opaque); - basic_fetch(request.clone(), cache, target, done_chan) + basic_fetch(request.clone(), cache, target, done_chan, state) } else if !matches!(current_url.scheme(), "http" | "https") { Response::network_error() @@ -181,7 +186,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) { request.response_tainting.set(ResponseTainting::CORSTainting); request.redirect_mode.set(RedirectMode::Error); - let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan); + let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, state); if response.is_network_error() { // TODO clear cache entries using request } @@ -189,7 +194,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, } else { request.response_tainting.set(ResponseTainting::CORSTainting); - http_fetch(request.clone(), cache, true, false, false, target, done_chan) + http_fetch(request.clone(), cache, true, false, false, target, done_chan, state) } } }; @@ -253,10 +258,8 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, // Step 18 if request.synchronous { - if !response.is_network_error() { - if let Some(ref ch) = *done_chan { - let _ = ch.1.recv(); - } + if let Some(ref ch) = *done_chan { + let _ = ch.1.recv(); } return response; } @@ -273,23 +276,22 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, } } - { - // Step 20 - if let Some(ref mut target) = *target { - target.process_response(&response); - } + // Step 20 + if let Some(ref mut target) = *target { + target.process_response(&response); + } - // Step 21 - if !response.is_network_error() { - if let Some(ref ch) = *done_chan { - let _ = ch.1.recv(); - } - } + // Step 21 + if let Some(ref ch) = *done_chan { + let _ = ch.1.recv(); + } - // Step 22 - if let Some(ref mut target) = *target { - target.process_response_eof(&response); - } + // Step 22 + request.done.set(true); + + // Step 23 + if let Some(ref mut target) = *target { + target.process_response_eof(&response); } // TODO remove this line when only asynchronous fetches are used @@ -298,19 +300,22 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch) fn basic_fetch(request: Rc, cache: &mut CORSCache, - target: &mut Target, done_chan: &mut DoneChannel) -> Response { + target: &mut Target, done_chan: &mut DoneChannel, + state: &HttpState) -> Response { let url = request.current_url(); match url.scheme() { "about" if url.path() == "blank" => { let mut response = Response::new(); + // https://github.com/whatwg/fetch/issues/312 + response.url = Some(url); response.headers.set(ContentType(mime!(Text / Html; Charset = Utf8))); *response.body.lock().unwrap() = ResponseBody::Done(vec![]); response }, "http" | "https" => { - http_fetch(request.clone(), cache, false, false, false, target, done_chan) + http_fetch(request.clone(), cache, false, false, false, target, done_chan, state) }, "data" => { @@ -318,6 +323,8 @@ fn basic_fetch(request: Rc, cache: &mut CORSCache, match decode(&url) { Ok((mime, bytes)) => { let mut response = Response::new(); + // https://github.com/whatwg/fetch/issues/312 + response.url = Some(url.clone()); *response.body.lock().unwrap() = ResponseBody::Done(bytes); response.headers.set(ContentType(mime)); response @@ -339,6 +346,8 @@ fn basic_fetch(request: Rc, cache: &mut CORSCache, let mime = guess_mime_type(file_path); let mut response = Response::new(); + // https://github.com/whatwg/fetch/issues/312 + response.url = Some(url.clone()); *response.body.lock().unwrap() = ResponseBody::Done(bytes); response.headers.set(ContentType(mime)); response @@ -367,7 +376,8 @@ fn http_fetch(request: Rc, cors_preflight_flag: bool, authentication_fetch_flag: bool, target: &mut Target, - done_chan: &mut DoneChannel) -> Response { + done_chan: &mut DoneChannel, + state: &HttpState) -> Response { // This is a new async fetch, reset the channel we are waiting on *done_chan = None; @@ -423,7 +433,7 @@ fn http_fetch(request: Rc, // Sub-substep 1 if method_mismatch || header_mismatch { - let preflight_result = cors_preflight_fetch(request.clone(), cache); + let preflight_result = cors_preflight_fetch(request.clone(), cache, state); // Sub-substep 2 if preflight_result.response_type == ResponseType::Error { return Response::network_error(); @@ -444,7 +454,7 @@ fn http_fetch(request: Rc, // Substep 4 let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag, - done_chan); + done_chan, state); // Substep 5 if cors_flag && cors_check(request.clone(), &fetch_result).is_err() { @@ -472,7 +482,7 @@ fn http_fetch(request: Rc, // set back to default response.return_internal.set(true); http_redirect_fetch(request, cache, Rc::new(response), - cors_flag, target, done_chan) + cors_flag, target, done_chan, state) } } }, @@ -495,7 +505,7 @@ fn http_fetch(request: Rc, // Step 4 return http_fetch(request, cache, cors_flag, cors_preflight_flag, - true, target, done_chan); + true, target, done_chan, state); } // Code 407 @@ -513,7 +523,7 @@ fn http_fetch(request: Rc, return http_fetch(request, cache, cors_flag, cors_preflight_flag, authentication_fetch_flag, target, - done_chan); + done_chan, state); } _ => { } @@ -536,7 +546,8 @@ fn http_redirect_fetch(request: Rc, response: Rc, cors_flag: bool, target: &mut Target, - done_chan: &mut DoneChannel) -> Response { + done_chan: &mut DoneChannel, + state: &HttpState) -> Response { // Step 1 assert_eq!(response.return_internal.get(), true); @@ -610,14 +621,15 @@ fn http_redirect_fetch(request: Rc, request.url_list.borrow_mut().push(location_url); // Step 15 - main_fetch(request, cache, cors_flag, true, target, done_chan) + main_fetch(request, cache, cors_flag, true, target, done_chan, state) } /// [HTTP network or cache fetch](https://fetch.spec.whatwg.org#http-network-or-cache-fetch) fn http_network_or_cache_fetch(request: Rc, credentials_flag: bool, authentication_fetch_flag: bool, - done_chan: &mut DoneChannel) -> Response { + done_chan: &mut DoneChannel, + state: &HttpState) -> Response { // TODO: Implement Window enum for Request let request_has_no_window = true; @@ -648,15 +660,17 @@ fn http_network_or_cache_fetch(request: Rc, } // Step 6 - match http_request.referer { - Referer::NoReferer => + match *http_request.referer.borrow() { + // Referer::Client should not be here, but we don't set + // the referer yet, so we club it with NoReferer + Referer::NoReferer | Referer::Client => http_request.headers.borrow_mut().set(RefererHeader("".to_owned())), Referer::RefererUrl(ref http_request_referer) => http_request.headers.borrow_mut().set(RefererHeader(http_request_referer.to_string())), - Referer::Client => - // it should be impossible for referer to be anything else during fetching - // https://fetch.spec.whatwg.org/#concept-request-referrer - unreachable!() + // Referer::Client => + // // it should be impossible for referer to be anything else during fetching + // // https://fetch.spec.whatwg.org/#concept-request-referrer + // unreachable!() }; // Step 7 @@ -705,7 +719,9 @@ fn http_network_or_cache_fetch(request: Rc, if credentials_flag { // Substep 1 // TODO http://mxr.mozilla.org/servo/source/components/net/http_loader.rs#504 - + set_request_cookies(&http_request.current_url(), + &mut *http_request.headers.borrow_mut(), + &state.cookie_jar); // Substep 2 if !http_request.headers.borrow().has::>() { // Substep 3 @@ -780,7 +796,7 @@ fn http_network_or_cache_fetch(request: Rc, // Step 18 if response.is_none() { - response = Some(http_network_fetch(request.clone(), http_request.clone(), credentials_flag, done_chan)); + response = Some(http_network_fetch(http_request.clone(), http_request.clone(), credentials_flag, done_chan)); } let response = response.unwrap(); @@ -839,8 +855,8 @@ fn http_network_fetch(request: Rc, let wrapped_response = obtain_response(&factory, &url, &request.method.borrow(), &request.headers.borrow(), - &cancellation_listener, &None, &request.method.borrow(), - &None, request.redirect_count.get(), &None, ""); + &cancellation_listener, &request.body.borrow(), &request.method.borrow(), + &None, request.redirect_count.get()+1, &None, ""); let mut response = Response::new(); match wrapped_response { @@ -938,14 +954,14 @@ fn http_network_fetch(request: Rc, } /// [CORS preflight fetch](https://fetch.spec.whatwg.org#cors-preflight-fetch) -fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache) -> Response { +fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache, state: &HttpState) -> Response { // Step 1 let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()), false); *preflight.method.borrow_mut() = Method::Options; preflight.initiator = request.initiator.clone(); preflight.type_ = request.type_.clone(); preflight.destination = request.destination.clone(); - preflight.referer = request.referer.clone(); + *preflight.referer.borrow_mut() = request.referer.borrow().clone(); // Step 2 preflight.headers.borrow_mut().set::( @@ -966,7 +982,7 @@ fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache) -> Response // Step 6 let preflight = Rc::new(preflight); - let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None); + let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, state); // Step 7 if cors_check(request.clone(), &response).is_ok() && diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index f94b6ffce32..e62fc2dd415 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -453,9 +453,9 @@ fn determine_request_referrer(headers: &mut Headers, return None; } -pub fn set_request_cookies(url: Url, headers: &mut Headers, cookie_jar: &Arc>) { +pub fn set_request_cookies(url: &Url, headers: &mut Headers, cookie_jar: &Arc>) { let mut cookie_jar = cookie_jar.write().unwrap(); - if let Some(cookie_list) = cookie_jar.cookies_for_url(&url, CookieSource::HTTP) { + if let Some(cookie_list) = cookie_jar.cookies_for_url(url, CookieSource::HTTP) { let mut v = Vec::new(); v.push(cookie_list.into_bytes()); headers.set_raw("Cookie".to_owned(), v); @@ -665,7 +665,7 @@ pub fn modify_request_headers(headers: &mut Headers, // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11 if load_data.credentials_flag { if !block_cookies { - set_request_cookies(url.clone(), headers, cookie_jar); + set_request_cookies(&url, headers, cookie_jar); } // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12 diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index 95d360c62a6..18857e5005c 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -24,7 +24,7 @@ use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag}; use net_traits::LoadContext; use net_traits::ProgressMsg::Done; use net_traits::{AsyncResponseTarget, Metadata, ProgressMsg, ResponseAction, CoreResourceThread}; -use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, LoadConsumer}; +use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, FetchTaskTarget, LoadConsumer}; use net_traits::{LoadData, LoadResponse, NetworkError, ResourceId}; use net_traits::{WebSocketCommunicate, WebSocketConnectData, ResourceThreads}; use net_traits::request::{Request, RequestInit}; @@ -486,16 +486,24 @@ impl CoreResourceManager { cancel_listener)); } - fn fetch(&self, init: RequestInit, sender: IpcSender) { + fn fetch(&self, init: RequestInit, sender: IpcSender) { + let http_state = HttpState { + hsts_list: self.hsts_list.clone(), + cookie_jar: self.cookie_jar.clone(), + auth_cache: self.auth_cache.clone(), + blocked_content: BLOCKED_CONTENT_RULES.clone(), + }; spawn_named(format!("fetch thread for {}", init.url), move || { + let sync = init.synchronous; let request = Request::from_init(init); // XXXManishearth: Check origin against pipeline id // todo load context / mimesniff in fetch // todo referrer policy? - // todo worker stuff - fetch(Rc::new(request), Some(Box::new(sender))); + // todo service worker stuff + let mut target = Some(Box::new(sender) as Box); + let response = fetch(Rc::new(request), &mut target, http_state); + target.unwrap().fetch_done(&response, sync); }) - } fn websocket_connect(&self, diff --git a/components/net/websocket_loader.rs b/components/net/websocket_loader.rs index 60d6c8932f6..d748e309d20 100644 --- a/components/net/websocket_loader.rs +++ b/components/net/websocket_loader.rs @@ -43,7 +43,7 @@ fn establish_a_websocket_connection(resource_url: &Url, net_url: (Host, String, request.headers.set(WebSocketProtocol(protocols.clone())); }; - http_loader::set_request_cookies(resource_url.clone(), &mut request.headers, &cookie_jar); + http_loader::set_request_cookies(&resource_url, &mut request.headers, &cookie_jar); let response = try!(request.send()); try!(response.validate()); diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index 92251bf77d5..59656fbea9d 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -36,6 +36,8 @@ use hyper::method::Method; use hyper::mime::{Attr, Mime}; use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use msg::constellation_msg::{PipelineId, ReferrerPolicy}; +use request::{Request, RequestInit}; +use response::{HttpsState, Response, ResponseBody}; use std::io::Error as IOError; use std::sync::mpsc::Sender; use std::thread; @@ -162,72 +164,111 @@ pub enum FetchResponseMsg { ProcessRequestEOF, // todo: send more info about the response (or perhaps the entire Response) ProcessResponse(Result), - ProcessResponseEOF(Result, NetworkError>), + ProcessResponseEOF(Result>, NetworkError>), + FetchDone(Result<(Metadata, Option>), NetworkError>), } pub trait FetchTaskTarget { /// https://fetch.spec.whatwg.org/#process-request-body /// /// Fired when a chunk of the request body is transmitted - fn process_request_body(&mut self, request: &request::Request); + fn process_request_body(&mut self, request: &Request); /// https://fetch.spec.whatwg.org/#process-request-end-of-file /// /// Fired when the entire request finishes being transmitted - fn process_request_eof(&mut self, request: &request::Request); + fn process_request_eof(&mut self, request: &Request); /// https://fetch.spec.whatwg.org/#process-response /// /// Fired when headers are received - fn process_response(&mut self, response: &response::Response); + fn process_response(&mut self, response: &Response); /// https://fetch.spec.whatwg.org/#process-response-end-of-file /// /// Fired when the response is fully fetched - fn process_response_eof(&mut self, response: &response::Response); + fn process_response_eof(&mut self, response: &Response); + + /// Called when fetch terminates, useful for sync + fn fetch_done(&mut self, response: &Response, sync: bool); } pub trait FetchResponseListener { fn process_request_body(&mut self); fn process_request_eof(&mut self); fn process_response(&mut self, metadata: Result); - fn process_response_eof(&mut self, response: Result, NetworkError>); + fn process_response_eof(&mut self, response: Result>, NetworkError>); + + fn fetch_done(&mut self, response: Result<(Metadata, Option>), NetworkError>); } impl FetchTaskTarget for IpcSender { - fn process_request_body(&mut self, _: &request::Request) { - println!("PRqB"); + fn process_request_body(&mut self, _: &Request) { let _ = self.send(FetchResponseMsg::ProcessRequestBody); } - fn process_request_eof(&mut self, _: &request::Request) { - println!("PRqE"); + fn process_request_eof(&mut self, _: &Request) { let _ = self.send(FetchResponseMsg::ProcessRequestEOF); } - fn process_response(&mut self, response: &response::Response) { - println!("PR"); + fn process_response(&mut self, response: &Response) { let _ = self.send(FetchResponseMsg::ProcessResponse(response.metadata())); } - fn process_response_eof(&mut self, response: &response::Response) { - println!("PRE"); + fn process_response_eof(&mut self, response: &Response) { if response.is_network_error() { // todo: finer grained errors - let _ = self.send(FetchResponseMsg::ProcessResponse(Err(NetworkError::Internal("Network error".into())))); + let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Err(NetworkError::Internal("Network error".into())))); } if let Ok(ref guard) = response.body.lock() { - if let response::ResponseBody::Done(ref vec) = **guard { - let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(vec.clone()))); - return; + match **guard { + ResponseBody::Done(ref vec) => { + let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(Some(vec.clone())))); + return; + } + ResponseBody::Empty => { + let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(None))); + return; + } + _ => () } } // If something goes wrong, log it instead of crashing the resource thread let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Err(NetworkError::Internal("Incomplete body".into())))); } + + fn fetch_done(&mut self, response: &Response, sync: bool) { + if !sync { + // fetch_done is only used by sync XHR, avoid pointless data cloning + return; + } + if response.is_network_error() { + // todo: finer grained errors + let _ = self.send(FetchResponseMsg::FetchDone(Err(NetworkError::Internal("Network error".into())))); + } + if let Ok(ref guard) = response.body.lock() { + match **guard { + ResponseBody::Done(ref vec) => { + let ret = response.metadata().map(|m| (m, Some(vec.clone()))); + let _ = self.send(FetchResponseMsg::FetchDone(ret)); + return; + } + ResponseBody::Empty => { + let ret = response.metadata().map(|m| (m, None)); + let _ = self.send(FetchResponseMsg::FetchDone(ret)); + return; + } + _ => () + } + } + + // If something goes wrong, log it instead of crashing the resource thread + let _ = self.send(FetchResponseMsg::FetchDone(Err(NetworkError::Internal("Incomplete body".into())))); + } } + pub trait Action { fn process(self, listener: &mut Listener); } @@ -275,6 +316,7 @@ impl Action for FetchResponseMsg { FetchResponseMsg::ProcessRequestEOF => listener.process_request_eof(), FetchResponseMsg::ProcessResponse(meta) => listener.process_response(meta), FetchResponseMsg::ProcessResponseEOF(data) => listener.process_response_eof(data), + FetchResponseMsg::FetchDone(response) => listener.fetch_done(response), } } } @@ -416,7 +458,7 @@ pub struct WebSocketConnectData { pub enum CoreResourceMsg { /// Request the data associated with a particular URL Load(LoadData, LoadConsumer, Option>), - Fetch(request::RequestInit, IpcSender), + Fetch(RequestInit, IpcSender), /// Try to make a websocket connection to a URL. WebsocketConnect(WebSocketCommunicate, WebSocketConnectData), /// Store a set of cookies for a given originating URL @@ -555,7 +597,7 @@ pub struct Metadata { pub status: Option, /// Is successful HTTPS connection - pub https_state: response::HttpsState, + pub https_state: HttpsState, } impl Metadata { @@ -568,7 +610,7 @@ impl Metadata { headers: None, // https://fetch.spec.whatwg.org/#concept-response-status-message status: Some(RawStatus(200, "OK".into())), - https_state: response::HttpsState::None, + https_state: HttpsState::None, } } diff --git a/components/net_traits/request.rs b/components/net_traits/request.rs index 70ac600bb25..6909d7b25e7 100644 --- a/components/net_traits/request.rs +++ b/components/net_traits/request.rs @@ -151,7 +151,7 @@ pub struct Request { pub origin: RefCell, pub omit_origin_header: Cell, pub same_origin_data: Cell, - pub referer: Referer, + pub referer: RefCell, // TODO: referrer policy pub synchronous: bool, pub mode: RequestMode, @@ -190,7 +190,7 @@ impl Request { origin: RefCell::new(origin.unwrap_or(Origin::Client)), omit_origin_header: Cell::new(false), same_origin_data: Cell::new(false), - referer: Referer::Client, + referer: RefCell::new(Referer::Client), synchronous: false, mode: RequestMode::NoCORS, use_cors_preflight: false, @@ -245,7 +245,7 @@ impl Request { origin: RefCell::new(Origin::Client), omit_origin_header: Cell::new(false), same_origin_data: Cell::new(false), - referer: Referer::Client, + referer: RefCell::new(Referer::Client), synchronous: false, // Step 1-2 mode: match cors_attribute_state { diff --git a/components/script/dom/xmlhttprequest.rs b/components/script/dom/xmlhttprequest.rs index 7239ae5a745..7425a2feff3 100644 --- a/components/script/dom/xmlhttprequest.rs +++ b/components/script/dom/xmlhttprequest.rs @@ -230,12 +230,14 @@ impl XMLHttpRequest { *self.sync_status.borrow_mut() = Some(rv); } } - fn process_response_eof(&mut self, response: Result, NetworkError>) { + fn process_response_eof(&mut self, response: Result>, NetworkError>) { match response { Ok(buf) => { - *self.buf.borrow_mut() = buf; - // todo move to a process_chunk - self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); + if let Some(buf) = buf { + *self.buf.borrow_mut() = buf; + // todo move to a process_chunk + self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); + } let rv = self.xhr.root().process_response_complete(self.gen_id, Ok(())); *self.sync_status.borrow_mut() = Some(rv); } @@ -244,8 +246,17 @@ impl XMLHttpRequest { *self.sync_status.borrow_mut() = Some(rv); } } - - + } + fn fetch_done(&mut self, response: Result<(Metadata, Option>), NetworkError>) { + match response { + Ok(response) => { + self.process_response(Ok(response.0)); + self.process_response_eof(Ok(response.1)); + } + Err(err) => { + self.process_response_eof(Err(err)); + } + } } } @@ -261,7 +272,6 @@ impl XMLHttpRequest { script_chan: script_chan, }; ROUTER.add_route(action_receiver.to_opaque(), box move |message| { - println!("routing"); listener.notify_fetch(message.to().unwrap()); }); core_resource_thread.send(Fetch(init, action_sender)).unwrap(); @@ -525,6 +535,7 @@ impl XMLHttpRequestMethods for XMLHttpRequest { }; // Step 4 let extracted = data.as_ref().map(|d| d.extract()); + self.request_body_len.set(extracted.as_ref().map_or(0, |e| e.0.len())); // todo preserved headers? @@ -604,6 +615,7 @@ impl XMLHttpRequestMethods for XMLHttpRequest { debug!("request_headers = {:?}", *self.request_headers.borrow()); self.fetch_time.set(time::now().to_timespec().sec); + let rv = self.fetch(request, self.global().r()); // Step 10 if self.sync.get() { diff --git a/tests/unit/net/fetch.rs b/tests/unit/net/fetch.rs index 16dc664ffff..3eb0ff78567 100644 --- a/tests/unit/net/fetch.rs +++ b/tests/unit/net/fetch.rs @@ -15,6 +15,7 @@ use hyper::status::StatusCode; use hyper::uri::RequestUri; use net::fetch::cors_cache::CORSCache; use net::fetch::methods::{fetch, fetch_with_cors_cache}; +use net::http_loader::HttpState; use net_traits::FetchTaskTarget; use net_traits::request::{Origin, RedirectMode, Referer, Request, RequestMode}; use net_traits::response::{CacheState, Response, ResponseBody, ResponseType}; @@ -44,11 +45,12 @@ impl FetchTaskTarget for FetchResponseCollector { fn process_response_eof(&mut self, response: &Response) { self.sender.send(response.clone()); } + fn fetch_done(&mut self, _: &Response, _: bool) {} } fn fetch_async(request: Request, target: Box) { thread::spawn(move || { - fetch(Rc::new(request), Some(target)); + fetch(Rc::new(request), &mut Some(target), HttpState::new()); }); } @@ -72,10 +74,10 @@ fn test_fetch_response_is_not_network_error() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); if fetch_response.is_network_error() { @@ -93,10 +95,10 @@ fn test_fetch_response_body_matches_const_message() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -115,10 +117,10 @@ fn test_fetch_aboutblank() { let url = Url::parse("about:blank").unwrap(); let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); assert!(!fetch_response.is_network_error()); assert!(*fetch_response.body.lock().unwrap() == ResponseBody::Done(vec![])); } @@ -130,7 +132,7 @@ fn test_fetch_data() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); let expected_resp_body = "

Servo

".to_owned(); - let fetch_response = fetch(Rc::new(request), None); + let fetch_response = fetch(Rc::new(request), &mut None, HttpState::new()); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); @@ -159,7 +161,7 @@ fn test_fetch_file() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); - let fetch_response = fetch(Rc::new(request), None); + let fetch_response = fetch(Rc::new(request), &mut None, HttpState::new()); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); let content_type: &ContentType = fetch_response.headers.get().unwrap(); @@ -198,12 +200,12 @@ fn test_cors_preflight_fetch() { let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.use_cors_preflight = true; request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -237,14 +239,14 @@ fn test_cors_preflight_cache_fetch() { let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url.clone(), Some(origin.clone()), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.use_cors_preflight = true; request.mode = RequestMode::CORSMode; let wrapped_request0 = Rc::new(request.clone()); let wrapped_request1 = Rc::new(request); - let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache, None); - let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache, None); + let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache, &mut None, HttpState::new()); + let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache, &mut None, HttpState::new()); let _ = server.close(); assert!(!fetch_response0.is_network_error() && !fetch_response1.is_network_error()); @@ -287,12 +289,12 @@ fn test_cors_preflight_fetch_network_error() { let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); *request.method.borrow_mut() = Method::Extension("CHICKEN".to_owned()); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.use_cors_preflight = true; request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); assert!(fetch_response.is_network_error()); @@ -312,10 +314,10 @@ fn test_fetch_response_is_basic_filtered() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -359,11 +361,11 @@ fn test_fetch_response_is_cors_filtered() { // an origin mis-match will stop it from defaulting to a basic filtered response let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -393,10 +395,10 @@ fn test_fetch_response_is_opaque_filtered() { // an origin mis-match will fall through to an Opaque filtered response let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -442,11 +444,11 @@ fn test_fetch_response_is_opaque_redirect_filtered() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.redirect_mode.set(RedirectMode::Manual); let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -478,13 +480,13 @@ fn test_fetch_with_local_urls_only() { let do_fetch = |url: Url| { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; // Set the flag. request.local_urls_only = true; let wrapped_request = Rc::new(request); - fetch(wrapped_request, None) + fetch(wrapped_request, &mut None, HttpState::new()) }; let local_url = Url::parse("about:blank").unwrap(); @@ -520,10 +522,10 @@ fn setup_server_and_fetch(message: &'static [u8], redirect_cap: u32) -> Response let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, None); + let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); fetch_response } @@ -605,11 +607,11 @@ fn test_fetch_redirect_updates_method_runner(tx: Sender, status_code: Stat let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; *request.method.borrow_mut() = method; let wrapped_request = Rc::new(request); - let _ = fetch(wrapped_request, None); + let _ = fetch(wrapped_request, &mut None, HttpState::new()); let _ = server.close(); } @@ -681,7 +683,7 @@ fn test_fetch_async_returns_complete_response() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let (tx, rx) = channel(); let listener = Box::new(FetchResponseCollector { @@ -706,7 +708,7 @@ fn test_opaque_filtered_fetch_async_returns_complete_response() { // an origin mis-match will fall through to an Opaque filtered response let origin = Origin::Origin(UrlOrigin::new_opaque()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; let (tx, rx) = channel(); let listener = Box::new(FetchResponseCollector { @@ -746,7 +748,7 @@ fn test_opaque_redirect_filtered_fetch_async_returns_complete_response() { let origin = Origin::Origin(url.origin()); let mut request = Request::new(url, Some(origin), false); - request.referer = Referer::NoReferer; + *request.referer.borrow_mut() = Referer::NoReferer; request.redirect_mode.set(RedirectMode::Manual); let (tx, rx) = channel(); From bf99e73cb0dfb9a5977bfe2a1de779390d013e8b Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 3 Jun 2016 18:37:26 +0530 Subject: [PATCH 07/12] Re-add support for fetching chunks (and thus xhr download progress) --- components/net/fetch/methods.rs | 82 ++++++++++++++++++++++--- components/net/resource_thread.rs | 4 +- components/net_traits/lib.rs | 69 +++++---------------- components/script/dom/xmlhttprequest.rs | 26 +++----- tests/unit/net/fetch.rs | 2 +- 5 files changed, 96 insertions(+), 87 deletions(-) diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index d2bab7b16ad..49a4180e03f 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -28,6 +28,7 @@ use std::collections::HashSet; use std::fs::File; use std::io::Read; use std::iter::FromIterator; +use std::mem::swap; use std::rc::Rc; use std::sync::mpsc::{channel, Sender, Receiver}; use unicase::UniCase; @@ -36,7 +37,12 @@ use util::thread::spawn_named; pub type Target = Option>; -type DoneChannel = Option<(Sender<()>, Receiver<()>)>; +enum Data { + Payload(Vec), + Done, +} + +type DoneChannel = Option<(Sender, Receiver)>; /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch) pub fn fetch(request: Rc, target: &mut Target, state: HttpState) -> Response { @@ -258,8 +264,38 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, // Step 18 if request.synchronous { + if let Some(ref mut target) = *target { + // process_response is not supposed to be used + // by sync fetch, but we overload it here for simplicity + target.process_response(&response); + } + if let Some(ref ch) = *done_chan { - let _ = ch.1.recv(); + loop { + match ch.1.recv() + .expect("fetch worker should always send Done before terminating") { + Data::Payload(vec) => { + if let Some(ref mut target) = *target { + target.process_response_chunk(vec); + } + } + Data::Done => break, + } + } + } else { + if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { + // in case there was no channel to wait for, the body was + // obtained synchronously via basic_fetch for data/file/about/etc + // We should still send the body across as a chunk + if let Some(ref mut target) = *target { + target.process_response_chunk(vec.clone()); + } + } + } + + // overloaded similarly to process_response + if let Some(ref mut target) = *target { + target.process_response_eof(&response); } return response; } @@ -283,7 +319,26 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, // Step 21 if let Some(ref ch) = *done_chan { - let _ = ch.1.recv(); + loop { + match ch.1.recv() + .expect("fetch worker should always send Done before terminating") { + Data::Payload(vec) => { + if let Some(ref mut target) = *target { + target.process_response_chunk(vec); + } + } + Data::Done => break, + } + } + } else { + if let Some(ref mut target) = *target { + if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { + // in case there was no channel to wait for, the body was + // obtained synchronously via basic_fetch for data/file/about/etc + // We should still send the body across as a chunk + target.process_response_chunk(vec.clone()); + } + } } // Step 22 @@ -876,19 +931,28 @@ fn http_network_fetch(request: Rc, loop { match read_block(&mut res.response) { - Ok(ReadResult::Payload(ref mut chunk)) => { + Ok(ReadResult::Payload(chunk)) => { if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { - body.append(chunk); + + body.extend_from_slice(&chunk); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Payload(chunk)); + } } }, Ok(ReadResult::EOF) | Err(_) => { + let mut empty_vec = Vec::new(); let completed_body = match *res_body.lock().unwrap() { - ResponseBody::Receiving(ref body) => (*body).clone(), - _ => vec![] + ResponseBody::Receiving(ref mut body) => { + // avoid cloning the body + swap(body, &mut empty_vec); + empty_vec + }, + _ => empty_vec, }; *res_body.lock().unwrap() = ResponseBody::Done(completed_body); - if let Some(sender) = done_sender { - let _ = sender.send(()); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Done); } break; } diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index 18857e5005c..c7c6000844a 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -494,15 +494,13 @@ impl CoreResourceManager { blocked_content: BLOCKED_CONTENT_RULES.clone(), }; spawn_named(format!("fetch thread for {}", init.url), move || { - let sync = init.synchronous; let request = Request::from_init(init); // XXXManishearth: Check origin against pipeline id // todo load context / mimesniff in fetch // todo referrer policy? // todo service worker stuff let mut target = Some(Box::new(sender) as Box); - let response = fetch(Rc::new(request), &mut target, http_state); - target.unwrap().fetch_done(&response, sync); + fetch(Rc::new(request), &mut target, http_state); }) } diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index 59656fbea9d..0f3cd6a34ac 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -37,7 +37,7 @@ use hyper::mime::{Attr, Mime}; use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use msg::constellation_msg::{PipelineId, ReferrerPolicy}; use request::{Request, RequestInit}; -use response::{HttpsState, Response, ResponseBody}; +use response::{HttpsState, Response}; use std::io::Error as IOError; use std::sync::mpsc::Sender; use std::thread; @@ -164,8 +164,8 @@ pub enum FetchResponseMsg { ProcessRequestEOF, // todo: send more info about the response (or perhaps the entire Response) ProcessResponse(Result), - ProcessResponseEOF(Result>, NetworkError>), - FetchDone(Result<(Metadata, Option>), NetworkError>), + ProcessResponseChunk(Vec), + ProcessResponseEOF(Result<(), NetworkError>), } pub trait FetchTaskTarget { @@ -184,22 +184,21 @@ pub trait FetchTaskTarget { /// Fired when headers are received fn process_response(&mut self, response: &Response); + /// Fired when a chunk of response content is received + fn process_response_chunk(&mut self, chunk: Vec); + /// https://fetch.spec.whatwg.org/#process-response-end-of-file /// /// Fired when the response is fully fetched fn process_response_eof(&mut self, response: &Response); - - /// Called when fetch terminates, useful for sync - fn fetch_done(&mut self, response: &Response, sync: bool); } pub trait FetchResponseListener { fn process_request_body(&mut self); fn process_request_eof(&mut self); fn process_response(&mut self, metadata: Result); - fn process_response_eof(&mut self, response: Result>, NetworkError>); - - fn fetch_done(&mut self, response: Result<(Metadata, Option>), NetworkError>); + fn process_response_chunk(&mut self, chunk: Vec); + fn process_response_eof(&mut self, response: Result<(), NetworkError>); } impl FetchTaskTarget for IpcSender { @@ -214,57 +213,17 @@ impl FetchTaskTarget for IpcSender { fn process_response(&mut self, response: &Response) { let _ = self.send(FetchResponseMsg::ProcessResponse(response.metadata())); } + fn process_response_chunk(&mut self, chunk: Vec) { + let _ = self.send(FetchResponseMsg::ProcessResponseChunk(chunk)); + } fn process_response_eof(&mut self, response: &Response) { if response.is_network_error() { // todo: finer grained errors let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Err(NetworkError::Internal("Network error".into())))); + } else { + let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(()))); } - if let Ok(ref guard) = response.body.lock() { - match **guard { - ResponseBody::Done(ref vec) => { - let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(Some(vec.clone())))); - return; - } - ResponseBody::Empty => { - let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(None))); - return; - } - _ => () - } - } - - // If something goes wrong, log it instead of crashing the resource thread - let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Err(NetworkError::Internal("Incomplete body".into())))); - } - - fn fetch_done(&mut self, response: &Response, sync: bool) { - if !sync { - // fetch_done is only used by sync XHR, avoid pointless data cloning - return; - } - if response.is_network_error() { - // todo: finer grained errors - let _ = self.send(FetchResponseMsg::FetchDone(Err(NetworkError::Internal("Network error".into())))); - } - if let Ok(ref guard) = response.body.lock() { - match **guard { - ResponseBody::Done(ref vec) => { - let ret = response.metadata().map(|m| (m, Some(vec.clone()))); - let _ = self.send(FetchResponseMsg::FetchDone(ret)); - return; - } - ResponseBody::Empty => { - let ret = response.metadata().map(|m| (m, None)); - let _ = self.send(FetchResponseMsg::FetchDone(ret)); - return; - } - _ => () - } - } - - // If something goes wrong, log it instead of crashing the resource thread - let _ = self.send(FetchResponseMsg::FetchDone(Err(NetworkError::Internal("Incomplete body".into())))); } } @@ -315,8 +274,8 @@ impl Action for FetchResponseMsg { FetchResponseMsg::ProcessRequestBody => listener.process_request_body(), FetchResponseMsg::ProcessRequestEOF => listener.process_request_eof(), FetchResponseMsg::ProcessResponse(meta) => listener.process_response(meta), + FetchResponseMsg::ProcessResponseChunk(data) => listener.process_response_chunk(data), FetchResponseMsg::ProcessResponseEOF(data) => listener.process_response_eof(data), - FetchResponseMsg::FetchDone(response) => listener.fetch_done(response), } } } diff --git a/components/script/dom/xmlhttprequest.rs b/components/script/dom/xmlhttprequest.rs index 7425a2feff3..5a0866cc1bf 100644 --- a/components/script/dom/xmlhttprequest.rs +++ b/components/script/dom/xmlhttprequest.rs @@ -47,7 +47,7 @@ use net_traits::CoreResourceMsg::Fetch; use net_traits::trim_http_whitespace; use net_traits::{FetchResponseListener, Metadata, NetworkError, RequestSource}; use net_traits::{CoreResourceThread, LoadOrigin}; -use net_traits::request::{CredentialsMode, Destination, RequestInit, RequestMode, Origin}; +use net_traits::request::{CredentialsMode, Destination, RequestInit, RequestMode}; use network_listener::{NetworkListener, PreInvoke}; use parse::html::{ParseContext, parse_html}; use parse::xml::{self, parse_xml}; @@ -230,14 +230,13 @@ impl XMLHttpRequest { *self.sync_status.borrow_mut() = Some(rv); } } - fn process_response_eof(&mut self, response: Result>, NetworkError>) { + fn process_response_chunk(&mut self, mut chunk: Vec) { + self.buf.borrow_mut().append(&mut chunk); + self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); + } + fn process_response_eof(&mut self, response: Result<(), NetworkError>) { match response { - Ok(buf) => { - if let Some(buf) = buf { - *self.buf.borrow_mut() = buf; - // todo move to a process_chunk - self.xhr.root().process_data_available(self.gen_id, self.buf.borrow().clone()); - } + Ok(()) => { let rv = self.xhr.root().process_response_complete(self.gen_id, Ok(())); *self.sync_status.borrow_mut() = Some(rv); } @@ -247,17 +246,6 @@ impl XMLHttpRequest { } } } - fn fetch_done(&mut self, response: Result<(Metadata, Option>), NetworkError>) { - match response { - Ok(response) => { - self.process_response(Ok(response.0)); - self.process_response_eof(Ok(response.1)); - } - Err(err) => { - self.process_response_eof(Err(err)); - } - } - } } impl PreInvoke for XHRContext { diff --git a/tests/unit/net/fetch.rs b/tests/unit/net/fetch.rs index 3eb0ff78567..1439d6d42fe 100644 --- a/tests/unit/net/fetch.rs +++ b/tests/unit/net/fetch.rs @@ -41,11 +41,11 @@ impl FetchTaskTarget for FetchResponseCollector { fn process_request_body(&mut self, _: &Request) {} fn process_request_eof(&mut self, _: &Request) {} fn process_response(&mut self, _: &Response) {} + fn process_response_chunk(&mut self, _: Vec) {} /// Fired when the response is fully fetched fn process_response_eof(&mut self, response: &Response) { self.sender.send(response.clone()); } - fn fetch_done(&mut self, _: &Response, _: bool) {} } fn fetch_async(request: Request, target: Box) { From 909b99f1a6735d6a9dae9b92a6d23b1c5557fef3 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 3 Jun 2016 19:58:35 +0530 Subject: [PATCH 08/12] Modify request headers for fetch --- components/net/fetch/methods.rs | 25 ++++++++++++++++++++----- components/net/http_loader.rs | 33 ++++++++++++++------------------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 49a4180e03f..996a7baec28 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -5,13 +5,13 @@ use connector::create_http_connector; use data_loader::decode; use fetch::cors_cache::CORSCache; -use http_loader::{HttpState, set_request_cookies}; +use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies}; use http_loader::{NetworkHttpRequestFactory, ReadResult, obtain_response, read_block}; use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials}; use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods}; use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic}; use hyper::header::{CacheControl, CacheDirective, ContentEncoding, ContentLength, ContentLanguage, ContentType}; -use hyper::header::{Encoding, HeaderView, Headers, IfMatch, IfRange, IfUnmodifiedSince, IfModifiedSince}; +use hyper::header::{Encoding, HeaderView, Headers, Host, IfMatch, IfRange, IfUnmodifiedSince, IfModifiedSince}; use hyper::header::{IfNoneMatch, Pragma, Location, QualityItem, Referer as RefererHeader, UserAgent, q, qitem}; use hyper::method::Method; use hyper::mime::{Mime, SubLevel, TopLevel}; @@ -766,15 +766,31 @@ fn http_network_or_cache_fetch(request: Rc, _ => {} } + let current_url = http_request.current_url(); // Step 12 - // modify_request_headers(http_request.headers.borrow()); + // todo: pass referrer url and policy + // this can only be uncommented when the referer header is set, else it crashes + // in the meantime, we manually set the headers in the block below + // modify_request_headers(&mut http_request.headers.borrow_mut(), ¤t_url, + // None, None, None); + { + let headers = &mut *http_request.headers.borrow_mut(); + let host = Host { + hostname: current_url.host_str().unwrap().to_owned(), + port: current_url.port_or_known_default() + }; + headers.set(host); + // accept header should not be set here, unlike http + set_default_accept_encoding(headers); + } // Step 13 // TODO some of this step can't be implemented yet if credentials_flag { // Substep 1 // TODO http://mxr.mozilla.org/servo/source/components/net/http_loader.rs#504 - set_request_cookies(&http_request.current_url(), + // XXXManishearth http_loader has block_cookies, should we do this too? + set_request_cookies(¤t_url, &mut *http_request.headers.borrow_mut(), &state.cookie_jar); // Substep 2 @@ -787,7 +803,6 @@ fn http_network_or_cache_fetch(request: Rc, // Substep 5 if authentication_fetch_flag { - let current_url = http_request.current_url(); authorization_value = if has_credentials(¤t_url) { Some(Basic { diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index e62fc2dd415..43ae8af7fa3 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -370,7 +370,7 @@ impl Error for LoadErrorType { } } -fn set_default_accept_encoding(headers: &mut Headers) { +pub fn set_default_accept_encoding(headers: &mut Headers) { if headers.has::() { return } @@ -627,10 +627,7 @@ fn request_must_be_secured(url: &Url, hsts_list: &Arc>) -> bool pub fn modify_request_headers(headers: &mut Headers, url: &Url, user_agent: &str, - cookie_jar: &Arc>, - auth_cache: &Arc>, - load_data: &LoadData, - block_cookies: bool, + referrer_policy: Option, referrer_url: &mut Option) { // Ensure that the host header is set from the original url let host = Host { @@ -654,23 +651,13 @@ pub fn modify_request_headers(headers: &mut Headers, set_default_accept_encoding(headers); *referrer_url = determine_request_referrer(headers, - load_data.referrer_policy.clone(), + referrer_policy.clone(), referrer_url.clone(), url.clone()); if let Some(referer_val) = referrer_url.clone() { headers.set(Referer(referer_val.into_string())); } - - // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11 - if load_data.credentials_flag { - if !block_cookies { - set_request_cookies(&url, headers, cookie_jar); - } - - // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12 - set_auth_header(headers, url, auth_cache); - } } fn set_auth_header(headers: &mut Headers, @@ -956,10 +943,18 @@ pub fn load(load_data: &LoadData, let request_id = uuid::Uuid::new_v4().simple().to_string(); modify_request_headers(&mut request_headers, &doc_url, - &user_agent, &http_state.cookie_jar, - &http_state.auth_cache, &load_data, - block_cookies, &mut referrer_url); + &user_agent, load_data.referrer_policy, + &mut referrer_url); + // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11 + if load_data.credentials_flag { + if !block_cookies { + set_request_cookies(&doc_url, &mut request_headers, &http_state.cookie_jar); + } + + // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12 + set_auth_header(&mut request_headers, &doc_url, &http_state.auth_cache); + } //if there is a new auth header then set the request headers with it if let Some(ref auth_header) = new_auth_header { request_headers.set(auth_header.clone()); From f4e3e8e38e0db0cbc958019cc8ab182e3e755b26 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 6 Jun 2016 13:20:11 +0530 Subject: [PATCH 09/12] Re-support gzip --- components/net/fetch/methods.rs | 65 +++++++++++++++++++-------------- components/net/http_loader.rs | 2 +- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 996a7baec28..90d2d24cfe1 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -6,7 +6,7 @@ use connector::create_http_connector; use data_loader::decode; use fetch::cors_cache::CORSCache; use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies}; -use http_loader::{NetworkHttpRequestFactory, ReadResult, obtain_response, read_block}; +use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block}; use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials}; use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods}; use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic}; @@ -940,39 +940,50 @@ fn http_network_fetch(request: Rc, // We're about to spawn a thread to be waited on here *done_chan = Some(channel()); + let meta = response.metadata().expect("Response metadata should exist at this stage"); let done_sender = done_chan.as_ref().map(|ch| ch.0.clone()); spawn_named(format!("fetch worker thread"), move || { - *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); + match StreamedResponse::from_http_response(box res, meta) { + Ok(mut res) => { + *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); + loop { + match read_block(&mut res) { + Ok(ReadResult::Payload(chunk)) => { + if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { - loop { - match read_block(&mut res.response) { - Ok(ReadResult::Payload(chunk)) => { - if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { - - body.extend_from_slice(&chunk); - if let Some(ref sender) = done_sender { - let _ = sender.send(Data::Payload(chunk)); + body.extend_from_slice(&chunk); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Payload(chunk)); + } + } + }, + Ok(ReadResult::EOF) | Err(_) => { + let mut empty_vec = Vec::new(); + let completed_body = match *res_body.lock().unwrap() { + ResponseBody::Receiving(ref mut body) => { + // avoid cloning the body + swap(body, &mut empty_vec); + empty_vec + }, + _ => empty_vec, + }; + *res_body.lock().unwrap() = ResponseBody::Done(completed_body); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Done); + } + break; } } - }, - Ok(ReadResult::EOF) | Err(_) => { - let mut empty_vec = Vec::new(); - let completed_body = match *res_body.lock().unwrap() { - ResponseBody::Receiving(ref mut body) => { - // avoid cloning the body - swap(body, &mut empty_vec); - empty_vec - }, - _ => empty_vec, - }; - *res_body.lock().unwrap() = ResponseBody::Done(completed_body); - if let Some(ref sender) = done_sender { - let _ = sender.send(Data::Done); - } - break; + + } + } + Err(_) => { + // XXXManishearth we should propagate this error somehow + *res_body.lock().unwrap() = ResponseBody::Done(vec![]); + if let Some(ref sender) = done_sender { + let _ = sender.send(Data::Done); } } - } }); }, diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index 43ae8af7fa3..d647aa16717 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -540,7 +540,7 @@ impl StreamedResponse { StreamedResponse { metadata: m, decoder: d } } - fn from_http_response(response: Box, m: Metadata) -> Result { + pub fn from_http_response(response: Box, m: Metadata) -> Result { let decoder = match response.content_encoding() { Some(Encoding::Gzip) => { let result = GzDecoder::new(response); From fd6f9bd4110e9fa725a257148d537fa6f0392a61 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 6 Jun 2016 18:58:50 +0530 Subject: [PATCH 10/12] Test fixes; update for changes in spec --- components/net/fetch/methods.rs | 248 +++++++++--------- components/net/http_loader.rs | 20 +- components/net/resource_thread.rs | 14 +- components/net_traits/lib.rs | 4 +- components/net_traits/request.rs | 40 ++- components/net_traits/response.rs | 2 +- components/script/dom/xmlhttprequest.rs | 83 +++++- components/script/network_listener.rs | 6 +- tests/unit/net/fetch.rs | 44 ++-- ...-setrequestheader-existing-session.htm.ini | 5 - .../send-authentication-basic.htm.ini | 5 - .../XMLHttpRequest/send-conditional.htm.ini | 8 + .../send-content-type-charset.htm.ini | 9 +- .../send-entity-body-get-head.htm.ini | 5 - .../send-redirect-to-cors.htm.ini | 14 + .../send-redirect-to-non-cors.htm.ini | 14 - tests/wpt/metadata/cors/allow-headers.htm.ini | 3 - tests/wpt/metadata/cors/origin.htm.ini | 9 - .../wpt/metadata/cors/preflight-cache.htm.ini | 9 - .../wpt/metadata/cors/redirect-origin.htm.ini | 36 --- .../cors/redirect-preflight-2.htm.ini | 8 - .../metadata/cors/response-headers.htm.ini | 3 - 22 files changed, 320 insertions(+), 269 deletions(-) delete mode 100644 tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini delete mode 100644 tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini create mode 100644 tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini delete mode 100644 tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini create mode 100644 tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini delete mode 100644 tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini delete mode 100644 tests/wpt/metadata/cors/redirect-preflight-2.htm.ini diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 90d2d24cfe1..305eb1bdf83 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -7,6 +7,7 @@ use data_loader::decode; use fetch::cors_cache::CORSCache; use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies}; use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block}; +use http_loader::{auth_from_cache, determine_request_referrer}; use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials}; use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods}; use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic}; @@ -17,6 +18,7 @@ use hyper::method::Method; use hyper::mime::{Mime, SubLevel, TopLevel}; use hyper::status::StatusCode; use mime_guess::guess_mime_type; +use msg::constellation_msg::ReferrerPolicy; use net_traits::FetchTaskTarget; use net_traits::request::{CacheMode, CredentialsMode}; use net_traits::request::{RedirectMode, Referer, Request, RequestMode, ResponseTainting}; @@ -42,17 +44,22 @@ enum Data { Done, } +pub struct FetchContext { + pub state: HttpState, + pub user_agent: String, +} + type DoneChannel = Option<(Sender, Receiver)>; /// [Fetch](https://fetch.spec.whatwg.org#concept-fetch) -pub fn fetch(request: Rc, target: &mut Target, state: HttpState) -> Response { - fetch_with_cors_cache(request, &mut CORSCache::new(), target, state) +pub fn fetch(request: Rc, target: &mut Target, context: FetchContext) -> Response { + fetch_with_cors_cache(request, &mut CORSCache::new(), target, context) } pub fn fetch_with_cors_cache(request: Rc, cache: &mut CORSCache, target: &mut Target, - state: HttpState) -> Response { + context: FetchContext) -> Response { // Step 1 if request.window.get() == Window::Client { // TODO: Set window to request's client object if client is a Window object @@ -113,13 +120,13 @@ pub fn fetch_with_cors_cache(request: Rc, } // Step 7 - main_fetch(request, cache, false, false, target, &mut None, &state) + main_fetch(request, cache, false, false, target, &mut None, &context) } /// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch) fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel, - state: &HttpState) -> Response { + context: &FetchContext) -> Response { // TODO: Implement main fetch spec // Step 1 @@ -142,14 +149,25 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, // Step 5 // TODO this step (CSP port/content blocking) - // Step 6-7 + // Step 6 // TODO this step (referer policy) + // currently the clients themselves set referer policy in RequestInit + + // Step 7 + if request.referrer_policy.get().is_none() { + request.referrer_policy.set(Some(ReferrerPolicy::NoRefWhenDowngrade)); + } // Step 8 if *request.referer.borrow() != Referer::NoReferer { - // TODO be able to invoke "determine request's referer" - // once this is filled in be sure to update the match - // referer below to have an unreachable branch for client + // remove Referer headers set in past redirects/preflights + // this stops the assertion in determine_request_referrer from failing + request.headers.borrow_mut().remove::(); + let referrer_url = determine_request_referrer(&mut *request.headers.borrow_mut(), + request.referrer_policy.get(), + request.referer.borrow_mut().take(), + request.current_url().clone()); + *request.referer.borrow_mut() = Referer::from_url(referrer_url); } // Step 9 @@ -174,14 +192,14 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, (current_url.scheme() == "file" && request.same_origin_data.get()) || current_url.scheme() == "about" || request.mode == RequestMode::Navigate { - basic_fetch(request.clone(), cache, target, done_chan, state) + basic_fetch(request.clone(), cache, target, done_chan, context) } else if request.mode == RequestMode::SameOrigin { Response::network_error() } else if request.mode == RequestMode::NoCORS { request.response_tainting.set(ResponseTainting::Opaque); - basic_fetch(request.clone(), cache, target, done_chan, state) + basic_fetch(request.clone(), cache, target, done_chan, context) } else if !matches!(current_url.scheme(), "http" | "https") { Response::network_error() @@ -192,7 +210,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) { request.response_tainting.set(ResponseTainting::CORSTainting); request.redirect_mode.set(RedirectMode::Error); - let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, state); + let response = http_fetch(request.clone(), cache, true, true, false, target, done_chan, context); if response.is_network_error() { // TODO clear cache entries using request } @@ -200,7 +218,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, } else { request.response_tainting.set(ResponseTainting::CORSTainting); - http_fetch(request.clone(), cache, true, false, false, target, done_chan, state) + http_fetch(request.clone(), cache, true, false, false, target, done_chan, context) } } }; @@ -233,9 +251,14 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, }; // Step 15 - // TODO this step (CSP/blocking) + if internal_response.url_list.borrow().is_empty() { + *internal_response.url_list.borrow_mut() = request.url_list.borrow().clone(); + } // Step 16 + // TODO this step (CSP/blocking) + + // Step 17 if !response.is_network_error() && (is_null_body_status(&internal_response.status) || match *request.method.borrow() { Method::Head | Method::Connect => true, @@ -247,7 +270,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, *body = ResponseBody::Empty; } - // Step 17 + // Step 18 // TODO be able to compare response integrity against request integrity metadata // if !response.is_network_error() { @@ -262,7 +285,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, // } } - // Step 18 + // Step 19 if request.synchronous { if let Some(ref mut target) = *target { // process_response is not supposed to be used @@ -282,14 +305,12 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, Data::Done => break, } } - } else { - if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { - // in case there was no channel to wait for, the body was - // obtained synchronously via basic_fetch for data/file/about/etc - // We should still send the body across as a chunk - if let Some(ref mut target) = *target { - target.process_response_chunk(vec.clone()); - } + } else if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { + // in case there was no channel to wait for, the body was + // obtained synchronously via basic_fetch for data/file/about/etc + // We should still send the body across as a chunk + if let Some(ref mut target) = *target { + target.process_response_chunk(vec.clone()); } } @@ -300,7 +321,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, return response; } - // Step 19 + // Step 20 if request.body.borrow().is_some() && matches!(request.current_url().scheme(), "http" | "https") { if let Some(ref mut target) = *target { // XXXManishearth: We actually should be calling process_request @@ -312,12 +333,12 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, } } - // Step 20 + // Step 21 if let Some(ref mut target) = *target { target.process_response(&response); } - // Step 21 + // Step 22 if let Some(ref ch) = *done_chan { loop { match ch.1.recv() @@ -330,21 +351,19 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, Data::Done => break, } } - } else { - if let Some(ref mut target) = *target { - if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { - // in case there was no channel to wait for, the body was - // obtained synchronously via basic_fetch for data/file/about/etc - // We should still send the body across as a chunk - target.process_response_chunk(vec.clone()); - } + } else if let Some(ref mut target) = *target { + if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() { + // in case there was no channel to wait for, the body was + // obtained synchronously via basic_fetch for data/file/about/etc + // We should still send the body across as a chunk + target.process_response_chunk(vec.clone()); } } - // Step 22 + // Step 23 request.done.set(true); - // Step 23 + // Step 24 if let Some(ref mut target) = *target { target.process_response_eof(&response); } @@ -356,7 +375,7 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, /// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch) fn basic_fetch(request: Rc, cache: &mut CORSCache, target: &mut Target, done_chan: &mut DoneChannel, - state: &HttpState) -> Response { + context: &FetchContext) -> Response { let url = request.current_url(); match url.scheme() { @@ -370,7 +389,7 @@ fn basic_fetch(request: Rc, cache: &mut CORSCache, }, "http" | "https" => { - http_fetch(request.clone(), cache, false, false, false, target, done_chan, state) + http_fetch(request.clone(), cache, false, false, false, target, done_chan, context) }, "data" => { @@ -432,8 +451,7 @@ fn http_fetch(request: Rc, authentication_fetch_flag: bool, target: &mut Target, done_chan: &mut DoneChannel, - state: &HttpState) -> Response { - + context: &FetchContext) -> Response { // This is a new async fetch, reset the channel we are waiting on *done_chan = None; // Step 1 @@ -463,17 +481,18 @@ fn http_fetch(request: Rc, } // Substep 4 - let actual_response = res.actual_response(); - if actual_response.url_list.borrow().is_empty() { - *actual_response.url_list.borrow_mut() = request.url_list.borrow().clone(); - } - - // Substep 5 // TODO: set response's CSP list on actual_response } } // Step 4 + let credentials = match request.credentials_mode { + CredentialsMode::Include => true, + CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic + => true, + _ => false + }; + // Step 5 if response.is_none() { // Substep 1 if cors_preflight_flag { @@ -488,7 +507,7 @@ fn http_fetch(request: Rc, // Sub-substep 1 if method_mismatch || header_mismatch { - let preflight_result = cors_preflight_fetch(request.clone(), cache, state); + let preflight_result = cors_preflight_fetch(request.clone(), cache, context); // Sub-substep 2 if preflight_result.response_type == ResponseType::Error { return Response::network_error(); @@ -500,18 +519,10 @@ fn http_fetch(request: Rc, request.skip_service_worker.set(true); // Substep 3 - let credentials = match request.credentials_mode { - CredentialsMode::Include => true, - CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic - => true, - _ => false - }; + let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag, + done_chan, context); // Substep 4 - let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag, - done_chan, state); - - // Substep 5 if cors_flag && cors_check(request.clone(), &fetch_result).is_err() { return Response::network_error(); } @@ -537,7 +548,7 @@ fn http_fetch(request: Rc, // set back to default response.return_internal.set(true); http_redirect_fetch(request, cache, Rc::new(response), - cors_flag, target, done_chan, state) + cors_flag, target, done_chan, context) } } }, @@ -546,7 +557,7 @@ fn http_fetch(request: Rc, StatusCode::Unauthorized => { // Step 1 // FIXME: Figure out what to do with request window objects - if cors_flag || request.credentials_mode != CredentialsMode::Include { + if cors_flag || !credentials { return response; } @@ -556,11 +567,15 @@ fn http_fetch(request: Rc, // Step 3 if !request.use_url_credentials || authentication_fetch_flag { // TODO: Prompt the user for username and password from the window + // Wrong, but will have to do until we are able to prompt the user + // otherwise this creates an infinite loop + // We basically pretend that the user declined to enter credentials + return response; } // Step 4 return http_fetch(request, cache, cors_flag, cors_preflight_flag, - true, target, done_chan, state); + true, target, done_chan, context); } // Code 407 @@ -573,12 +588,16 @@ fn http_fetch(request: Rc, // Step 3 // TODO: Prompt the user for proxy authentication credentials + // Wrong, but will have to do until we are able to prompt the user + // otherwise this creates an infinite loop + // We basically pretend that the user declined to enter credentials + return response; // Step 4 - return http_fetch(request, cache, - cors_flag, cors_preflight_flag, - authentication_fetch_flag, target, - done_chan, state); + // return http_fetch(request, cache, + // cors_flag, cors_preflight_flag, + // authentication_fetch_flag, target, + // done_chan, context); } _ => { } @@ -602,7 +621,7 @@ fn http_redirect_fetch(request: Rc, cors_flag: bool, target: &mut Target, done_chan: &mut DoneChannel, - state: &HttpState) -> Response { + context: &FetchContext) -> Response { // Step 1 assert_eq!(response.return_internal.get(), true); @@ -676,7 +695,7 @@ fn http_redirect_fetch(request: Rc, request.url_list.borrow_mut().push(location_url); // Step 15 - main_fetch(request, cache, cors_flag, true, target, done_chan, state) + main_fetch(request, cache, cors_flag, true, target, done_chan, context) } /// [HTTP network or cache fetch](https://fetch.spec.whatwg.org#http-network-or-cache-fetch) @@ -684,14 +703,14 @@ fn http_network_or_cache_fetch(request: Rc, credentials_flag: bool, authentication_fetch_flag: bool, done_chan: &mut DoneChannel, - state: &HttpState) -> Response { + context: &FetchContext) -> Response { // TODO: Implement Window enum for Request let request_has_no_window = true; // Step 1 let http_request = if request_has_no_window && - request.redirect_mode.get() != RedirectMode::Follow { - request.clone() + request.redirect_mode.get() == RedirectMode::Error { + request } else { Rc::new((*request).clone()) }; @@ -716,16 +735,13 @@ fn http_network_or_cache_fetch(request: Rc, // Step 6 match *http_request.referer.borrow() { - // Referer::Client should not be here, but we don't set - // the referer yet, so we club it with NoReferer - Referer::NoReferer | Referer::Client => - http_request.headers.borrow_mut().set(RefererHeader("".to_owned())), + Referer::NoReferer => (), Referer::RefererUrl(ref http_request_referer) => http_request.headers.borrow_mut().set(RefererHeader(http_request_referer.to_string())), - // Referer::Client => - // // it should be impossible for referer to be anything else during fetching - // // https://fetch.spec.whatwg.org/#concept-request-referrer - // unreachable!() + Referer::Client => + // it should be impossible for referer to be anything else during fetching + // https://fetch.spec.whatwg.org/#concept-request-referrer + unreachable!() }; // Step 7 @@ -736,7 +752,7 @@ fn http_network_or_cache_fetch(request: Rc, // Step 8 if !http_request.headers.borrow().has::() { - http_request.headers.borrow_mut().set(UserAgent(global_user_agent().to_owned())); + http_request.headers.borrow_mut().set(UserAgent(context.user_agent.clone())); } match http_request.cache_mode.get() { @@ -780,7 +796,8 @@ fn http_network_or_cache_fetch(request: Rc, port: current_url.port_or_known_default() }; headers.set(host); - // accept header should not be set here, unlike http + // unlike http_loader, we should not set the accept header + // here, according to the fetch spec set_default_accept_encoding(headers); } @@ -789,28 +806,27 @@ fn http_network_or_cache_fetch(request: Rc, if credentials_flag { // Substep 1 // TODO http://mxr.mozilla.org/servo/source/components/net/http_loader.rs#504 - // XXXManishearth http_loader has block_cookies, should we do this too? + // XXXManishearth http_loader has block_cookies: support content blocking here too set_request_cookies(¤t_url, &mut *http_request.headers.borrow_mut(), - &state.cookie_jar); + &context.state.cookie_jar); // Substep 2 if !http_request.headers.borrow().has::>() { // Substep 3 let mut authorization_value = None; // Substep 4 - // TODO be able to retrieve https://fetch.spec.whatwg.org/#authentication-entry - - // Substep 5 - if authentication_fetch_flag { - - authorization_value = if has_credentials(¤t_url) { - Some(Basic { + if let Some(basic) = auth_from_cache(&context.state.auth_cache, ¤t_url) { + if !http_request.use_url_credentials || !has_credentials(¤t_url) { + authorization_value = Some(basic); + } + } else if authentication_fetch_flag { + // Substep 5 + if has_credentials(¤t_url) { + authorization_value = Some(Basic { username: current_url.username().to_owned(), password: current_url.password().map(str::to_owned) }) - } else { - None } } @@ -866,7 +882,7 @@ fn http_network_or_cache_fetch(request: Rc, // Step 18 if response.is_none() { - response = Some(http_network_fetch(http_request.clone(), http_request.clone(), credentials_flag, done_chan)); + response = Some(http_network_fetch(http_request.clone(), credentials_flag, done_chan)); } let response = response.unwrap(); @@ -901,7 +917,6 @@ fn http_network_or_cache_fetch(request: Rc, /// [HTTP network fetch](https://fetch.spec.whatwg.org/#http-network-fetch) fn http_network_fetch(request: Rc, - _http_request: Rc, _credentials_flag: bool, done_chan: &mut DoneChannel) -> Response { // TODO: Implement HTTP network fetch spec @@ -926,12 +941,12 @@ fn http_network_fetch(request: Rc, let wrapped_response = obtain_response(&factory, &url, &request.method.borrow(), &request.headers.borrow(), &cancellation_listener, &request.body.borrow(), &request.method.borrow(), - &None, request.redirect_count.get()+1, &None, ""); + &None, request.redirect_count.get() + 1, &None, ""); let mut response = Response::new(); match wrapped_response { - Ok((mut res, _)) => { - response.url = Some(res.response.url.clone()); + Ok((res, _)) => { + response.url = Some(url.clone()); response.status = Some(res.response.status); response.raw_status = Some(res.response.status_raw().clone()); response.headers = res.response.headers.clone(); @@ -940,17 +955,16 @@ fn http_network_fetch(request: Rc, // We're about to spawn a thread to be waited on here *done_chan = Some(channel()); - let meta = response.metadata().expect("Response metadata should exist at this stage"); + let meta = response.metadata().expect("Response metadata should exist at this stage"); let done_sender = done_chan.as_ref().map(|ch| ch.0.clone()); spawn_named(format!("fetch worker thread"), move || { match StreamedResponse::from_http_response(box res, meta) { Ok(mut res) => { - *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); + *res_body.lock().unwrap() = ResponseBody::Receiving(vec![]); loop { match read_block(&mut res) { Ok(ReadResult::Payload(chunk)) => { if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() { - body.extend_from_slice(&chunk); if let Some(ref sender) = done_sender { let _ = sender.send(Data::Payload(chunk)); @@ -1003,7 +1017,10 @@ fn http_network_fetch(request: Rc, // TODO Read request - // Step 5 + // Step 5-9 + // (needs stream bodies) + + // Step 10 // TODO when https://bugzilla.mozilla.org/show_bug.cgi?id=1030660 // is resolved, this step will become uneccesary // TODO this step @@ -1015,22 +1032,19 @@ fn http_network_fetch(request: Rc, } }; - // Step 6 - *response.url_list.borrow_mut() = request.url_list.borrow().clone(); + // Step 11 + // TODO this step isn't possible yet (CSP) - // Step 7 - // TODO this step isn't possible yet - - // Step 8 + // Step 12 if response.is_network_error() && request.cache_mode.get() == CacheMode::NoStore { // TODO update response in the HTTP cache for request } // TODO this step isn't possible yet - // Step 9 + // Step 13 // TODO these steps - // Step 10 + // Step 14 // Substep 1 // Substep 2 // Sub-substep 1 @@ -1039,12 +1053,12 @@ fn http_network_fetch(request: Rc, // Sub-substep 4 // Substep 3 - // Step 11 + // Step 15 response } /// [CORS preflight fetch](https://fetch.spec.whatwg.org#cors-preflight-fetch) -fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache, state: &HttpState) -> Response { +fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache, context: &FetchContext) -> Response { // Step 1 let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()), false); *preflight.method.borrow_mut() = Method::Options; @@ -1052,6 +1066,7 @@ fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache, state: &Htt preflight.type_ = request.type_.clone(); preflight.destination = request.destination.clone(); *preflight.referer.borrow_mut() = request.referer.borrow().clone(); + preflight.referrer_policy.set(preflight.referrer_policy.get()); // Step 2 preflight.headers.borrow_mut().set::( @@ -1072,7 +1087,7 @@ fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache, state: &Htt // Step 6 let preflight = Rc::new(preflight); - let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, state); + let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, context); // Step 7 if cors_check(request.clone(), &response).is_ok() && @@ -1105,12 +1120,16 @@ fn cors_preflight_fetch(request: Rc, cache: &mut CORSCache, state: &Htt } // Substep 5 + debug!("CORS check: Allowed methods: {:?}, current method: {:?}", + methods, request.method.borrow()); if methods.iter().all(|method| *method != *request.method.borrow()) && !is_simple_method(&*request.method.borrow()) { return Response::network_error(); } // Substep 6 + debug!("CORS check: Allowed headers: {:?}, current headers: {:?}", + header_names, request.headers.borrow()); let set: HashSet<&UniCase> = HashSet::from_iter(header_names.iter()); if request.headers.borrow().iter().any(|ref hv| !set.contains(&UniCase(hv.name().to_owned())) && !is_simple_header(hv)) { @@ -1183,12 +1202,6 @@ fn cors_check(request: Rc, response: &Response) -> Result<(), ()> { Err(()) } -fn global_user_agent() -> String { - // TODO have a better useragent string - const USER_AGENT_STRING: &'static str = "Servo"; - USER_AGENT_STRING.to_owned() -} - fn has_credentials(url: &Url) -> bool { !url.username().is_empty() || url.password().is_some() } @@ -1199,6 +1212,7 @@ fn is_no_store_cache(headers: &Headers) -> bool { headers.has::() } +/// https://fetch.spec.whatwg.org/#cors-safelisted-request-header fn is_simple_header(h: &HeaderView) -> bool { if h.is::() { match h.value() { diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index d647aa16717..1123af628ab 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -434,7 +434,7 @@ fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option { } /// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer -fn determine_request_referrer(headers: &mut Headers, +pub fn determine_request_referrer(headers: &mut Headers, referrer_policy: Option, referrer_url: Option, url: Url) -> Option { @@ -667,18 +667,21 @@ fn set_auth_header(headers: &mut Headers, if let Some(auth) = auth_from_url(url) { headers.set(auth); } else { - if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) { - auth_from_entry(&auth_entry, headers); + if let Some(basic) = auth_from_cache(auth_cache, url) { + headers.set(Authorization(basic)); } } } } -fn auth_from_entry(auth_entry: &AuthCacheEntry, headers: &mut Headers) { - let user_name = auth_entry.user_name.clone(); - let password = Some(auth_entry.password.clone()); - - headers.set(Authorization(Basic { username: user_name, password: password })); +pub fn auth_from_cache(auth_cache: &Arc>, url: &Url) -> Option { + if let Some(ref auth_entry) = auth_cache.read().unwrap().entries.get(url) { + let user_name = auth_entry.user_name.clone(); + let password = Some(auth_entry.password.clone()); + Some(Basic { username: user_name, password: password }) + } else { + None + } } fn auth_from_url(doc_url: &Url) -> Option> { @@ -955,6 +958,7 @@ pub fn load(load_data: &LoadData, // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12 set_auth_header(&mut request_headers, &doc_url, &http_state.auth_cache); } + //if there is a new auth header then set the request headers with it if let Some(ref auth_header) = new_auth_header { request_headers.set(auth_header.clone()); diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index c7c6000844a..6e94fd1cf03 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -11,7 +11,7 @@ use cookie; use cookie_storage::CookieStorage; use data_loader; use devtools_traits::DevtoolsControlMsg; -use fetch::methods::fetch; +use fetch::methods::{fetch, FetchContext}; use file_loader; use filemanager_thread::FileManagerThreadFactory; use hsts::HstsList; @@ -23,11 +23,11 @@ use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; use mime_classifier::{ApacheBugFlag, MIMEClassifier, NoSniffFlag}; use net_traits::LoadContext; use net_traits::ProgressMsg::Done; +use net_traits::request::{Request, RequestInit}; use net_traits::{AsyncResponseTarget, Metadata, ProgressMsg, ResponseAction, CoreResourceThread}; use net_traits::{CoreResourceMsg, CookieSource, FetchResponseMsg, FetchTaskTarget, LoadConsumer}; use net_traits::{LoadData, LoadResponse, NetworkError, ResourceId}; use net_traits::{WebSocketCommunicate, WebSocketConnectData, ResourceThreads}; -use net_traits::request::{Request, RequestInit}; use profile_traits::time::ProfilerChan; use rustc_serialize::json; use rustc_serialize::{Decodable, Encodable}; @@ -197,7 +197,7 @@ impl ResourceChannelManager { match self.from_client.recv().unwrap() { CoreResourceMsg::Load(load_data, consumer, id_sender) => self.resource_manager.load(load_data, consumer, id_sender, control_sender.clone()), - CoreResourceMsg::Fetch(init, sender) => + CoreResourceMsg::Fetch(init, sender) => self.resource_manager.fetch(init, sender), CoreResourceMsg::WebsocketConnect(connect, connect_data) => self.resource_manager.websocket_connect(connect, connect_data), @@ -486,21 +486,23 @@ impl CoreResourceManager { cancel_listener)); } - fn fetch(&self, init: RequestInit, sender: IpcSender) { + fn fetch(&self, init: RequestInit, sender: IpcSender) { let http_state = HttpState { hsts_list: self.hsts_list.clone(), cookie_jar: self.cookie_jar.clone(), auth_cache: self.auth_cache.clone(), blocked_content: BLOCKED_CONTENT_RULES.clone(), }; + let ua = self.user_agent.clone(); spawn_named(format!("fetch thread for {}", init.url), move || { let request = Request::from_init(init); - // XXXManishearth: Check origin against pipeline id + // XXXManishearth: Check origin against pipeline id (also ensure that the mode is allowed) // todo load context / mimesniff in fetch // todo referrer policy? // todo service worker stuff let mut target = Some(Box::new(sender) as Box); - fetch(Rc::new(request), &mut target, http_state); + let context = FetchContext { state: http_state, user_agent: ua }; + fetch(Rc::new(request), &mut target, context); }) } diff --git a/components/net_traits/lib.rs b/components/net_traits/lib.rs index 0f3cd6a34ac..bb8638d7c3c 100644 --- a/components/net_traits/lib.rs +++ b/components/net_traits/lib.rs @@ -213,6 +213,7 @@ impl FetchTaskTarget for IpcSender { fn process_response(&mut self, response: &Response) { let _ = self.send(FetchResponseMsg::ProcessResponse(response.metadata())); } + fn process_response_chunk(&mut self, chunk: Vec) { let _ = self.send(FetchResponseMsg::ProcessResponseChunk(chunk)); } @@ -220,7 +221,8 @@ impl FetchTaskTarget for IpcSender { fn process_response_eof(&mut self, response: &Response) { if response.is_network_error() { // todo: finer grained errors - let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Err(NetworkError::Internal("Network error".into())))); + let _ = self.send(FetchResponseMsg::ProcessResponseEOF( + Err(NetworkError::Internal("Network error".into())))); } else { let _ = self.send(FetchResponseMsg::ProcessResponseEOF(Ok(()))); } diff --git a/components/net_traits/request.rs b/components/net_traits/request.rs index 6909d7b25e7..4c7f4081f77 100644 --- a/components/net_traits/request.rs +++ b/components/net_traits/request.rs @@ -4,7 +4,9 @@ use hyper::header::Headers; use hyper::method::Method; +use msg::constellation_msg::ReferrerPolicy; use std::cell::{Cell, RefCell}; +use std::mem::swap; use url::{Origin as UrlOrigin, Url}; /// An [initiator](https://fetch.spec.whatwg.org/#concept-request-initiator) @@ -125,6 +127,9 @@ pub struct RequestInit { // this should actually be set by fetch, but fetch // doesn't have info about the client right now pub origin: Url, + // XXXManishearth these should be part of the client object + pub referer_url: Option, + pub referrer_policy: Option, } /// A [Request](https://fetch.spec.whatwg.org/#requests) as defined by the Fetch spec @@ -151,8 +156,9 @@ pub struct Request { pub origin: RefCell, pub omit_origin_header: Cell, pub same_origin_data: Cell, + /// https://fetch.spec.whatwg.org/#concept-request-referrer pub referer: RefCell, - // TODO: referrer policy + pub referrer_policy: Cell>, pub synchronous: bool, pub mode: RequestMode, pub use_cors_preflight: bool, @@ -191,6 +197,7 @@ impl Request { omit_origin_header: Cell::new(false), same_origin_data: Cell::new(false), referer: RefCell::new(Referer::Client), + referrer_policy: Cell::new(None), synchronous: false, mode: RequestMode::NoCORS, use_cors_preflight: false, @@ -220,6 +227,12 @@ impl Request { req.credentials_mode = init.credentials_mode; req.use_url_credentials = init.use_url_credentials; *req.origin.borrow_mut() = Origin::Origin(init.origin.origin()); + *req.referer.borrow_mut() = if let Some(url) = init.referer_url { + Referer::RefererUrl(url) + } else { + Referer::NoReferer + }; + req.referrer_policy.set(init.referrer_policy); req } @@ -246,6 +259,7 @@ impl Request { omit_origin_header: Cell::new(false), same_origin_data: Cell::new(false), referer: RefCell::new(Referer::Client), + referrer_policy: Cell::new(None), synchronous: false, // Step 1-2 mode: match cors_attribute_state { @@ -296,3 +310,27 @@ impl Request { } } } + +impl Referer { + pub fn to_url(&self) -> Option<&Url> { + match *self { + Referer::NoReferer | Referer::Client => None, + Referer::RefererUrl(ref url) => Some(url) + } + } + pub fn from_url(url: Option) -> Self { + if let Some(url) = url { + Referer::RefererUrl(url) + } else { + Referer::NoReferer + } + } + pub fn take(&mut self) -> Option { + let mut new = Referer::Client; + swap(self, &mut new); + match new { + Referer::NoReferer | Referer::Client => None, + Referer::RefererUrl(url) => Some(url) + } + } +} diff --git a/components/net_traits/response.rs b/components/net_traits/response.rs index 6db2228b4fe..e20b7b1dbeb 100644 --- a/components/net_traits/response.rs +++ b/components/net_traits/response.rs @@ -7,11 +7,11 @@ use hyper::header::{AccessControlExposeHeaders, ContentType, Headers}; use hyper::http::RawStatus; use hyper::status::StatusCode; -use {Metadata, NetworkError}; use std::ascii::AsciiExt; use std::cell::{Cell, RefCell}; use std::sync::{Arc, Mutex}; use url::Url; +use {Metadata, NetworkError}; /// [Response type](https://fetch.spec.whatwg.org/#concept-response-type) #[derive(Clone, PartialEq, Copy, Debug, Deserialize, Serialize)] diff --git a/components/script/dom/xmlhttprequest.rs b/components/script/dom/xmlhttprequest.rs index 5a0866cc1bf..12fba1ff2e8 100644 --- a/components/script/dom/xmlhttprequest.rs +++ b/components/script/dom/xmlhttprequest.rs @@ -36,7 +36,7 @@ use hyper::header::Headers; use hyper::header::{ContentLength, ContentType}; use hyper::http::RawStatus; use hyper::method::Method; -use hyper::mime::{self, Mime}; +use hyper::mime::{self, Mime, Attr as MimeAttr, Value as MimeValue}; use ipc_channel::ipc; use ipc_channel::router::ROUTER; use js::jsapi::JS_ClearPendingException; @@ -44,10 +44,10 @@ use js::jsapi::{JSContext, JS_ParseJSON, RootedValue}; use js::jsval::{JSVal, NullValue, UndefinedValue}; use msg::constellation_msg::{PipelineId, ReferrerPolicy}; use net_traits::CoreResourceMsg::Fetch; -use net_traits::trim_http_whitespace; -use net_traits::{FetchResponseListener, Metadata, NetworkError, RequestSource}; -use net_traits::{CoreResourceThread, LoadOrigin}; use net_traits::request::{CredentialsMode, Destination, RequestInit, RequestMode}; +use net_traits::trim_http_whitespace; +use net_traits::{CoreResourceThread, LoadOrigin}; +use net_traits::{FetchResponseListener, Metadata, NetworkError, RequestSource}; use network_listener::{NetworkListener, PreInvoke}; use parse::html::{ParseContext, parse_html}; use parse::xml::{self, parse_xml}; @@ -62,6 +62,7 @@ use string_cache::Atom; use time; use timers::{OneshotTimerCallback, OneshotTimerHandle}; use url::{Url, Position}; +use util::prefs::mozbrowser_enabled; #[derive(JSTraceable, PartialEq, Copy, Clone, HeapSizeOf)] enum XMLHttpRequestState { @@ -521,7 +522,7 @@ impl XMLHttpRequestMethods for XMLHttpRequest { Method::Get | Method::Head => None, _ => data }; - // Step 4 + // Step 4 (first half) let extracted = data.as_ref().map(|d| d.extract()); self.request_body_len.set(extracted.as_ref().map_or(0, |e| e.0.len())); @@ -572,6 +573,22 @@ impl XMLHttpRequestMethods for XMLHttpRequest { } else { unreachable!() }; + + let bypass_cross_origin_check = { + // We want to be able to do cross-origin requests in browser.html. + // If the XHR happens in a top level window and the mozbrowser + // preference is enabled, we allow bypassing the CORS check. + // This is a temporary measure until we figure out Servo privilege + // story. See https://github.com/servo/servo/issues/9582 + if let GlobalRoot::Window(win) = self.global() { + let is_root_pipeline = win.parent_info().is_none(); + let is_mozbrowser_enabled = mozbrowser_enabled(); + is_root_pipeline && is_mozbrowser_enabled + } else { + false + } + }; + let mut request = RequestInit { method: self.request_method.borrow().clone(), url: self.request_url.borrow().clone().unwrap(), @@ -589,18 +606,59 @@ impl XMLHttpRequestMethods for XMLHttpRequest { credentials_mode: credentials_mode, use_url_credentials: use_url_credentials, origin: self.global().r().get_url(), + referer_url: self.referrer_url.clone(), + referrer_policy: self.referrer_policy.clone(), }; - // XHR spec differs from http, and says UTF-8 should be in capitals, - // instead of "utf-8", which is what Hyper defaults to. So not - // using content types provided by Hyper. - let n = "content-type"; + + if bypass_cross_origin_check { + request.mode = RequestMode::Navigate; + } + + // step 4 (second half) match extracted { - Some((_, Some(ref content_type))) => - request.headers.set_raw(n.to_owned(), vec![content_type.bytes().collect()]), + Some((_, ref content_type)) => { + // this should handle Document bodies too, not just BodyInit + let encoding = if let Some(BodyInit::String(_)) = data { + // XHR spec differs from http, and says UTF-8 should be in capitals, + // instead of "utf-8", which is what Hyper defaults to. So not + // using content types provided by Hyper. + Some(MimeValue::Ext("UTF-8".to_string())) + } else { + None + }; + + let mut content_type_set = false; + if let Some(ref ct) = *content_type { + if !request.headers.has::() { + request.headers.set_raw("content-type", vec![ct.bytes().collect()]); + content_type_set = true; + } + } + + if !content_type_set { + let ct = request.headers.get::().map(|x| x.clone()); + if let Some(mut ct) = ct { + if let Some(encoding) = encoding { + for param in &mut (ct.0).2 { + if param.0 == MimeAttr::Charset { + if !param.0.as_str().eq_ignore_ascii_case(encoding.as_str()) { + *param = (MimeAttr::Charset, encoding.clone()); + } + } + } + } + // remove instead of mutate in place + // https://github.com/hyperium/hyper/issues/821 + request.headers.remove_raw("content-type"); + request.headers.set(ct); + } + } + + } _ => (), } - debug!("request_headers = {:?}", *self.request_headers.borrow()); + debug!("request.headers = {:?}", request.headers); self.fetch_time.set(time::now().to_timespec().sec); @@ -1233,7 +1291,6 @@ impl XMLHttpRequest { fn fetch(&self, init: RequestInit, global: GlobalRef) -> ErrorResult { - let xhr = Trusted::new(self); let context = Arc::new(Mutex::new(XHRContext { diff --git a/components/script/network_listener.rs b/components/script/network_listener.rs index cbaa073af25..13aa26c25e4 100644 --- a/components/script/network_listener.rs +++ b/components/script/network_listener.rs @@ -17,7 +17,7 @@ pub struct NetworkListener { } impl NetworkListener { - pub fn notify+Send+'static>(&self, action: A) { + pub fn notify + Send + 'static>(&self, action: A) { if let Err(err) = self.script_chan.send(CommonScriptMsg::RunnableMsg(NetworkEvent, box ListenerRunnable { context: self.context.clone(), action: action, @@ -51,12 +51,12 @@ pub trait PreInvoke { } /// A runnable for moving the async network events between threads. -struct ListenerRunnable+Send+'static, Listener: PreInvoke + Send> { +struct ListenerRunnable + Send + 'static, Listener: PreInvoke + Send> { context: Arc>, action: A, } -impl+Send+'static, Listener: PreInvoke + Send> Runnable for ListenerRunnable { +impl + Send + 'static, Listener: PreInvoke + Send> Runnable for ListenerRunnable { fn handler(self: Box>) { let this = *self; let mut context = this.context.lock().unwrap(); diff --git a/tests/unit/net/fetch.rs b/tests/unit/net/fetch.rs index 1439d6d42fe..a9cd5bd5ceb 100644 --- a/tests/unit/net/fetch.rs +++ b/tests/unit/net/fetch.rs @@ -14,7 +14,7 @@ use hyper::server::{Request as HyperRequest, Response as HyperResponse}; use hyper::status::StatusCode; use hyper::uri::RequestUri; use net::fetch::cors_cache::CORSCache; -use net::fetch::methods::{fetch, fetch_with_cors_cache}; +use net::fetch::methods::{FetchContext, fetch, fetch_with_cors_cache}; use net::http_loader::HttpState; use net_traits::FetchTaskTarget; use net_traits::request::{Origin, RedirectMode, Referer, Request, RequestMode}; @@ -22,10 +22,10 @@ use net_traits::response::{CacheState, Response, ResponseBody, ResponseType}; use std::fs::File; use std::io::Read; use std::rc::Rc; -use std::thread; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::{Sender, channel}; use std::sync::{Arc, Mutex}; +use std::thread; use time::{self, Duration}; use unicase::UniCase; use url::{Origin as UrlOrigin, Url}; @@ -37,6 +37,12 @@ struct FetchResponseCollector { sender: Sender, } +fn new_fetch_context() -> FetchContext { + FetchContext { + state: HttpState::new(), + user_agent: "Such Browser. Very Layout. Wow.".into(), + } +} impl FetchTaskTarget for FetchResponseCollector { fn process_request_body(&mut self, _: &Request) {} fn process_request_eof(&mut self, _: &Request) {} @@ -50,7 +56,7 @@ impl FetchTaskTarget for FetchResponseCollector { fn fetch_async(request: Request, target: Box) { thread::spawn(move || { - fetch(Rc::new(request), &mut Some(target), HttpState::new()); + fetch(Rc::new(request), &mut Some(target), new_fetch_context()); }); } @@ -77,7 +83,7 @@ fn test_fetch_response_is_not_network_error() { *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); if fetch_response.is_network_error() { @@ -98,7 +104,7 @@ fn test_fetch_response_body_matches_const_message() { *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -120,7 +126,7 @@ fn test_fetch_aboutblank() { *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); assert!(!fetch_response.is_network_error()); assert!(*fetch_response.body.lock().unwrap() == ResponseBody::Done(vec![])); } @@ -132,7 +138,7 @@ fn test_fetch_data() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); let expected_resp_body = "

Servo

".to_owned(); - let fetch_response = fetch(Rc::new(request), &mut None, HttpState::new()); + let fetch_response = fetch(Rc::new(request), &mut None, new_fetch_context()); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); @@ -161,7 +167,7 @@ fn test_fetch_file() { let request = Request::new(url, Some(origin), false); request.same_origin_data.set(true); - let fetch_response = fetch(Rc::new(request), &mut None, HttpState::new()); + let fetch_response = fetch(Rc::new(request), &mut None, new_fetch_context()); assert!(!fetch_response.is_network_error()); assert_eq!(fetch_response.headers.len(), 1); let content_type: &ContentType = fetch_response.headers.get().unwrap(); @@ -205,7 +211,7 @@ fn test_cors_preflight_fetch() { request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -245,8 +251,8 @@ fn test_cors_preflight_cache_fetch() { let wrapped_request0 = Rc::new(request.clone()); let wrapped_request1 = Rc::new(request); - let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache, &mut None, HttpState::new()); - let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache, &mut None, HttpState::new()); + let fetch_response0 = fetch_with_cors_cache(wrapped_request0.clone(), &mut cache, &mut None, new_fetch_context()); + let fetch_response1 = fetch_with_cors_cache(wrapped_request1.clone(), &mut cache, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response0.is_network_error() && !fetch_response1.is_network_error()); @@ -294,7 +300,7 @@ fn test_cors_preflight_fetch_network_error() { request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); assert!(fetch_response.is_network_error()); @@ -317,7 +323,7 @@ fn test_fetch_response_is_basic_filtered() { *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -365,7 +371,7 @@ fn test_fetch_response_is_cors_filtered() { request.mode = RequestMode::CORSMode; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -398,7 +404,7 @@ fn test_fetch_response_is_opaque_filtered() { *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -448,7 +454,7 @@ fn test_fetch_response_is_opaque_redirect_filtered() { request.redirect_mode.set(RedirectMode::Manual); let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); assert!(!fetch_response.is_network_error()); @@ -486,7 +492,7 @@ fn test_fetch_with_local_urls_only() { request.local_urls_only = true; let wrapped_request = Rc::new(request); - fetch(wrapped_request, &mut None, HttpState::new()) + fetch(wrapped_request, &mut None, new_fetch_context()) }; let local_url = Url::parse("about:blank").unwrap(); @@ -525,7 +531,7 @@ fn setup_server_and_fetch(message: &'static [u8], redirect_cap: u32) -> Response *request.referer.borrow_mut() = Referer::NoReferer; let wrapped_request = Rc::new(request); - let fetch_response = fetch(wrapped_request, &mut None, HttpState::new()); + let fetch_response = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); fetch_response } @@ -611,7 +617,7 @@ fn test_fetch_redirect_updates_method_runner(tx: Sender, status_code: Stat *request.method.borrow_mut() = method; let wrapped_request = Rc::new(request); - let _ = fetch(wrapped_request, &mut None, HttpState::new()); + let _ = fetch(wrapped_request, &mut None, new_fetch_context()); let _ = server.close(); } diff --git a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini deleted file mode 100644 index 61fdcbeb2f1..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic-setrequestheader-existing-session.htm.ini +++ /dev/null @@ -1,5 +0,0 @@ -[send-authentication-basic-setrequestheader-existing-session.htm] - type: testharness - [XMLHttpRequest: send() - "Basic" authenticated request using setRequestHeader() when there is an existing session] - expected: FAIL - diff --git a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini deleted file mode 100644 index 4aa67f4e1f8..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-authentication-basic.htm.ini +++ /dev/null @@ -1,5 +0,0 @@ -[send-authentication-basic.htm] - type: testharness - [XMLHttpRequest: send() - "Basic" authenticated requests with user name and password passed to open()] - expected: FAIL - diff --git a/tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini new file mode 100644 index 00000000000..98f1333fabb --- /dev/null +++ b/tests/wpt/metadata/XMLHttpRequest/send-conditional.htm.ini @@ -0,0 +1,8 @@ +[send-conditional.htm] + type: testharness + [XMLHttpRequest: send() - conditional requests (tag)] + expected: FAIL + + [XMLHttpRequest: send() - conditional requests (date)] + expected: FAIL + diff --git a/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini index b3144791ecf..9c6e15cc513 100644 --- a/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini +++ b/tests/wpt/metadata/XMLHttpRequest/send-content-type-charset.htm.ini @@ -15,9 +15,12 @@ [charset given but wrong, fix it (known MIME, bogus charset)] expected: FAIL - [charset given but wrong, fix it (known MIME, actual charset)] - expected: FAIL - [If multiple charset parameters are given, all should be rewritten] expected: FAIL + [Correct text/plain MIME with charset] + expected: FAIL + + [charset given but wrong, fix it (known MIME, actual charset)] + expected: FAIL + diff --git a/tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini deleted file mode 100644 index c1ab904b4bb..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-entity-body-get-head.htm.ini +++ /dev/null @@ -1,5 +0,0 @@ -[send-entity-body-get-head.htm] - type: testharness - [XMLHttpRequest: send() - non-empty data argument and GET/HEAD (HEAD)] - expected: FAIL - diff --git a/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini new file mode 100644 index 00000000000..8a2f5593ca5 --- /dev/null +++ b/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-cors.htm.ini @@ -0,0 +1,14 @@ +[send-redirect-to-cors.htm] + type: testharness + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (301)] + expected: FAIL + + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (302)] + expected: FAIL + + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (303)] + expected: FAIL + + [XMLHttpRequest: send() - Redirect to CORS-enabled resource (307)] + expected: FAIL + diff --git a/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini b/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini deleted file mode 100644 index eabe809eb86..00000000000 --- a/tests/wpt/metadata/XMLHttpRequest/send-redirect-to-non-cors.htm.ini +++ /dev/null @@ -1,14 +0,0 @@ -[send-redirect-to-non-cors.htm] - type: testharness - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (301)] - expected: FAIL - - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (302)] - expected: FAIL - - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (303)] - expected: FAIL - - [XMLHttpRequest: send() - Redirect to cross-origin resource, not CORS-enabled (307)] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/allow-headers.htm.ini b/tests/wpt/metadata/cors/allow-headers.htm.ini index 07bd1905cc5..c46e502e53c 100644 --- a/tests/wpt/metadata/cors/allow-headers.htm.ini +++ b/tests/wpt/metadata/cors/allow-headers.htm.ini @@ -33,6 +33,3 @@ [Allow origin: [tab\]http://web-platform.test:8000] expected: FAIL - [Disallow origin: http://web-platform.test:8000/] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/origin.htm.ini b/tests/wpt/metadata/cors/origin.htm.ini index 84071a32cfa..6b627bf7b8c 100644 --- a/tests/wpt/metadata/cors/origin.htm.ini +++ b/tests/wpt/metadata/cors/origin.htm.ini @@ -33,12 +33,3 @@ [Allow origin: [tab\]http://web-platform.test:8000] expected: FAIL - [Disallow origin: http://web-platform.test:8000/] - expected: FAIL - - [Disallow multiple headers (, *)] - expected: FAIL - - [Disallow multiple headers (*, )] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/preflight-cache.htm.ini b/tests/wpt/metadata/cors/preflight-cache.htm.ini index c61b6a45bdc..951b025fc02 100644 --- a/tests/wpt/metadata/cors/preflight-cache.htm.ini +++ b/tests/wpt/metadata/cors/preflight-cache.htm.ini @@ -1,17 +1,8 @@ [preflight-cache.htm] type: testharness - [Test preflight] - expected: FAIL - [preflight for x-print should be cached] expected: FAIL - [age = 0, should not be cached] - expected: FAIL - - [age = -1, should not be cached] - expected: FAIL - [preflight first request, second from cache, wait, third should preflight again] expected: FAIL diff --git a/tests/wpt/metadata/cors/redirect-origin.htm.ini b/tests/wpt/metadata/cors/redirect-origin.htm.ini index 10780608fa9..ef3413cca5a 100644 --- a/tests/wpt/metadata/cors/redirect-origin.htm.ini +++ b/tests/wpt/metadata/cors/redirect-origin.htm.ini @@ -62,24 +62,12 @@ [local (*) to remote (http://web-platform.test:8000), expect origin=http://web-platform.test:8000] expected: FAIL - [local (*) to remote (null), expect to fail] - expected: FAIL - - [local (*) to remote (none), expect to fail] - expected: FAIL - [local (http://web-platform.test:8000) to remote (*), expect origin=http://web-platform.test:8000] expected: FAIL [local (http://web-platform.test:8000) to remote (http://web-platform.test:8000), expect origin=http://web-platform.test:8000] expected: FAIL - [local (http://web-platform.test:8000) to remote (null), expect to fail] - expected: FAIL - - [local (http://web-platform.test:8000) to remote (none), expect to fail] - expected: FAIL - [local (null) to remote (*), expect origin=http://web-platform.test:8000] expected: FAIL @@ -110,27 +98,3 @@ [remote (http://web-platform.test:8000) to remote2 (null), expect origin=null] expected: FAIL - [remote (http://www1.web-platform.test:8000) to remote (*), expect to fail] - expected: FAIL - - [remote (null) to remote2 (*), expect to fail] - expected: FAIL - - [remote (none) to remote2 (*), expect to fail] - expected: FAIL - - [remote (none) to remote2 (*), expect to fail] - expected: FAIL - - [remote (null) to remote (*), expect to fail] - expected: FAIL - - [remote (none) to remote (*), expect to fail] - expected: FAIL - - [remote (none) to local (*), expect to fail] - expected: FAIL - - [remote (null) to local (*), expect to fail] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/redirect-preflight-2.htm.ini b/tests/wpt/metadata/cors/redirect-preflight-2.htm.ini deleted file mode 100644 index 3db95f5aa7e..00000000000 --- a/tests/wpt/metadata/cors/redirect-preflight-2.htm.ini +++ /dev/null @@ -1,8 +0,0 @@ -[redirect-preflight-2.htm] - type: testharness - [Same-origin custom-header request, redirect to cross-origin succeeds after doing a preflight] - expected: FAIL - - [Same-origin custom-header request, redirect to cross-origin fails after doing a non-successful preflight] - expected: FAIL - diff --git a/tests/wpt/metadata/cors/response-headers.htm.ini b/tests/wpt/metadata/cors/response-headers.htm.ini index 15d1c72e334..46c5d996971 100644 --- a/tests/wpt/metadata/cors/response-headers.htm.ini +++ b/tests/wpt/metadata/cors/response-headers.htm.ini @@ -9,6 +9,3 @@ [getResponse: don't expose x-nonexposed] expected: FAIL - [getAllResponseHeaders: don't expose x-nonexposed] - expected: FAIL - From 8b9479f8bf13efa0cc174a84e1775c4926278cef Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 10 Jun 2016 15:34:05 +0530 Subject: [PATCH 11/12] Make send-usp tests less timeout-prone --- .../XMLHttpRequest/send-usp.js | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js b/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js index 56e9e09640f..c00b2e0e0ca 100644 --- a/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js +++ b/tests/wpt/web-platform-tests/XMLHttpRequest/send-usp.js @@ -1,3 +1,5 @@ +const NUM_TESTS = 128; + function encode(n) { if (n === 0x20) { return "\x2B"; @@ -13,27 +15,34 @@ function encode(n) { return "%" + (s.length === 2 ? s : '0' + s); } -function do_test(n) { - async_test(function() { - var x = new XMLHttpRequest(); - x.onload = this.step_func_done(function(e) { - assert_equals(x.response, "a=" + encode(n)) - }); - x.onerror = this.unreached_func(); - x.open("POST", "resources/content.py"); - var usp = new URLSearchParams(); - usp.append("a", String.fromCharCode(n)); - x.send(usp) - }, "XMLHttpRequest.send(URLSearchParams) (" + n + ")"); -} - function run_test() { - var i = 0; - add_result_callback(function() { - if (++i === 128) { - return; + var tests = []; + var overall_test = async_test("Overall fetch with URLSearchParams"); + for (var i = 0; i < NUM_TESTS; i++) { + // Multiple subtests so that failures can be fine-grained + tests[i] = async_test("XMLHttpRequest.send(URLSearchParams) (" + i + ")"); + } + + // We use a single XHR since this test tends to time out + // with 128 consecutive fetches when run in parallel + // with many other WPT tests. + var x = new XMLHttpRequest(); + x.onload = overall_test.step_func(function() { + var response_split = x.response.split("&"); + overall_test.done(); + for (var i = 0; i < NUM_TESTS; i++) { + tests[i].step(function() { + assert_equals(response_split[i], "a" + i + "="+encode(i)); + tests[i].done(); + }); } - do_test(i); }); - do_test(i); + x.onerror = overall_test.unreached_func(); + + x.open("POST", "resources/content.py"); + var usp = new URLSearchParams(); + for (var i = 0; i < NUM_TESTS; i++) { + usp.append("a" + i, String.fromCharCode(i)); + } + x.send(usp) } From 9deecd793c6859e52c2ce9df7bba28eb3159cb97 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 10 Jun 2016 15:46:12 +0530 Subject: [PATCH 12/12] Review fixes --- components/net/fetch/methods.rs | 10 +- components/net/http_loader.rs | 6 +- components/net_traits/request.rs | 7 +- components/net_traits/response.rs | 6 +- components/script/Cargo.toml | 1 - components/script/cors.rs | 490 ------------------------ components/script/dom/xmlhttprequest.rs | 15 +- components/script/lib.rs | 2 - components/servo/Cargo.lock | 1 - ports/cef/Cargo.lock | 1 - tests/unit/net/fetch.rs | 54 +-- 11 files changed, 44 insertions(+), 549 deletions(-) delete mode 100644 components/script/cors.rs diff --git a/components/net/fetch/methods.rs b/components/net/fetch/methods.rs index 305eb1bdf83..b357edae05e 100644 --- a/components/net/fetch/methods.rs +++ b/components/net/fetch/methods.rs @@ -312,6 +312,8 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, if let Some(ref mut target) = *target { target.process_response_chunk(vec.clone()); } + } else { + assert!(*response.body.lock().unwrap() == ResponseBody::Empty) } // overloaded similarly to process_response @@ -357,6 +359,8 @@ fn main_fetch(request: Rc, cache: &mut CORSCache, cors_flag: bool, // obtained synchronously via basic_fetch for data/file/about/etc // We should still send the body across as a chunk target.process_response_chunk(vec.clone()); + } else { + assert!(*response.body.lock().unwrap() == ResponseBody::Empty) } } @@ -820,8 +824,10 @@ fn http_network_or_cache_fetch(request: Rc, if !http_request.use_url_credentials || !has_credentials(¤t_url) { authorization_value = Some(basic); } - } else if authentication_fetch_flag { - // Substep 5 + } + + // Substep 5 + if authentication_fetch_flag && authorization_value.is_none() { if has_credentials(¤t_url) { authorization_value = Some(Basic { username: current_url.username().to_owned(), diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index 1123af628ab..2855e38bf45 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -435,9 +435,9 @@ fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option { /// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer pub fn determine_request_referrer(headers: &mut Headers, - referrer_policy: Option, - referrer_url: Option, - url: Url) -> Option { + referrer_policy: Option, + referrer_url: Option, + url: Url) -> Option { //TODO - algorithm step 2 not addressed assert!(!headers.has::()); if let Some(ref_url) = referrer_url { diff --git a/components/net_traits/request.rs b/components/net_traits/request.rs index 4c7f4081f77..1fbcdabe868 100644 --- a/components/net_traits/request.rs +++ b/components/net_traits/request.rs @@ -117,7 +117,7 @@ pub struct RequestInit { pub unsafe_request: bool, pub same_origin_data: bool, pub body: Option>, - // TODO: cleint object + // TODO: client object pub destination: Destination, pub synchronous: bool, pub mode: RequestMode, @@ -214,7 +214,9 @@ impl Request { } pub fn from_init(init: RequestInit) -> Request { - let mut req = Request::new(init.url, None, false); + let mut req = Request::new(init.url, + Some(Origin::Origin(init.origin.origin())), + false); *req.method.borrow_mut() = init.method; *req.headers.borrow_mut() = init.headers; req.unsafe_request = init.unsafe_request; @@ -226,7 +228,6 @@ impl Request { req.use_cors_preflight = init.use_cors_preflight; req.credentials_mode = init.credentials_mode; req.use_url_credentials = init.use_url_credentials; - *req.origin.borrow_mut() = Origin::Origin(init.origin.origin()); *req.referer.borrow_mut() = if let Some(url) = init.referer_url { Referer::RefererUrl(url) } else { diff --git a/components/net_traits/response.rs b/components/net_traits/response.rs index e20b7b1dbeb..ab1d5b1c8f5 100644 --- a/components/net_traits/response.rs +++ b/components/net_traits/response.rs @@ -226,9 +226,13 @@ impl Response { let mut metadata = if let Some(ref url) = self.url { Metadata::default(url.clone()) } else { - return Err(NetworkError::Internal("No url found".to_string())); + return Err(NetworkError::Internal("No url found in response".to_string())); }; + if self.is_network_error() { + return Err(NetworkError::Internal("Cannot extract metadata from network error".to_string())); + } + metadata.set_content_type(match self.headers.get() { Some(&ContentType(ref mime)) => Some(mime), None => None diff --git a/components/script/Cargo.toml b/components/script/Cargo.toml index 056bb6f72bb..95f1044fd36 100644 --- a/components/script/Cargo.toml +++ b/components/script/Cargo.toml @@ -60,7 +60,6 @@ smallvec = "0.1" string_cache = {version = "0.2.18", features = ["heap_size", "unstable"]} style = {path = "../style"} time = "0.1.12" -unicase = "1.0" url = {version = "1.0.0", features = ["heap_size", "query_encoding"]} util = {path = "../util"} uuid = {version = "0.2", features = ["v4"]} diff --git a/components/script/cors.rs b/components/script/cors.rs deleted file mode 100644 index f9d0bbbd264..00000000000 --- a/components/script/cors.rs +++ /dev/null @@ -1,490 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! A partial implementation of CORS -//! For now this library is XHR-specific. -//! For stuff involving ``, `