/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use connector::create_http_connector;
use data_loader::decode;
use devtools_traits::DevtoolsControlMsg;
use fetch::cors_cache::CORSCache;
use http_loader::{HttpState, set_default_accept_encoding, set_request_cookies};
use http_loader::{NetworkHttpRequestFactory, ReadResult, StreamedResponse, obtain_response, read_block};
use http_loader::{auth_from_cache, determine_request_referrer};
use http_loader::{send_response_to_devtools, send_request_to_devtools};
use hyper::header::{Accept, AcceptLanguage, Authorization, AccessControlAllowCredentials};
use hyper::header::{AccessControlAllowOrigin, AccessControlAllowHeaders, AccessControlAllowMethods};
use hyper::header::{AccessControlRequestHeaders, AccessControlMaxAge, AccessControlRequestMethod, Basic};
use hyper::header::{CacheControl, CacheDirective, ContentEncoding, ContentLength, ContentLanguage, ContentType};
use hyper::header::{Encoding, HeaderView, Headers, Host, IfMatch, IfRange, IfUnmodifiedSince, IfModifiedSince};
use hyper::header::{IfNoneMatch, Pragma, Location, QualityItem, Referer as RefererHeader, UserAgent, q, qitem};
use hyper::method::Method;
use hyper::mime::{Mime, SubLevel, TopLevel};
use hyper::status::StatusCode;
use hyper_serde::Serde;
use mime_guess::guess_mime_type;
use msg::constellation_msg::ReferrerPolicy;
use net_traits::FetchTaskTarget;
use net_traits::request::{CacheMode, CredentialsMode, Destination};
use net_traits::request::{RedirectMode, Referer, Request, RequestMode, ResponseTainting};
use net_traits::request::{Type, Origin, Window};
use net_traits::response::{HttpsState, TerminationReason};
use net_traits::response::{Response, ResponseBody, ResponseType};
use resource_thread::CancellationListener;
use std::collections::HashSet;
use std::fs::File;
use std::io::Read;
use std::iter::FromIterator;
use std::mem::swap;
use std::ops::Deref;
use std::rc::Rc;
use std::sync::mpsc::{channel, Sender, Receiver};
use unicase::UniCase;
use url::{Origin as UrlOrigin, Url};
use util::thread::spawn_named;
use uuid;
pub type Target = Option<Box<FetchTaskTarget + Send>>;
enum Data {
Payload(Vec<u8>),
Done,
}
pub struct FetchContext {
pub state: HttpState,
pub user_agent: String,
pub devtools_chan: Option<Sender<DevtoolsControlMsg>>,
}
type DoneChannel = Option<(Sender<Data>, Receiver<Data>)>;
/// [Fetch](https://fetch.spec.whatwg.org#concept-fetch)
pub fn fetch(request: Rc<Request>, target: &mut Target, context: FetchContext) -> Response {
fetch_with_cors_cache(request, &mut CORSCache::new(), target, context)
}
pub fn fetch_with_cors_cache(request: Rc<Request>,
cache: &mut CORSCache,
target: &mut Target,
context: FetchContext) -> Response {
// Step 1
if request.window.get() == Window::Client {
// TODO: Set window to request's client object if client is a Window object
} else {
request.window.set(Window::NoWindow);
}
// Step 2
if *request.origin.borrow() == Origin::Client {
// TODO: set request's origin to request's client's origin
unimplemented!()
}
// Step 3
if !request.headers.borrow().has::<Accept>() {
let value = match request.type_ {
// Substep 2
_ if request.is_navigation_request() =>
vec![qitem(mime!(Text / Html)),
// FIXME: This should properly generate a MimeType that has a
// SubLevel of xhtml+xml (https://github.com/hyperium/mime.rs/issues/22)
qitem(mime!(Application / ("xhtml+xml") )),
QualityItem::new(mime!(Application / Xml), q(0.9)),
QualityItem::new(mime!(_ / _), q(0.8))],
// Substep 3
Type::Image =>
vec![qitem(mime!(Image / Png)),
// FIXME: This should properly generate a MimeType that has a
// SubLevel of svg+xml (https://github.com/hyperium/mime.rs/issues/22)
qitem(mime!(Image / ("svg+xml") )),
QualityItem::new(mime!(Image / _), q(0.8)),
QualityItem::new(mime!(_ / _), q(0.5))],
// Substep 3
Type::Style =>
vec![qitem(mime!(Text / Css)),
QualityItem::new(mime!(_ / _), q(0.1))],
// Substep 1
_ => vec![qitem(mime!(_ / _))]
};
// Substep 4
request.headers.borrow_mut().set(Accept(value));
}
// Step 4
if !request.headers.borrow().has::<AcceptLanguage>() {
request.headers.borrow_mut().set(AcceptLanguage(vec![qitem("en-US".parse().unwrap())]));
}
// Step 5
// TODO: Figure out what a Priority object is
// Step 6
if request.is_subresource_request() {
// TODO: create a fetch record and append it to request's client's fetch group list
}
// Step 7
main_fetch(request, cache, false, false, target, &mut None, &context)
}
/// [Main fetch](https://fetch.spec.whatwg.org/#concept-main-fetch)
fn main_fetch(request: Rc<Request>, cache: &mut CORSCache, cors_flag: bool,
recursive_flag: bool, target: &mut Target, done_chan: &mut DoneChannel,
context: &FetchContext) -> Response {
// TODO: Implement main fetch spec
// Step 1
let mut response = None;
// Step 2
if request.local_urls_only {
match request.current_url().scheme() {
"about" | "blob" | "data" | "filesystem" => (), // Ok, the URL is local.
_ => response = Some(Response::network_error())
}
}
// Step 3
// TODO be able to execute report CSP
// Step 4
// TODO this step, based off of http_loader.rs (upgrade)
// Step 5
// TODO this step (CSP port/content blocking)
// Step 6
// TODO this step (referer policy)
// currently the clients themselves set referer policy in RequestInit
// Step 7
if request.referrer_policy.get().is_none() {
request.referrer_policy.set(Some(ReferrerPolicy::NoReferrerWhenDowngrade));
}
// Step 8
if *request.referer.borrow() != Referer::NoReferer {
// remove Referer headers set in past redirects/preflights
// this stops the assertion in determine_request_referrer from failing
request.headers.borrow_mut().remove::<RefererHeader>();
let referrer_url = determine_request_referrer(&mut *request.headers.borrow_mut(),
request.referrer_policy.get(),
request.referer.borrow_mut().take(),
request.current_url().clone());
*request.referer.borrow_mut() = Referer::from_url(referrer_url);
}
// Step 9
// TODO this step (HSTS)
// Step 10
// this step is obsoleted by fetch_async
// Step 11
let response = match response {
Some(response) => response,
None => {
let current_url = request.current_url();
let same_origin = if let Origin::Origin(ref origin) = *request.origin.borrow() {
*origin == current_url.origin()
} else {
false
};
if (same_origin && !cors_flag ) ||
(current_url.scheme() == "data" && request.same_origin_data.get()) ||
(current_url.scheme() == "file" && request.same_origin_data.get()) ||
current_url.scheme() == "about" ||
request.mode == RequestMode::Navigate {
basic_fetch(request.clone(), cache, target, done_chan, context)
} else if request.mode == RequestMode::SameOrigin {
Response::network_error()
} else if request.mode == RequestMode::NoCORS {
request.response_tainting.set(ResponseTainting::Opaque);
basic_fetch(request.clone(), cache, target, done_chan, context)
} else if !matches!(current_url.scheme(), "http" | "https") {
Response::network_error()
} else if request.use_cors_preflight ||
(request.unsafe_request &&
(!is_simple_method(&request.method.borrow()) ||
request.headers.borrow().iter().any(|h| !is_simple_header(&h)))) {
request.response_tainting.set(ResponseTainting::CORSTainting);
request.redirect_mode.set(RedirectMode::Error);
let response = http_fetch(request.clone(), cache, true, true, false,
target, done_chan, context);
if response.is_network_error() {
// TODO clear cache entries using request
}
response
} else {
request.response_tainting.set(ResponseTainting::CORSTainting);
http_fetch(request.clone(), cache, true, false, false, target, done_chan, context)
}
}
};
// Step 12
if recursive_flag {
return response;
}
// Step 13
// no need to check if response is a network error, since the type would not be `Default`
let response = if response.response_type == ResponseType::Default {
let response_type = match request.response_tainting.get() {
ResponseTainting::Basic => ResponseType::Basic,
ResponseTainting::CORSTainting => ResponseType::CORS,
ResponseTainting::Opaque => ResponseType::Opaque,
};
response.to_filtered(response_type)
} else {
response
};
{
// Step 14
let network_error_res = Response::network_error();
let internal_response = if response.is_network_error() {
&network_error_res
} else {
response.actual_response()
};
// Step 15
if internal_response.url_list.borrow().is_empty() {
*internal_response.url_list.borrow_mut() = request.url_list.borrow().clone();
}
// Step 16
// TODO this step (CSP/blocking)
// Step 17
if !response.is_network_error() && (is_null_body_status(&internal_response.status) ||
match *request.method.borrow() {
Method::Head | Method::Connect => true,
_ => false })
{
// when Fetch is used only asynchronously, we will need to make sure
// that nothing tries to write to the body at this point
let mut body = internal_response.body.lock().unwrap();
*body = ResponseBody::Empty;
}
// Step 18
// TODO be able to compare response integrity against request integrity metadata
// if !response.is_network_error() {
// // Substep 1
// response.wait_until_done();
// // Substep 2
// if response.termination_reason.is_none() {
// response = Response::network_error();
// internal_response = Response::network_error();
// }
// }
}
// Step 19
if request.synchronous {
if let Some(ref mut target) = *target {
// process_response is not supposed to be used
// by sync fetch, but we overload it here for simplicity
target.process_response(&response);
}
if let Some(ref ch) = *done_chan {
loop {
match ch.1.recv()
.expect("fetch worker should always send Done before terminating") {
Data::Payload(vec) => {
if let Some(ref mut target) = *target {
target.process_response_chunk(vec);
}
}
Data::Done => break,
}
}
} else if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
// in case there was no channel to wait for, the body was
// obtained synchronously via basic_fetch for data/file/about/etc
// We should still send the body across as a chunk
if let Some(ref mut target) = *target {
target.process_response_chunk(vec.clone());
}
} else {
assert!(*response.body.lock().unwrap() == ResponseBody::Empty)
}
// overloaded similarly to process_response
if let Some(ref mut target) = *target {
target.process_response_eof(&response);
}
return response;
}
// Step 20
if request.body.borrow().is_some() && matches!(request.current_url().scheme(), "http" | "https") {
if let Some(ref mut target) = *target {
// XXXManishearth: We actually should be calling process_request
// in http_network_fetch. However, we can't yet follow the request
// upload progress, so I'm keeping it here for now and pretending
// the body got sent in one chunk
target.process_request_body(&request);
target.process_request_eof(&request);
}
}
// Step 21
if let Some(ref mut target) = *target {
target.process_response(&response);
}
// Step 22
if let Some(ref ch) = *done_chan {
loop {
match ch.1.recv()
.expect("fetch worker should always send Done before terminating") {
Data::Payload(vec) => {
if let Some(ref mut target) = *target {
target.process_response_chunk(vec);
}
}
Data::Done => break,
}
}
} else if let Some(ref mut target) = *target {
if let ResponseBody::Done(ref vec) = *response.body.lock().unwrap() {
// in case there was no channel to wait for, the body was
// obtained synchronously via basic_fetch for data/file/about/etc
// We should still send the body across as a chunk
target.process_response_chunk(vec.clone());
} else {
assert!(*response.body.lock().unwrap() == ResponseBody::Empty)
}
}
// Step 23
request.done.set(true);
// Step 24
if let Some(ref mut target) = *target {
target.process_response_eof(&response);
}
// TODO remove this line when only asynchronous fetches are used
return response;
}
/// [Basic fetch](https://fetch.spec.whatwg.org#basic-fetch)
fn basic_fetch(request: Rc<Request>, cache: &mut CORSCache,
target: &mut Target, done_chan: &mut DoneChannel,
context: &FetchContext) -> Response {
let url = request.current_url();
match url.scheme() {
"about" if url.path() == "blank" => {
let mut response = Response::new();
// https://github.com/whatwg/fetch/issues/312
response.url = Some(url);
response.headers.set(ContentType(mime!(Text / Html; Charset = Utf8)));
*response.body.lock().unwrap() = ResponseBody::Done(vec![]);
response
},
"http" | "https" => {
http_fetch(request.clone(), cache, false, false, false, target, done_chan, context)
},
"data" => {
if *request.method.borrow() == Method::Get {
match decode(&url) {
Ok((mime, bytes)) => {
let mut response = Response::new();
// https://github.com/whatwg/fetch/issues/312
response.url = Some(url.clone());
*response.body.lock().unwrap() = ResponseBody::Done(bytes);
response.headers.set(ContentType(mime));
response
},
Err(_) => Response::network_error()
}
} else {
Response::network_error()
}
},
"file" => {
if *request.method.borrow() == Method::Get {
match url.to_file_path() {
Ok(file_path) => {
File::open(file_path.clone()).ok().map_or(Response::network_error(), |mut file| {
let mut bytes = vec![];
let _ = file.read_to_end(&mut bytes);
let mime = guess_mime_type(file_path);
let mut response = Response::new();
// https://github.com/whatwg/fetch/issues/312
response.url = Some(url.clone());
*response.body.lock().unwrap() = ResponseBody::Done(bytes);
response.headers.set(ContentType(mime));
response
})
},
_ => Response::network_error()
}
} else {
Response::network_error()
}
},
"blob" | "ftp" => {
// XXXManishearth handle these
panic!("Unimplemented scheme for Fetch")
},
_ => Response::network_error()
}
}
/// [HTTP fetch](https://fetch.spec.whatwg.org#http-fetch)
fn http_fetch(request: Rc<Request>,
cache: &mut CORSCache,
cors_flag: bool,
cors_preflight_flag: bool,
authentication_fetch_flag: bool,
target: &mut Target,
done_chan: &mut DoneChannel,
context: &FetchContext) -> Response {
// This is a new async fetch, reset the channel we are waiting on
*done_chan = None;
// Step 1
let mut response: Option<Response> = None;
// Step 2
// nothing to do, since actual_response is a function on response
// Step 3
if !request.skip_service_worker.get() && !request.is_service_worker_global_scope {
// Substep 1
// TODO (handle fetch unimplemented)
if let Some(ref res) = response {
// Substep 2
// nothing to do, since actual_response is a function on response
// Substep 3
if (res.response_type == ResponseType::Opaque &&
request.mode != RequestMode::NoCORS) ||
(res.response_type == ResponseType::OpaqueRedirect &&
request.redirect_mode.get() != RedirectMode::Manual) ||
(res.url_list.borrow().len() > 1 &&
request.redirect_mode.get() != RedirectMode::Follow) ||
res.response_type == ResponseType::Error {
return Response::network_error();
}
// Substep 4
// TODO: set response's CSP list on actual_response
}
}
// Step 4
let credentials = match request.credentials_mode {
CredentialsMode::Include => true,
CredentialsMode::CredentialsSameOrigin if request.response_tainting.get() == ResponseTainting::Basic
=> true,
_ => false
};
// Step 5
if response.is_none() {
// Substep 1
if cors_preflight_flag {
let method_cache_match = cache.match_method(&*request,
request.method.borrow().clone());
let method_mismatch = !method_cache_match && (!is_simple_method(&request.method.borrow()) ||
request.use_cors_preflight);
let header_mismatch = request.headers.borrow().iter().any(|view|
!cache.match_header(&*request, view.name()) && !is_simple_header(&view)
);
// Sub-substep 1
if method_mismatch || header_mismatch {
let preflight_result = cors_preflight_fetch(request.clone(), cache, context);
// Sub-substep 2
if preflight_result.response_type == ResponseType::Error {
return Response::network_error();
}
}
}
// Substep 2
request.skip_service_worker.set(true);
// Substep 3
let fetch_result = http_network_or_cache_fetch(request.clone(), credentials, authentication_fetch_flag,
done_chan, context);
// Substep 4
if cors_flag && cors_check(request.clone(), &fetch_result).is_err() {
return Response::network_error();
}
fetch_result.return_internal.set(false);
response = Some(fetch_result);
}
// response is guaranteed to be something by now
let mut response = response.unwrap();
// Step 5
match response.actual_response().status.unwrap() {
// Code 301, 302, 303, 307, 308
StatusCode::MovedPermanently | StatusCode::Found | StatusCode::SeeOther |
StatusCode::TemporaryRedirect | StatusCode::PermanentRedirect => {
response = match request.redirect_mode.get() {
RedirectMode::Error => Response::network_error(),
RedirectMode::Manual => {
response.to_filtered(ResponseType::OpaqueRedirect)
},
RedirectMode::Follow => {
// set back to default
response.return_internal.set(true);
http_redirect_fetch(request, cache, Rc::new(response),
cors_flag, target, done_chan, context)
}
}
},
// Code 401
StatusCode::Unauthorized => {
// Step 1
// FIXME: Figure out what to do with request window objects
if cors_flag || !credentials {
return response;
}
// Step 2
// TODO: Spec says requires testing on multiple WWW-Authenticate headers
// Step 3
if !request.use_url_credentials || authentication_fetch_flag {
// TODO: Prompt the user for username and password from the window
// Wrong, but will have to do until we are able to prompt the user
// otherwise this creates an infinite loop
// We basically pretend that the user declined to enter credentials
return response;
}
// Step 4
return http_fetch(request, cache, cors_flag, cors_preflight_flag,
true, target, done_chan, context);
}
// Code 407
StatusCode::ProxyAuthenticationRequired => {
// Step 1
// TODO: Figure out what to do with request window objects
// Step 2
// TODO: Spec says requires testing on Proxy-Authenticate headers
// Step 3
// TODO: Prompt the user for proxy authentication credentials
// Wrong, but will have to do until we are able to prompt the user
// otherwise this creates an infinite loop
// We basically pretend that the user declined to enter credentials
return response;
// Step 4
// return http_fetch(request, cache,
// cors_flag, cors_preflight_flag,
// authentication_fetch_flag, target,
// done_chan, context);
}
_ => { }
}
// Step 6
if authentication_fetch_flag {
// TODO: Create authentication entry for this request
}
// set back to default
response.return_internal.set(true);
// Step 7
response
}
/// [HTTP redirect fetch](https://fetch.spec.whatwg.org#http-redirect-fetch)
fn http_redirect_fetch(request: Rc<Request>,
cache: &mut CORSCache,
response: Rc<Response>,
cors_flag: bool,
target: &mut Target,
done_chan: &mut DoneChannel,
context: &FetchContext) -> Response {
// Step 1
assert_eq!(response.return_internal.get(), true);
// Step 3
// this step is done early, because querying if Location exists says
// if it is None or Some, making it easy to seperate from the retrieval failure case
if !response.actual_response().headers.has::<Location>() {
return Rc::try_unwrap(response).ok().unwrap();
}
// Step 2
let location = match response.actual_response().headers.get::<Location>() {
Some(&Location(ref location)) => location.clone(),
// Step 4
_ => return Response::network_error()
};
// Step 5
let response_url = response.actual_response().url.as_ref().unwrap();
let location_url = response_url.join(&*location);
// Step 6
let location_url = match location_url {
Ok(url) => url,
_ => return Response::network_error()
};
// Step 7
if request.redirect_count.get() >= 20 {
return Response::network_error();
}
// Step 8
request.redirect_count.set(request.redirect_count.get() + 1);
// Step 9
request.same_origin_data.set(false);
let same_origin = if let Origin::Origin(ref origin) = *request.origin.borrow() {
*origin == request.current_url().origin()
} else {
false
};
let has_credentials = has_credentials(&location_url);
// Step 10
if request.mode == RequestMode::CORSMode && !same_origin && has_credentials {
return Response::network_error();
}
// Step 11
if cors_flag && has_credentials {
return Response::network_error();
}
// Step 12
if cors_flag && !same_origin {
*request.origin.borrow_mut() = Origin::Origin(UrlOrigin::new_opaque());
}
// Step 13
let status_code = response.actual_response().status.unwrap();
if ((status_code == StatusCode::MovedPermanently || status_code == StatusCode::Found) &&
*request.method.borrow() == Method::Post) ||
status_code == StatusCode::SeeOther {
*request.method.borrow_mut() = Method::Get;
*request.body.borrow_mut() = None;
}
// Step 14
request.url_list.borrow_mut().push(location_url);
// Step 15
main_fetch(request, cache, cors_flag, true, target, done_chan, context)
}
/// [HTTP network or cache fetch](https://fetch.spec.whatwg.org#http-network-or-cache-fetch)
fn http_network_or_cache_fetch(request: Rc<Request>,
credentials_flag: bool,
authentication_fetch_flag: bool,
done_chan: &mut DoneChannel,
context: &FetchContext) -> Response {
// TODO: Implement Window enum for Request
let request_has_no_window = true;
// Step 1
let http_request = if request_has_no_window &&
request.redirect_mode.get() == RedirectMode::Error {
request
} else {
Rc::new((*request).clone())
};
let content_length_value = match *http_request.body.borrow() {
None =>
match *http_request.method.borrow() {
// Step 3
Method::Head | Method::Post | Method::Put =>
Some(0),
// Step 2
_ => None
},
// Step 4
Some(ref http_request_body) => Some(http_request_body.len() as u64)
};
// Step 5
if let Some(content_length_value) = content_length_value {
http_request.headers.borrow_mut().set(ContentLength(content_length_value));
}
// Step 6
match *http_request.referer.borrow() {
Referer::NoReferer => (),
Referer::RefererUrl(ref http_request_referer) =>
http_request.headers.borrow_mut().set(RefererHeader(http_request_referer.to_string())),
Referer::Client =>
// it should be impossible for referer to be anything else during fetching
// https://fetch.spec.whatwg.org/#concept-request-referrer
unreachable!()
};
// Step 7
if http_request.omit_origin_header.get() == false {
// TODO update this when https://github.com/hyperium/hyper/pull/691 is finished
// http_request.headers.borrow_mut().set_raw("origin", origin);
}
// Step 8
if !http_request.headers.borrow().has::<UserAgent>() {
http_request.headers.borrow_mut().set(UserAgent(context.user_agent.clone()));
}
match http_request.cache_mode.get() {
// Step 9
CacheMode::Default if is_no_store_cache(&http_request.headers.borrow()) => {
http_request.cache_mode.set(CacheMode::NoStore);
},
// Step 10
CacheMode::NoCache if !http_request.headers.borrow().has::<CacheControl>() => {
http_request.headers.borrow_mut().set(CacheControl(vec![CacheDirective::MaxAge(0)]));
},
// Step 11
CacheMode::Reload => {
// Substep 1
if !http_request.headers.borrow().has::<Pragma>() {
http_request.headers.borrow_mut().set(Pragma::NoCache);
}
// Substep 2
if !http_request.headers.borrow().has::<CacheControl>() {
http_request.headers.borrow_mut().set(CacheControl(vec![CacheDirective::NoCache]));
}
},
_ => {}
}
let current_url = http_request.current_url();
// Step 12
// todo: pass referrer url and policy
// this can only be uncommented when the referer header is set, else it crashes
// in the meantime, we manually set the headers in the block below
// modify_request_headers(&mut http_request.headers.borrow_mut(), ¤t_url,
// None, None, None);
{
let headers = &mut *http_request.headers.borrow_mut();
let host = Host {
hostname: current_url.host_str().unwrap().to_owned(),
port: current_url.port_or_known_default()
};
headers.set(host);
// unlike http_loader, we should not set the accept header
// here, according to the fetch spec
set_default_accept_encoding(headers);
}
// Step 13
// TODO some of this step can't be implemented yet
if credentials_flag {
// Substep 1
// TODO http://mxr.mozilla.org/servo/source/components/net/http_loader.rs#504
// XXXManishearth http_loader has block_cookies: support content blocking here too
set_request_cookies(¤t_url,
&mut *http_request.headers.borrow_mut(),
&context.state.cookie_jar);
// Substep 2
if !http_request.headers.borrow().has::<Authorization<String>>() {
// Substep 3
let mut authorization_value = None;
// Substep 4
if let Some(basic) = auth_from_cache(&context.state.auth_cache, ¤t_url) {
if !http_request.use_url_credentials || !has_credentials(¤t_url) {
authorization_value = Some(basic);
}
}
// Substep 5
if authentication_fetch_flag && authorization_value.is_none() {
if has_credentials(¤t_url) {
authorization_value = Some(Basic {
username: current_url.username().to_owned(),
password: current_url.password().map(str::to_owned)
})
}
}
// Substep 6
if let Some(basic) = authorization_value {
http_request.headers.borrow_mut().set(Authorization(basic));
}
}
}
// Step 14
// TODO this step can't be implemented yet
// Step 15
let mut response: Option<Response> = None;
// Step 16
// TODO have a HTTP cache to check for a completed response
let complete_http_response_from_cache: Option<Response> = None;
if http_request.cache_mode.get() != CacheMode::NoStore &&
http_request.cache_mode.get() != CacheMode::Reload &&
complete_http_response_from_cache.is_some() {
// Substep 1
if http_request.cache_mode.get() == CacheMode::ForceCache {
// TODO pull response from HTTP cache
// response = http_request
}
let revalidation_needed = match response {
Some(ref response) => response_needs_revalidation(&response),
_ => false
};
// Substep 2
if !revalidation_needed && http_request.cache_mode.get() == CacheMode::Default {
// TODO pull response from HTTP cache
// response = http_request
// response.cache_state = CacheState::Local;
}
// Substep 3
if revalidation_needed && http_request.cache_mode.get() == CacheMode::Default ||
http_request.cache_mode.get() == CacheMode::NoCache {
// TODO this substep
}
// Step 17
// TODO have a HTTP cache to check for a partial response
} else if http_request.cache_mode.get() == CacheMode::Default ||
http_request.cache_mode.get() == CacheMode::ForceCache {
// TODO this substep
}
// Step 18
if response.is_none() {
response = Some(http_network_fetch(http_request.clone(), credentials_flag,
done_chan, context.devtools_chan.clone()));
}
let response = response.unwrap();
// Step 19
if let Some(status) = response.status {
if status == StatusCode::NotModified &&
(http_request.cache_mode.get() == CacheMode::Default ||
http_request.cache_mode.get() == CacheMode::NoCache) {
// Substep 1
// TODO this substep
// let cached_response: Option<Response> = None;
// Substep 2
// if cached_response.is_none() {
// return Response::network_error();
// }
// Substep 3
// Substep 4
// response = cached_response;
// Substep 5
// TODO cache_state is immutable?
// response.cache_state = CacheState::Validated;
}
}
// Step 20
response
}
/// [HTTP network fetch](https://fetch.spec.whatwg.org/#http-network-fetch)
fn http_network_fetch(request: Rc<Request>,
_credentials_flag: bool,
done_chan: &mut DoneChannel,
devtools_chan: Option<Sender<DevtoolsControlMsg>>) -> Response {
// TODO: Implement HTTP network fetch spec
// Step 1
// nothing to do here, since credentials_flag is already a boolean
// Step 2
// TODO be able to create connection using current url's origin and credentials
let connection = create_http_connector();
// Step 3
// TODO be able to tell if the connection is a failure
// Step 4
let factory = NetworkHttpRequestFactory {
connector: connection,
};
let url = request.current_url();
let cancellation_listener = CancellationListener::new(None);
let request_id = devtools_chan.as_ref().map(|_| {
uuid::Uuid::new_v4().simple().to_string()
});
// XHR uses the default destination; other kinds of fetches (which haven't been implemented yet)
// do not. Once we support other kinds of fetches we'll need to be more fine grained here
// since things like image fetches are classified differently by devtools
let is_xhr = request.destination == Destination::None;
let wrapped_response = obtain_response(&factory, &url, &request.method.borrow(),
&request.headers.borrow(),
&cancellation_listener, &request.body.borrow(), &request.method.borrow(),
&request.pipeline_id.get(), request.redirect_count.get() + 1,
request_id.as_ref().map(Deref::deref), is_xhr);
let pipeline_id = request.pipeline_id.get();
let mut response = Response::new();
match wrapped_response {
Ok((res, msg)) => {
response.url = Some(url.clone());
response.status = Some(res.response.status);
response.raw_status = Some(res.response.status_raw().clone());
response.headers = res.response.headers.clone();
let res_body = response.body.clone();
// We're about to spawn a thread to be waited on here
*done_chan = Some(channel());
let meta = response.metadata().expect("Response metadata should exist at this stage");
let done_sender = done_chan.as_ref().map(|ch| ch.0.clone());
let devtools_sender = devtools_chan.clone();
let meta_status = meta.status.clone();
let meta_headers = meta.headers.clone();
spawn_named(format!("fetch worker thread"), move || {
match StreamedResponse::from_http_response(box res, meta) {
Ok(mut res) => {
*res_body.lock().unwrap() = ResponseBody::Receiving(vec![]);
if let Some(ref sender) = devtools_sender {
if let Some(m) = msg {
send_request_to_devtools(m, &sender);
}
// --- Tell devtools that we got a response
// Send an HttpResponse message to devtools with the corresponding request_id
if let Some(pipeline_id) = pipeline_id {
send_response_to_devtools(
&sender, request_id.unwrap(),
meta_headers.map(Serde::into_inner),
meta_status.map(Serde::into_inner),
pipeline_id);
}
}
loop {
match read_block(&mut res) {
Ok(ReadResult::Payload(chunk)) => {
if let ResponseBody::Receiving(ref mut body) = *res_body.lock().unwrap() {
body.extend_from_slice(&chunk);
if let Some(ref sender) = done_sender {
let _ = sender.send(Data::Payload(chunk));
}
}
},
Ok(ReadResult::EOF) | Err(_) => {
let mut empty_vec = Vec::new();
let completed_body = match *res_body.lock().unwrap() {
ResponseBody::Receiving(ref mut body) => {
// avoid cloning the body
swap(body, &mut empty_vec);
empty_vec
},
_ => empty_vec,
};
*res_body.lock().unwrap() = ResponseBody::Done(completed_body);
if let Some(ref sender) = done_sender {
let _ = sender.send(Data::Done);
}
break;
}
}
}
}
Err(_) => {
// XXXManishearth we should propagate this error somehow
*res_body.lock().unwrap() = ResponseBody::Done(vec![]);
if let Some(ref sender) = done_sender {
let _ = sender.send(Data::Done);
}
}
}
});
},
Err(_) => {
response.termination_reason = Some(TerminationReason::Fatal);
}
};
// TODO these substeps aren't possible yet
// Substep 1
// Substep 2
// TODO Determine if response was retrieved over HTTPS
// TODO Servo needs to decide what ciphers are to be treated as "deprecated"
response.https_state = HttpsState::None;
// TODO Read request
// Step 5-9
// (needs stream bodies)
// Step 10
// TODO when https://bugzilla.mozilla.org/show_bug.cgi?id=1030660
// is resolved, this step will become uneccesary
// TODO this step
if let Some(encoding) = response.headers.get::<ContentEncoding>() {
if encoding.contains(&Encoding::Gzip) {
}
else if encoding.contains(&Encoding::Compress) {
}
};
// Step 11
// TODO this step isn't possible yet (CSP)
// Step 12
if response.is_network_error() && request.cache_mode.get() == CacheMode::NoStore {
// TODO update response in the HTTP cache for request
}
// TODO this step isn't possible yet
// Step 13
// TODO these steps
// Step 14
// Substep 1
// Substep 2
// Sub-substep 1
// Sub-substep 2
// Sub-substep 3
// Sub-substep 4
// Substep 3
// Step 15
response
}
/// [CORS preflight fetch](https://fetch.spec.whatwg.org#cors-preflight-fetch)
fn cors_preflight_fetch(request: Rc<Request>, cache: &mut CORSCache,
context: &FetchContext) -> Response {
// Step 1
let mut preflight = Request::new(request.current_url(), Some(request.origin.borrow().clone()),
false, request.pipeline_id.get());
*preflight.method.borrow_mut() = Method::Options;
preflight.initiator = request.initiator.clone();
preflight.type_ = request.type_.clone();
preflight.destination = request.destination.clone();
*preflight.referer.borrow_mut() = request.referer.borrow().clone();
preflight.referrer_policy.set(preflight.referrer_policy.get());
// Step 2
preflight.headers.borrow_mut().set::<AccessControlRequestMethod>(
AccessControlRequestMethod(request.method.borrow().clone()));
// Step 3, 4
let mut value = request.headers.borrow().iter()
.filter_map(|ref view| if is_simple_header(view) {
None
} else {
Some(UniCase(view.name().to_owned()))
}).collect::<Vec<UniCase<String>>>();
value.sort();
// Step 5
preflight.headers.borrow_mut().set::<AccessControlRequestHeaders>(
AccessControlRequestHeaders(value));
// Step 6
let preflight = Rc::new(preflight);
let response = http_network_or_cache_fetch(preflight.clone(), false, false, &mut None, context);
// Step 7
if cors_check(request.clone(), &response).is_ok() &&
response.status.map_or(false, |status| status.is_success()) {
// Substep 1
let mut methods = if response.headers.has::<AccessControlAllowMethods>() {
match response.headers.get::<AccessControlAllowMethods>() {
Some(&AccessControlAllowMethods(ref m)) => m.clone(),
// Substep 3
None => return Response::network_error()
}
} else {
vec![]
};
// Substep 2
let header_names = if response.headers.has::<AccessControlAllowHeaders>() {
match response.headers.get::<AccessControlAllowHeaders>() {
Some(&AccessControlAllowHeaders(ref hn)) => hn.clone(),
// Substep 3
None => return Response::network_error()
}
} else {
vec![]
};
// Substep 4
if methods.is_empty() && request.use_cors_preflight {
methods = vec![request.method.borrow().clone()];
}
// Substep 5
debug!("CORS check: Allowed methods: {:?}, current method: {:?}",
methods, request.method.borrow());
if methods.iter().all(|method| *method != *request.method.borrow()) &&
!is_simple_method(&*request.method.borrow()) {
return Response::network_error();
}
// Substep 6
debug!("CORS check: Allowed headers: {:?}, current headers: {:?}",
header_names, request.headers.borrow());
let set: HashSet<&UniCase<String>> = HashSet::from_iter(header_names.iter());
if request.headers.borrow().iter().any(|ref hv| !set.contains(&UniCase(hv.name().to_owned())) &&
!is_simple_header(hv)) {
return Response::network_error();
}
// Substep 7, 8
let max_age = response.headers.get::<AccessControlMaxAge>().map(|acma| acma.0).unwrap_or(0);
// TODO: Substep 9 - Need to define what an imposed limit on max-age is
// Substep 11, 12
for method in &methods {
cache.match_method_and_update(&*request, method.clone(), max_age);
}
// Substep 13, 14
for header_name in &header_names {
cache.match_header_and_update(&*request, &*header_name, max_age);
}
// Substep 15
return response;
}
// Step 8
Response::network_error()
}
/// [CORS check](https://fetch.spec.whatwg.org#concept-cors-check)
fn cors_check(request: Rc<Request>, response: &Response) -> Result<(), ()> {
// Step 1
let origin = response.headers.get::<AccessControlAllowOrigin>().cloned();
// Step 2
let origin = try!(origin.ok_or(()));
// Step 3
if request.credentials_mode != CredentialsMode::Include &&
origin == AccessControlAllowOrigin::Any {
return Ok(());
}
// Step 4
let origin = match origin {
AccessControlAllowOrigin::Value(origin) => origin,
// if it's Any or Null at this point, there's nothing to do but return Err(())
_ => return Err(())
};
match *request.origin.borrow() {
Origin::Origin(ref o) if o.ascii_serialization() == origin => {},
_ => return Err(())
}
// Step 5
if request.credentials_mode != CredentialsMode::Include {
return Ok(());
}
// Step 6
let credentials = request.headers.borrow().get::<AccessControlAllowCredentials>().cloned();
// Step 7
if credentials.is_some() {
return Ok(());
}
// Step 8
Err(())
}
fn has_credentials(url: &Url) -> bool {
!url.username().is_empty() || url.password().is_some()
}
fn is_no_store_cache(headers: &Headers) -> bool {
headers.has::<IfModifiedSince>() | headers.has::<IfNoneMatch>() |
headers.has::<IfUnmodifiedSince>() | headers.has::<IfMatch>() |
headers.has::<IfRange>()
}
/// https://fetch.spec.whatwg.org/#cors-safelisted-request-header
fn is_simple_header(h: &HeaderView) -> bool {
if h.is::<ContentType>() {
match h.value() {
Some(&ContentType(Mime(TopLevel::Text, SubLevel::Plain, _))) |
Some(&ContentType(Mime(TopLevel::Application, SubLevel::WwwFormUrlEncoded, _))) |
Some(&ContentType(Mime(TopLevel::Multipart, SubLevel::FormData, _))) => true,
_ => false
}
} else {
h.is::<Accept>() || h.is::<AcceptLanguage>() || h.is::<ContentLanguage>()
}
}
fn is_simple_method(m: &Method) -> bool {
match *m {
Method::Get | Method::Head | Method::Post => true,
_ => false
}
}
fn response_needs_revalidation(_response: &Response) -> bool {
// TODO this function
false
}
// fn modify_request_headers(headers: &mut Headers) -> {
// // TODO this function
// }
fn is_null_body_status(status: &Option<StatusCode>) -> bool {
match *status {
Some(status) => match status {
StatusCode::SwitchingProtocols | StatusCode::NoContent |
StatusCode::ResetContent | StatusCode::NotModified => true,
_ => false
},
_ => false
}
}