Auto merge of #10696 - rebstar6:referrerPolicy, r=jdm

Referer header

PR1 for https://github.com/servo/servo/issues/10311

This puts the code and data structures in place to set the Referer header based on the Referrer Policy for a given document. Note that document:: get_referrer_policy() always returns the 'No Referrer' option, so for now, this should have no impact on production code, and that policy requires that the Referer header is not added.

Later PRs will determine the policy and edit that get_referrer_policy() accordingly.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/10696)
<!-- Reviewable:end -->

components/compositing/compositor.rs

@@ -1305,7 +1305,7 @@ impl<Window: WindowMethods> IOCompositor<Window> {
             Ok(url) => {
                 self.window.set_page_url(url.clone());
                 let msg = match self.scene.root {
-                    Some(ref layer) => ConstellationMsg::LoadUrl(layer.pipeline_id(), LoadData::new(url)),
+                    Some(ref layer) => ConstellationMsg::LoadUrl(layer.pipeline_id(), LoadData::new(url, None, None)),
                     None => ConstellationMsg::InitLoadUrl(url)
                 };
                 if let Err(e) = self.constellation_chan.send(msg) {

components/compositing/constellation.rs

@@ -887,7 +887,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
                           parent_info,
                           window_size,
                           None,
-                          LoadData::new(Url::parse("about:failure").expect("infallible")));
+                          LoadData::new(Url::parse("about:failure").expect("infallible"), None, None));
 
         self.push_pending_frame(new_pipeline_id, Some(pipeline_id));
 
@@ -897,7 +897,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
         let window_size = self.window_size.visible_viewport;
         let root_pipeline_id = PipelineId::new();
         debug_assert!(PipelineId::fake_root_pipeline_id() == root_pipeline_id);
-        self.new_pipeline(root_pipeline_id, None, Some(window_size), None, LoadData::new(url.clone()));
+        self.new_pipeline(root_pipeline_id, None, Some(window_size), None, LoadData::new(url.clone(), None, None));
         self.handle_load_start_msg(&root_pipeline_id);
         self.push_pending_frame(root_pipeline_id, None);
         self.compositor_proxy.send(ToCompositorMsg::ChangePageUrl(root_pipeline_id, url));
@@ -1009,11 +1009,12 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
         };
 
         // Create the new pipeline, attached to the parent and push to pending frames
+        // TODO - loaddata here should have referrer info (not None, None)
         self.new_pipeline(load_info.new_pipeline_id,
                           Some((load_info.containing_pipeline_id, load_info.new_subpage_id)),
                           window_size,
                           script_chan,
-                          LoadData::new(new_url));
+                          LoadData::new(new_url, None, None));
 
         self.subpage_map.insert((load_info.containing_pipeline_id, load_info.new_subpage_id),
                                 load_info.new_pipeline_id);
@@ -1419,7 +1420,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
             },
             WebDriverCommandMsg::Refresh(pipeline_id, reply) => {
                 let load_data = match self.pipelines.get(&pipeline_id) {
-                    Some(pipeline) => LoadData::new(pipeline.url.clone()),
+                    Some(pipeline) => LoadData::new(pipeline.url.clone(), None, None),
                     None => return warn!("Pipeline {:?} Refresh after closure.", pipeline_id),
                 };
                 self.load_url_for_webdriver(pipeline_id, load_data, reply);

components/gfx/font_cache_thread.rs

@@ -170,6 +170,8 @@ impl FontCache {
                             let load = PendingAsyncLoad::new(LoadContext::Font,
                                                              self.resource_thread.clone(),
                                                              url.clone(),
+                                                             None,
+                                                             None,
                                                              None);
                             let (data_sender, data_receiver) = ipc::channel().unwrap();
                             let data_target = AsyncResponseTarget {

components/msg/constellation_msg.rs

@@ -246,15 +246,19 @@ pub struct LoadData {
     pub method: Method,
     pub headers: Headers,
     pub data: Option<Vec<u8>>,
+    pub referrer_policy: Option<ReferrerPolicy>,
+    pub referrer_url: Option<Url>,
 }
 
 impl LoadData {
-    pub fn new(url: Url) -> LoadData {
+    pub fn new(url: Url, referrer_policy: Option<ReferrerPolicy>, referrer_url: Option<Url>) -> LoadData {
         LoadData {
             url: url,
             method: Method::Get,
             headers: Headers::new(),
             data: None,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url,
         }
     }
 }
@@ -385,3 +389,15 @@ impl ConvertPipelineIdFromWebRender for webrender_traits::PipelineId {
         }
     }
 }
+
+/// [Policies](https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states)
+/// for providing a referrer header for a request
+#[derive(HeapSizeOf, Clone, Deserialize, Serialize)]
+pub enum ReferrerPolicy {
+    NoReferrer,
+    NoRefWhenDowngrade,
+    OriginOnly,
+    OriginWhenCrossOrigin,
+    UnsafeUrl,
+}
+

components/net/file_loader.rs

@@ -84,7 +84,7 @@ pub fn factory(load_data: LoadData,
                 // http://doc.rust-lang.org/std/fs/struct.OpenOptions.html#method.open
                 // but, we'll go for a "file not found!"
                 let url = Url::parse("about:not-found").unwrap();
-                let load_data_404 = LoadData::new(load_data.context, url, None);
+                let load_data_404 = LoadData::new(load_data.context, url, None, None, None);
                 about_loader::factory(load_data_404, senders, classifier, cancel_listener);
                 return;
             }

components/net/http_loader.rs

@@ -12,7 +12,7 @@ use flate2::read::{DeflateDecoder, GzDecoder};
 use hsts::{HstsEntry, HstsList, secure_url};
 use hyper::Error as HttpError;
 use hyper::client::{Pool, Request, Response};
-use hyper::header::{Accept, AcceptEncoding, ContentLength, ContentType, Host};
+use hyper::header::{Accept, AcceptEncoding, ContentLength, ContentType, Host, Referer};
 use hyper::header::{Authorization, Basic};
 use hyper::header::{ContentEncoding, Encoding, Header, Headers, Quality, QualityItem};
 use hyper::header::{Location, SetCookie, StrictTransportSecurity, UserAgent, qitem};
@@ -23,7 +23,7 @@ use hyper::net::{Fresh, HttpsConnector, Openssl};
 use hyper::status::{StatusClass, StatusCode};
 use log;
 use mime_classifier::MIMEClassifier;
-use msg::constellation_msg::{PipelineId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use net_traits::ProgressMsg::{Done, Payload};
 use net_traits::hosts::replace_hosts;
 use net_traits::response::HttpsState;
@@ -356,6 +356,49 @@ fn set_default_accept(headers: &mut Headers) {
     }
 }
 
+/// https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-state-no-referrer-when-downgrade
+fn no_ref_when_downgrade_header(referrer_url: Url, url: Url) -> Option<Url> {
+    if referrer_url.scheme() == "https" && url.scheme() != "https" {
+        return None;
+    }
+    return strip_url(referrer_url, false);
+}
+
+/// https://w3c.github.io/webappsec-referrer-policy/#strip-url
+fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option<Url> {
+    if referrer_url.scheme() == "https" || referrer_url.scheme() == "http" {
+        referrer_url.set_username("").unwrap();
+        referrer_url.set_password(None).unwrap();
+        referrer_url.set_fragment(None);
+        if origin_only {
+            referrer_url.set_path("");
+            referrer_url.set_query(None);
+        }
+        return Some(referrer_url);
+    }
+    return None;
+}
+
+/// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
+fn determine_request_referrer(headers: &mut Headers,
+                              referrer_policy: Option<ReferrerPolicy>,
+                              referrer_url: Option<Url>,
+                              url: Url) -> Option<Url> {
+    //TODO - algorithm step 2 not addressed
+    assert!(!headers.has::<Referer>());
+    if let Some(ref_url) = referrer_url {
+        let cross_origin = ref_url.origin() != url.origin();
+        return match referrer_policy {
+            Some(ReferrerPolicy::NoReferrer) => None,
+            Some(ReferrerPolicy::OriginOnly) => strip_url(ref_url, true),
+            Some(ReferrerPolicy::UnsafeUrl) => strip_url(ref_url, false),
+            Some(ReferrerPolicy::OriginWhenCrossOrigin) => strip_url(ref_url, cross_origin),
+            Some(ReferrerPolicy::NoRefWhenDowngrade) | None => no_ref_when_downgrade_header(ref_url, url),
+        };
+    }
+    return None;
+}
+
 pub fn set_request_cookies(url: Url, headers: &mut Headers, cookie_jar: &Arc<RwLock<CookieStorage>>) {
     let mut cookie_jar = cookie_jar.write().unwrap();
     if let Some(cookie_list) = cookie_jar.cookies_for_url(&url, CookieSource::HTTP) {
@@ -539,6 +582,13 @@ pub fn modify_request_headers(headers: &mut Headers,
     set_default_accept(headers);
     set_default_accept_encoding(headers);
 
+    if let Some(referer_val) = determine_request_referrer(headers,
+                                                          load_data.referrer_policy.clone(),
+                                                          load_data.referrer_url.clone(),
+                                                          url.clone()) {
+        headers.set(Referer(referer_val.into_string()));
+    }
+
     // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
     if load_data.credentials_flag {
         set_request_cookies(url.clone(), headers, cookie_jar);

components/net/image_cache_thread.rs

@@ -517,7 +517,7 @@ impl ImageCache {
                     CacheResult::Miss => {
                         // A new load request! Request the load from
                         // the resource thread.
-                        let load_data = LoadData::new(LoadContext::Image, (*ref_url).clone(), None);
+                        let load_data = LoadData::new(LoadContext::Image, (*ref_url).clone(), None, None, None);
                         let (action_sender, action_receiver) = ipc::channel().unwrap();
                         let response_target = AsyncResponseTarget {
                             sender: action_sender,

components/net_traits/lib.rs

@@ -33,7 +33,7 @@ use hyper::http::RawStatus;
 use hyper::method::Method;
 use hyper::mime::{Attr, Mime};
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
-use msg::constellation_msg::{PipelineId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use serde::{Deserializer, Serializer};
 use std::sync::mpsc::Sender;
 use std::thread;
@@ -88,10 +88,18 @@ pub struct LoadData {
     // https://fetch.spec.whatwg.org/#concept-http-fetch step 4.3
     pub credentials_flag: bool,
     pub context: LoadContext,
+    /// The policy and referring URL for the originator of this request
+    pub referrer_policy: Option<ReferrerPolicy>,
+    pub referrer_url: Option<Url>,
+
 }
 
 impl LoadData {
-    pub fn new(context: LoadContext, url: Url, id: Option<PipelineId>) -> LoadData {
+    pub fn new(context: LoadContext,
+               url: Url,
+               id: Option<PipelineId>,
+               referrer_policy: Option<ReferrerPolicy>,
+               referrer_url: Option<Url>) -> LoadData {
         LoadData {
             url: url,
             method: Method::Get,
@@ -101,7 +109,9 @@ impl LoadData {
             cors: None,
             pipeline_id: id,
             credentials_flag: true,
-            context: context
+            context: context,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url
         }
     }
 }
@@ -235,6 +245,8 @@ pub struct PendingAsyncLoad {
     pipeline: Option<PipelineId>,
     guard: PendingLoadGuard,
     context: LoadContext,
+    referrer_policy: Option<ReferrerPolicy>,
+    referrer_url: Option<Url>,
 }
 
 struct PendingLoadGuard {
@@ -256,21 +268,28 @@ impl Drop for PendingLoadGuard {
 }
 
 impl PendingAsyncLoad {
-    pub fn new(context: LoadContext, resource_thread: ResourceThread, url: Url, pipeline: Option<PipelineId>)
+    pub fn new(context: LoadContext,
+               resource_thread: ResourceThread,
+               url: Url,
+               pipeline: Option<PipelineId>,
+               referrer_policy: Option<ReferrerPolicy>,
+               referrer_url: Option<Url>)
                -> PendingAsyncLoad {
         PendingAsyncLoad {
             resource_thread: resource_thread,
             url: url,
             pipeline: pipeline,
             guard: PendingLoadGuard { loaded: false, },
-            context: context
+            context: context,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url
         }
     }
 
     /// Initiate the network request associated with this pending load, using the provided target.
     pub fn load_async(mut self, listener: AsyncResponseTarget) {
         self.guard.neuter();
-        let load_data = LoadData::new(self.context, self.url, self.pipeline);
+        let load_data = LoadData::new(self.context, self.url, self.pipeline, self.referrer_policy, self.referrer_url);
         let consumer = LoadConsumer::Listener(listener);
         self.resource_thread.send(ControlMsg::Load(load_data, consumer, None)).unwrap();
     }
@@ -387,7 +406,7 @@ pub fn load_whole_resource(context: LoadContext,
                            pipeline_id: Option<PipelineId>)
         -> Result<(Metadata, Vec<u8>), NetworkError> {
     let (start_chan, start_port) = ipc::channel().unwrap();
-    resource_thread.send(ControlMsg::Load(LoadData::new(context, url, pipeline_id),
+    resource_thread.send(ControlMsg::Load(LoadData::new(context, url, pipeline_id, None, None),
                        LoadConsumer::Channel(start_chan), None)).unwrap();
     let response = start_port.recv().unwrap();
 

components/script/document_loader.rs

@@ -127,16 +127,21 @@ impl DocumentLoader {
 
     /// Create a new pending network request, which can be initiated at some point in
     /// the future.
-    pub fn prepare_async_load(&mut self, load: LoadType) -> PendingAsyncLoad {
+    pub fn prepare_async_load(&mut self, load: LoadType, referrer: &Document) -> PendingAsyncLoad {
         let context = load.to_load_context();
         let url = load.url().clone();
         self.add_blocking_load(load);
-        PendingAsyncLoad::new(context, (*self.resource_thread).clone(), url, self.pipeline)
+        PendingAsyncLoad::new(context,
+                              (*self.resource_thread).clone(),
+                              url,
+                              self.pipeline,
+                              referrer.get_referrer_policy(),
+                              Some(referrer.url().clone()))
     }
 
     /// Create and initiate a new network request.
-    pub fn load_async(&mut self, load: LoadType, listener: AsyncResponseTarget) {
+    pub fn load_async(&mut self, load: LoadType, listener: AsyncResponseTarget, referrer: &Document) {
-        let pending = self.prepare_async_load(load);
+        let pending = self.prepare_async_load(load, referrer);
         pending.load_async(listener)
     }
 

components/script/dom/bindings/trace.rs

@@ -56,7 +56,7 @@ use js::rust::Runtime;
 use layout_interface::{LayoutChan, LayoutRPC};
 use libc;
 use msg::constellation_msg::ConstellationChan;
-use msg::constellation_msg::{PipelineId, SubpageId, WindowSizeData, WindowSizeType};
+use msg::constellation_msg::{PipelineId, SubpageId, WindowSizeData, WindowSizeType, ReferrerPolicy};
 use net_traits::image::base::{Image, ImageMetadata};
 use net_traits::image_cache_thread::{ImageCacheChan, ImageCacheThread};
 use net_traits::response::HttpsState;
@@ -325,6 +325,7 @@ no_jsmanaged_fields!(ElementSnapshot);
 no_jsmanaged_fields!(HttpsState);
 no_jsmanaged_fields!(SharedRt);
 no_jsmanaged_fields!(TouchpadPressurePhase);
+no_jsmanaged_fields!(ReferrerPolicy);
 
 impl JSTraceable for ConstellationChan<ScriptMsg> {
     #[inline]

components/script/dom/document.rs

@@ -90,7 +90,7 @@ use js::jsapi::{JSContext, JSObject, JSRuntime};
 use layout_interface::{LayoutChan, Msg, ReflowQueryType};
 use msg::constellation_msg::{ALT, CONTROL, SHIFT, SUPER};
 use msg::constellation_msg::{ConstellationChan, Key, KeyModifiers, KeyState};
-use msg::constellation_msg::{PipelineId, SubpageId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy, SubpageId};
 use net_traits::ControlMsg::{GetCookiesForUrl, SetCookiesForUrl};
 use net_traits::CookieSource::NonHTTP;
 use net_traits::response::HttpsState;
@@ -228,6 +228,8 @@ pub struct Document {
     touchpad_pressure_phase: Cell<TouchpadPressurePhase>,
     /// The document's origin.
     origin: Origin,
+    ///  https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states
+    referrer_policy: Option<ReferrerPolicy>,
 }
 
 #[derive(JSTraceable, HeapSizeOf)]
@@ -1328,12 +1330,12 @@ impl Document {
 
     pub fn prepare_async_load(&self, load: LoadType) -> PendingAsyncLoad {
         let mut loader = self.loader.borrow_mut();
-        loader.prepare_async_load(load)
+        loader.prepare_async_load(load, self)
     }
 
     pub fn load_async(&self, load: LoadType, listener: AsyncResponseTarget) {
         let mut loader = self.loader.borrow_mut();
-        loader.load_async(load, listener)
+        loader.load_async(load, listener, self)
     }
 
     pub fn finish_load(&self, load: LoadType) {
@@ -1686,6 +1688,8 @@ impl Document {
             https_state: Cell::new(HttpsState::None),
             touchpad_pressure_phase: Cell::new(TouchpadPressurePhase::BeforeClick),
             origin: origin,
+            //TODO - setting this for now so no Referer header set
+            referrer_policy: Some(ReferrerPolicy::NoReferrer),
         }
     }
 
@@ -1814,6 +1818,11 @@ impl Document {
             snapshot.attrs = Some(attrs);
         }
     }
+
+    //TODO - for now, returns no-referrer for all until reading in the value
+    pub fn get_referrer_policy(&self) -> Option<ReferrerPolicy> {
+        return self.referrer_policy.clone();
+    }
 }
 
 

components/script/dom/htmlformelement.rs

@@ -283,7 +283,7 @@ impl HTMLFormElement {
         let _target = submitter.target();
         // TODO: Handle browsing contexts, partially loaded documents (step 16-17)
 
-        let mut load_data = LoadData::new(action_components);
+        let mut load_data = LoadData::new(action_components, doc.get_referrer_policy(), Some(doc.url().clone()));
 
         let parsed_data = match enctype {
             FormEncType::UrlEncoded => {

components/script/dom/window.rs

@@ -1199,8 +1199,10 @@ impl Window {
 
     /// Commence a new URL load which will either replace this window or scroll to a fragment.
     pub fn load_url(&self, url: Url) {
+        let doc = self.Document();
         self.main_thread_script_chan().send(
-            MainThreadScriptMsg::Navigate(self.id, LoadData::new(url))).unwrap();
+            MainThreadScriptMsg::Navigate(self.id,
+                LoadData::new(url, doc.get_referrer_policy(), Some(doc.url().clone())))).unwrap();
     }
 
     pub fn handle_fire_timer(&self, timer_id: TimerEventId) {

components/script/dom/xmlhttprequest.rs

@@ -576,10 +576,13 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
         // Step 5
         let global = self.global();
         let pipeline_id = global.r().pipeline();
+        //TODO - set referrer_policy/referrer_url in load_data
         let mut load_data =
             LoadData::new(LoadContext::Browsing,
                           self.request_url.borrow().clone().unwrap(),
-                          Some(pipeline_id));
+                          Some(pipeline_id),
+                          None,
+                          None);
         if load_data.url.origin().ne(&global.r().get_url().origin()) {
             load_data.credentials_flag = self.WithCredentials();
         }

components/script/script_thread.rs

@@ -1887,6 +1887,8 @@ impl ScriptThread {
             cors: None,
             pipeline_id: Some(id),
             credentials_flag: true,
+            referrer_policy: load_data.referrer_policy,
+            referrer_url: load_data.referrer_url,
         }, LoadConsumer::Listener(response_target), None)).unwrap();
 
         self.incomplete_loads.borrow_mut().push(incomplete);

components/webdriver_server/lib.rs

@@ -322,7 +322,7 @@ impl Handler {
 
         let (sender, receiver) = ipc::channel().unwrap();
 
-        let load_data = LoadData::new(url);
+        let load_data = LoadData::new(url, None, None);
         let cmd_msg = WebDriverCommandMsg::LoadUrl(pipeline_id, load_data, sender.clone());
         self.constellation_chan.send(ConstellationMsg::WebDriverCommand(cmd_msg)).unwrap();
 

tests/unit/net/data_loader.rs

@@ -24,7 +24,7 @@ fn assert_parse(url:          &'static str,
 
     let (start_chan, start_port) = ipc::channel().unwrap();
     let classifier = Arc::new(MIMEClassifier::new());
-    load(LoadData::new(LoadContext::Browsing, Url::parse(url).unwrap(), None),
+    load(LoadData::new(LoadContext::Browsing, Url::parse(url).unwrap(), None, None, None),
          Channel(start_chan),
          classifier, CancellationListener::new(None));
 

tests/unit/net/http_loader.rs

@@ -10,13 +10,13 @@ use flate2::Compression;
 use flate2::write::{GzEncoder, DeflateEncoder};
 use hyper::header::{Accept, AcceptEncoding, ContentEncoding, ContentLength, Cookie as CookieHeader};
 use hyper::header::{Authorization, Basic};
-use hyper::header::{Encoding, Headers, Host, Location, Quality, QualityItem, qitem, SetCookie};
+use hyper::header::{Encoding, Headers, Host, Location, Quality, QualityItem, qitem, Referer, SetCookie};
 use hyper::header::{StrictTransportSecurity, UserAgent};
 use hyper::http::RawStatus;
 use hyper::method::Method;
 use hyper::mime::{Mime, SubLevel, TopLevel};
 use hyper::status::StatusCode;
-use msg::constellation_msg::PipelineId;
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use net::cookie::Cookie;
 use net::cookie_storage::CookieStorage;
 use net::hsts::HstsEntry;
@@ -328,7 +328,7 @@ fn test_check_default_headers_loaded_in_every_request() {
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
 
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     load_data.data = None;
     load_data.method = Method::Get;
 
@@ -372,7 +372,7 @@ fn test_load_when_request_is_not_get_or_head_and_there_is_no_body_content_length
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
 
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     load_data.data = None;
     load_data.method = Method::Post;
 
@@ -410,7 +410,7 @@ fn test_request_and_response_data_with_network_messages() {
     let (devtools_chan, devtools_port) = mpsc::channel::<DevtoolsControlMsg>();
     // This will probably have to be changed as it uses fake_root_pipeline_id which is marked for removal.
     let pipeline_id = PipelineId::fake_root_pipeline_id();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), Some(pipeline_id));
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), Some(pipeline_id), None, None);
     let mut request_headers = Headers::new();
     request_headers.set(Host { hostname: "bar.foo".to_owned(), port: None });
     load_data.headers = request_headers.clone();
@@ -483,7 +483,7 @@ fn test_request_and_response_message_from_devtool_without_pipeline_id() {
 
     let url = Url::parse("https://mozilla.com").unwrap();
     let (devtools_chan, devtools_port) = mpsc::channel::<DevtoolsControlMsg>();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     let _ = load(&load_data, &ui_provider, &http_state, Some(devtools_chan), &Factory,
                  DEFAULT_USER_AGENT.to_owned(), &CancellationListener::new(None));
 
@@ -512,7 +512,8 @@ fn test_load_when_redirecting_from_a_post_should_rewrite_next_request_as_get() {
     }
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
+
     load_data.method = Method::Post;
 
     let http_state = HttpState::new();
@@ -541,7 +542,7 @@ fn test_load_should_decode_the_response_as_deflate_when_response_headers_have_co
     }
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -575,7 +576,8 @@ fn test_load_should_decode_the_response_as_gzip_when_response_headers_have_conte
     }
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
+
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
 
@@ -617,7 +619,8 @@ fn test_load_doesnt_send_request_body_on_any_redirect() {
     }
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
+
     load_data.data = Some(<[_]>::to_vec("Body on POST!".as_bytes()));
 
     let http_state = HttpState::new();
@@ -648,7 +651,7 @@ fn test_load_doesnt_add_host_to_sts_list_when_url_is_http_even_if_sts_headers_ar
 
     let url = Url::parse("http://mozilla.com").unwrap();
 
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -680,7 +683,7 @@ fn test_load_adds_host_to_sts_list_when_url_is_https_and_sts_headers_are_present
 
     let url = Url::parse("https://mozilla.com").unwrap();
 
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -717,7 +720,7 @@ fn test_load_sets_cookies_in_the_resource_manager_when_it_get_set_cookie_header_
 
     assert_cookie_for_domain(http_state.cookie_jar.clone(), "http://mozilla.com", "");
 
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let _ = load(&load_data,
                  &ui_provider, &http_state,
@@ -733,7 +736,7 @@ fn test_load_sets_cookies_in_the_resource_manager_when_it_get_set_cookie_header_
 fn test_load_sets_requests_cookies_header_for_url_by_getting_cookies_from_the_resource_manager() {
     let url = Url::parse("http://mozilla.com").unwrap();
 
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
 
     let http_state = HttpState::new();
@@ -789,7 +792,7 @@ fn test_load_sends_secure_cookie_if_http_changed_to_https_due_to_entry_in_hsts_s
         cookie_jar.push(cookie, CookieSource::HTTP);
     }
 
-    let mut load_data = LoadData::new(LoadContext::Browsing, url, None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url, None, None, None);
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
 
     let mut headers = Headers::new();
@@ -821,7 +824,7 @@ fn test_load_sends_cookie_if_nonhttp() {
         cookie_jar.push(cookie, CookieSource::HTTP);
     }
 
-    let mut load_data = LoadData::new(LoadContext::Browsing, url, None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url, None, None, None);
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
 
     let mut headers = Headers::new();
@@ -855,7 +858,7 @@ fn test_cookie_set_with_httponly_should_not_be_available_using_getcookiesforurl(
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
 
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     let _ = load(&load_data,
                  &ui_provider, &http_state,
                  None,
@@ -885,7 +888,7 @@ fn test_when_cookie_received_marked_secure_is_ignored_for_http() {
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
 
-    let load_data = LoadData::new(LoadContext::Browsing, Url::parse("http://mozilla.com").unwrap(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, Url::parse("http://mozilla.com").unwrap(), None, None, None);
     let _ = load(&load_data,
                  &ui_provider, &http_state,
                  None,
@@ -916,7 +919,7 @@ fn test_when_cookie_set_marked_httpsonly_secure_isnt_sent_on_http_request() {
         cookie_jar.push(cookie, CookieSource::HTTP);
     }
 
-    let mut load_data = LoadData::new(LoadContext::Browsing, url, None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url, None, None, None);
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
 
     assert_cookie_for_domain(http_state.cookie_jar.clone(), "https://mozilla.com", "mozillaIs=theBest");
@@ -934,7 +937,8 @@ fn test_load_sets_content_length_to_length_of_request_body() {
     let content = "This is a request body";
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
+
     load_data.data = Some(<[_]>::to_vec(content.as_bytes()));
 
     let mut content_len_headers = Headers::new();
@@ -959,7 +963,8 @@ fn test_load_uses_explicit_accept_from_headers_in_load_data() {
     accept_headers.set(Accept(vec![text_html.clone()]));
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
+
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
     load_data.headers.set(Accept(vec![text_html.clone()]));
 
@@ -987,7 +992,8 @@ fn test_load_sets_default_accept_to_html_xhtml_xml_and_then_anything_else() {
     ]));
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
+
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
 
     let http_state = HttpState::new();
@@ -1009,7 +1015,7 @@ fn test_load_uses_explicit_accept_encoding_from_load_data_headers() {
     accept_encoding_headers.set(AcceptEncoding(vec![qitem(Encoding::Chunked)]));
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
     load_data.headers.set(AcceptEncoding(vec![qitem(Encoding::Chunked)]));
 
@@ -1034,7 +1040,7 @@ fn test_load_sets_default_accept_encoding_to_gzip_and_deflate() {
                                                     qitem(Encoding::EncodingExt("br".to_owned()))]));
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     load_data.data = Some(<[_]>::to_vec("Yay!".as_bytes()));
 
     let http_state = HttpState::new();
@@ -1069,7 +1075,7 @@ fn test_load_errors_when_there_a_redirect_loop() {
     }
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -1099,7 +1105,7 @@ fn test_load_errors_when_there_is_too_many_redirects() {
     }
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -1146,8 +1152,7 @@ fn test_load_follows_a_redirect() {
     }
 
     let url = Url::parse("http://mozilla.com").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
-
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
 
@@ -1174,7 +1179,7 @@ impl HttpRequestFactory for DontConnectFactory {
 #[test]
 fn test_load_errors_when_scheme_is_not_http_or_https() {
     let url = Url::parse("ftp://not-supported").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -1193,7 +1198,7 @@ fn test_load_errors_when_scheme_is_not_http_or_https() {
 #[test]
 fn test_load_errors_when_viewing_source_and_inner_url_scheme_is_not_http_or_https() {
     let url = Url::parse("view-source:ftp://not-supported").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -1236,7 +1241,7 @@ fn test_load_errors_when_cancelled() {
     cancel_sender.send(()).unwrap();
 
     let url = Url::parse("https://mozilla.com").unwrap();
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
 
@@ -1285,7 +1290,7 @@ fn  test_redirect_from_x_to_y_provides_y_cookies_from_y() {
         }
     }
 
-    let load_data = LoadData::new(LoadContext::Browsing, url_x.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url_x.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -1351,7 +1356,7 @@ fn test_redirect_from_x_to_x_provides_x_with_cookie_from_first_response() {
         }
     }
 
-    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None);
+    let load_data = LoadData::new(LoadContext::Browsing, url.clone(), None, None, None);
 
     let http_state = HttpState::new();
     let ui_provider = TestProvider::new();
@@ -1384,7 +1389,7 @@ fn test_if_auth_creds_not_in_url_but_in_cache_it_sets_it() {
 
     http_state.auth_cache.write().unwrap().entries.insert(url.clone(), auth_entry);
 
-    let mut load_data = LoadData::new(LoadContext::Browsing, url, None);
+    let mut load_data = LoadData::new(LoadContext::Browsing, url, None, None, None);
     load_data.credentials_flag = true;
 
     let mut auth_header = Headers::new();
@@ -1423,7 +1428,7 @@ fn test_auth_ui_sets_header_on_401() {
        )
     );
 
-    let load_data = LoadData::new(LoadContext::Browsing, url, None);
+    let load_data = LoadData::new(LoadContext::Browsing, url, None, None, None);
 
     match load(
         &load_data, &ui_provider, &http_state,
@@ -1437,3 +1442,188 @@ fn test_auth_ui_sets_header_on_401() {
         }
     }
 }
+
+fn assert_referer_header_matches(request_url: &str,
+                                 referrer_url: &str,
+                                 referrer_policy: Option<ReferrerPolicy>,
+                                 expected_referrer: &str) {
+    let ref_url = Url::parse(referrer_url).unwrap();
+    let url = Url::parse(request_url).unwrap();
+    let ui_provider = TestProvider::new();
+
+    let load_data = LoadData::new(LoadContext::Browsing,
+                                  url.clone(),
+                                  None,
+                                  referrer_policy,
+                                  Some(ref_url));
+
+    let mut referer_headers = Headers::new();
+    referer_headers.set(Referer(expected_referrer.to_owned()));
+
+    let http_state = HttpState::new();
+
+    let _ = load(&load_data.clone(), &ui_provider, &http_state, None,
+                 &AssertMustIncludeHeadersRequestFactory {
+                     expected_headers: referer_headers,
+                     body: <[_]>::to_vec(&[])
+                 }, DEFAULT_USER_AGENT.to_owned(),
+                 &CancellationListener::new(None));
+}
+
+fn assert_referer_header_not_included(request_url: &str, referrer_url: &str, referrer_policy: Option<ReferrerPolicy>) {
+    let ref_url = Url::parse(referrer_url).unwrap();
+    let url = Url::parse(request_url).unwrap();
+    let ui_provider = TestProvider::new();
+
+    let load_data = LoadData::new(LoadContext::Browsing,
+                                  url.clone(),
+                                  None,
+                                  referrer_policy,
+                                  Some(ref_url));
+
+    let http_state = HttpState::new();
+
+    let _ = load(
+        &load_data.clone(), &ui_provider, &http_state, None,
+        &AssertMustNotIncludeHeadersRequestFactory {
+            headers_not_expected: vec!["Referer".to_owned()],
+            body: <[_]>::to_vec(&[])
+        }, DEFAULT_USER_AGENT.to_owned(), &CancellationListener::new(None));
+}
+
+#[test]
+fn test_referer_set_to_origin_with_originonly_policy() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "http://username:password@someurl.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::OriginOnly);
+    let expected_referrer = "http://someurl.com/";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_referer_set_to_stripped_url_with_unsafeurl_policy() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "http://username:password@someurl.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::UnsafeUrl);
+    let expected_referrer = "http://someurl.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_referer_with_originwhencrossorigin_policy_cross_orig() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "http://username:password@someurl.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::OriginWhenCrossOrigin);
+    let expected_referrer = "http://someurl.com/";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_referer_with_originwhencrossorigin_policy_same_orig() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "http://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::OriginWhenCrossOrigin);
+    let expected_referrer = "http://mozilla.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_http_to_https_considered_cross_origin_for_referer_header_logic() {
+    let request_url = "https://mozilla.com";
+    let referrer_url = "http://mozilla.com/some/path";
+    let referrer_policy = Some(ReferrerPolicy::OriginWhenCrossOrigin);
+    let expected_referrer = "http://mozilla.com/";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_referer_set_to_ref_url_with_noreferrerwhendowngrade_policy_https_to_https() {
+    let request_url = "https://mozilla.com";
+    let referrer_url = "https://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::NoRefWhenDowngrade);
+    let expected_referrer = "https://mozilla.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_no_referer_set_with_noreferrerwhendowngrade_policy_https_to_http() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "https://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::NoRefWhenDowngrade);
+
+    assert_referer_header_not_included(request_url, referrer_url, referrer_policy)
+}
+
+#[test]
+fn test_referer_set_to_ref_url_with_noreferrerwhendowngrade_policy_http_to_https() {
+    let request_url = "https://mozilla.com";
+    let referrer_url = "http://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::NoRefWhenDowngrade);
+    let expected_referrer = "http://mozilla.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_referer_set_to_ref_url_with_noreferrerwhendowngrade_policy_http_to_http() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "http://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = Some(ReferrerPolicy::NoRefWhenDowngrade);
+    let expected_referrer = "http://mozilla.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_no_referrer_policy_follows_noreferrerwhendowngrade_https_to_https() {
+    let request_url = "https://mozilla.com";
+    let referrer_url = "https://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = None;
+    let expected_referrer = "https://mozilla.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_no_referrer_policy_follows_noreferrerwhendowngrade_https_to_http() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "https://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = None;
+
+    assert_referer_header_not_included(request_url, referrer_url, referrer_policy);
+}
+
+#[test]
+fn test_no_referrer_policy_follows_noreferrerwhendowngrade_http_to_https() {
+    let request_url = "https://mozilla.com";
+    let referrer_url = "http://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = None;
+    let expected_referrer = "http://mozilla.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_no_referrer_policy_follows_noreferrerwhendowngrade_http_to_http() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "http://username:password@mozilla.com/some/path#fragment";
+    let referrer_policy = None;
+    let expected_referrer = "http://mozilla.com/some/path";
+
+    assert_referer_header_matches(request_url, referrer_url, referrer_policy, expected_referrer);
+}
+
+#[test]
+fn test_no_referer_set_with_noreferrer_policy() {
+    let request_url = "http://mozilla.com";
+    let referrer_url = "http://someurl.com";
+    let referrer_policy = Some(ReferrerPolicy::NoReferrer);
+
+    assert_referer_header_not_included(request_url, referrer_url, referrer_policy)
+}

tests/unit/net/resource_thread.rs

@@ -27,8 +27,9 @@ fn test_bad_scheme() {
     let resource_thread = new_resource_thread("".to_owned(), None);
     let (start_chan, start) = ipc::channel().unwrap();
     let url = Url::parse("bogus://whatever").unwrap();
-    resource_thread.send(ControlMsg::Load(LoadData::new(LoadContext::Browsing, url, None),
+    resource_thread.send(ControlMsg::Load(LoadData::new(LoadContext::Browsing, url, None, None, None),
-                                        LoadConsumer::Channel(start_chan), None)).unwrap();
+
+    LoadConsumer::Channel(start_chan), None)).unwrap();
     let response = start.recv().unwrap();
     match response.progress_port.recv().unwrap() {
       ProgressMsg::Done(result) => { assert!(result.is_err()) }
@@ -205,7 +206,7 @@ fn test_cancelled_listener() {
     let (sync_sender, sync_receiver) = ipc::channel().unwrap();
     let url = Url::parse(&format!("http://127.0.0.1:{}", port)).unwrap();
 
-    resource_thread.send(ControlMsg::Load(LoadData::new(LoadContext::Browsing, url, None),
+    resource_thread.send(ControlMsg::Load(LoadData::new(LoadContext::Browsing, url, None, None, None),
                                         LoadConsumer::Channel(sender),
                                         Some(id_sender))).unwrap();
     // get the `ResourceId` and send a cancel message, which should stop the loading loop