Add referrer policy pass-through and referrer header logic

add pass-through from doc to http-loader for referrer_policy, ref_URL
add logic for setting referer header
add script pass-through for referrer
add unit tests for setting referer header

components/compositing/compositor.rs

@@ -1305,7 +1305,7 @@ impl<Window: WindowMethods> IOCompositor<Window> {
             Ok(url) => {
                 self.window.set_page_url(url.clone());
                 let msg = match self.scene.root {
-                    Some(ref layer) => ConstellationMsg::LoadUrl(layer.pipeline_id(), LoadData::new(url)),
+                    Some(ref layer) => ConstellationMsg::LoadUrl(layer.pipeline_id(), LoadData::new(url, None, None)),
                     None => ConstellationMsg::InitLoadUrl(url)
                 };
                 if let Err(e) = self.constellation_chan.send(msg) {

components/compositing/constellation.rs

@@ -887,7 +887,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
                           parent_info,
                           window_size,
                           None,
-                          LoadData::new(Url::parse("about:failure").expect("infallible")));
+                          LoadData::new(Url::parse("about:failure").expect("infallible"), None, None));
 
         self.push_pending_frame(new_pipeline_id, Some(pipeline_id));
 
@@ -897,7 +897,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
         let window_size = self.window_size.visible_viewport;
         let root_pipeline_id = PipelineId::new();
         debug_assert!(PipelineId::fake_root_pipeline_id() == root_pipeline_id);
-        self.new_pipeline(root_pipeline_id, None, Some(window_size), None, LoadData::new(url.clone()));
+        self.new_pipeline(root_pipeline_id, None, Some(window_size), None, LoadData::new(url.clone(), None, None));
         self.handle_load_start_msg(&root_pipeline_id);
         self.push_pending_frame(root_pipeline_id, None);
         self.compositor_proxy.send(ToCompositorMsg::ChangePageUrl(root_pipeline_id, url));
@@ -1009,11 +1009,12 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
         };
 
         // Create the new pipeline, attached to the parent and push to pending frames
+        // TODO - loaddata here should have referrer info (not None, None)
         self.new_pipeline(load_info.new_pipeline_id,
                           Some((load_info.containing_pipeline_id, load_info.new_subpage_id)),
                           window_size,
                           script_chan,
-                          LoadData::new(new_url));
+                          LoadData::new(new_url, None, None));
 
         self.subpage_map.insert((load_info.containing_pipeline_id, load_info.new_subpage_id),
                                 load_info.new_pipeline_id);
@@ -1419,7 +1420,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
             },
             WebDriverCommandMsg::Refresh(pipeline_id, reply) => {
                 let load_data = match self.pipelines.get(&pipeline_id) {
-                    Some(pipeline) => LoadData::new(pipeline.url.clone()),
+                    Some(pipeline) => LoadData::new(pipeline.url.clone(), None, None),
                     None => return warn!("Pipeline {:?} Refresh after closure.", pipeline_id),
                 };
                 self.load_url_for_webdriver(pipeline_id, load_data, reply);

components/gfx/font_cache_thread.rs

@@ -170,6 +170,8 @@ impl FontCache {
                             let load = PendingAsyncLoad::new(LoadContext::Font,
                                                              self.resource_thread.clone(),
                                                              url.clone(),
+                                                             None,
+                                                             None,
                                                              None);
                             let (data_sender, data_receiver) = ipc::channel().unwrap();
                             let data_target = AsyncResponseTarget {

components/msg/constellation_msg.rs

@@ -246,15 +246,19 @@ pub struct LoadData {
     pub method: Method,
     pub headers: Headers,
     pub data: Option<Vec<u8>>,
+    pub referrer_policy: Option<ReferrerPolicy>,
+    pub referrer_url: Option<Url>,
 }
 
 impl LoadData {
-    pub fn new(url: Url) -> LoadData {
+    pub fn new(url: Url, referrer_policy: Option<ReferrerPolicy>, referrer_url: Option<Url>) -> LoadData {
         LoadData {
             url: url,
             method: Method::Get,
             headers: Headers::new(),
             data: None,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url,
         }
     }
 }
@@ -385,3 +389,15 @@ impl ConvertPipelineIdFromWebRender for webrender_traits::PipelineId {
         }
     }
 }
+
+/// [Policies](https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states)
+/// for providing a referrer header for a request
+#[derive(HeapSizeOf, Clone, Deserialize, Serialize)]
+pub enum ReferrerPolicy {
+    NoReferrer,
+    NoRefWhenDowngrade,
+    OriginOnly,
+    OriginWhenCrossOrigin,
+    UnsafeUrl,
+}
+

components/net/file_loader.rs

@@ -84,7 +84,7 @@ pub fn factory(load_data: LoadData,
                 // http://doc.rust-lang.org/std/fs/struct.OpenOptions.html#method.open
                 // but, we'll go for a "file not found!"
                 let url = Url::parse("about:not-found").unwrap();
-                let load_data_404 = LoadData::new(load_data.context, url, None);
+                let load_data_404 = LoadData::new(load_data.context, url, None, None, None);
                 about_loader::factory(load_data_404, senders, classifier, cancel_listener);
                 return;
             }

components/net/http_loader.rs

@@ -12,7 +12,7 @@ use flate2::read::{DeflateDecoder, GzDecoder};
 use hsts::{HstsEntry, HstsList, secure_url};
 use hyper::Error as HttpError;
 use hyper::client::{Pool, Request, Response};
-use hyper::header::{Accept, AcceptEncoding, ContentLength, ContentType, Host};
+use hyper::header::{Accept, AcceptEncoding, ContentLength, ContentType, Host, Referer};
 use hyper::header::{Authorization, Basic};
 use hyper::header::{ContentEncoding, Encoding, Header, Headers, Quality, QualityItem};
 use hyper::header::{Location, SetCookie, StrictTransportSecurity, UserAgent, qitem};
@@ -23,7 +23,7 @@ use hyper::net::{Fresh, HttpsConnector, Openssl};
 use hyper::status::{StatusClass, StatusCode};
 use log;
 use mime_classifier::MIMEClassifier;
-use msg::constellation_msg::{PipelineId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use net_traits::ProgressMsg::{Done, Payload};
 use net_traits::hosts::replace_hosts;
 use net_traits::response::HttpsState;
@@ -356,6 +356,49 @@ fn set_default_accept(headers: &mut Headers) {
     }
 }
 
+/// https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-state-no-referrer-when-downgrade
+fn no_ref_when_downgrade_header(referrer_url: Url, url: Url) -> Option<Url> {
+    if referrer_url.scheme() == "https" && url.scheme() != "https" {
+        return None;
+    }
+    return strip_url(referrer_url, false);
+}
+
+/// https://w3c.github.io/webappsec-referrer-policy/#strip-url
+fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option<Url> {
+    if referrer_url.scheme() == "https" || referrer_url.scheme() == "http" {
+        referrer_url.set_username("").unwrap();
+        referrer_url.set_password(None).unwrap();
+        referrer_url.set_fragment(None);
+        if origin_only {
+            referrer_url.set_path("");
+            referrer_url.set_query(None);
+        }
+        return Some(referrer_url);
+    }
+    return None;
+}
+
+/// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
+fn determine_request_referrer(headers: &mut Headers,
+                              referrer_policy: Option<ReferrerPolicy>,
+                              referrer_url: Option<Url>,
+                              url: Url) -> Option<Url> {
+    //TODO - algorithm step 2 not addressed
+    assert!(!headers.has::<Referer>());
+    if let Some(ref_url) = referrer_url {
+        let cross_origin = ref_url.origin() != url.origin();
+        return match referrer_policy {
+            Some(ReferrerPolicy::NoReferrer) => None,
+            Some(ReferrerPolicy::OriginOnly) => strip_url(ref_url, true),
+            Some(ReferrerPolicy::UnsafeUrl) => strip_url(ref_url, false),
+            Some(ReferrerPolicy::OriginWhenCrossOrigin) => strip_url(ref_url, cross_origin),
+            Some(ReferrerPolicy::NoRefWhenDowngrade) | None => no_ref_when_downgrade_header(ref_url, url),
+        };
+    }
+    return None;
+}
+
 pub fn set_request_cookies(url: Url, headers: &mut Headers, cookie_jar: &Arc<RwLock<CookieStorage>>) {
     let mut cookie_jar = cookie_jar.write().unwrap();
     if let Some(cookie_list) = cookie_jar.cookies_for_url(&url, CookieSource::HTTP) {
@@ -539,6 +582,13 @@ pub fn modify_request_headers(headers: &mut Headers,
     set_default_accept(headers);
     set_default_accept_encoding(headers);
 
+    if let Some(referer_val) = determine_request_referrer(headers,
+                                                          load_data.referrer_policy.clone(),
+                                                          load_data.referrer_url.clone(),
+                                                          url.clone()) {
+        headers.set(Referer(referer_val.into_string()));
+    }
+
     // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
     if load_data.credentials_flag {
         set_request_cookies(url.clone(), headers, cookie_jar);

components/net/image_cache_thread.rs

@@ -517,7 +517,7 @@ impl ImageCache {
                     CacheResult::Miss => {
                         // A new load request! Request the load from
                         // the resource thread.
-                        let load_data = LoadData::new(LoadContext::Image, (*ref_url).clone(), None);
+                        let load_data = LoadData::new(LoadContext::Image, (*ref_url).clone(), None, None, None);
                         let (action_sender, action_receiver) = ipc::channel().unwrap();
                         let response_target = AsyncResponseTarget {
                             sender: action_sender,

components/net_traits/lib.rs

@@ -33,7 +33,7 @@ use hyper::http::RawStatus;
 use hyper::method::Method;
 use hyper::mime::{Attr, Mime};
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
-use msg::constellation_msg::{PipelineId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use serde::{Deserializer, Serializer};
 use std::sync::mpsc::Sender;
 use std::thread;
@@ -88,10 +88,18 @@ pub struct LoadData {
     // https://fetch.spec.whatwg.org/#concept-http-fetch step 4.3
     pub credentials_flag: bool,
     pub context: LoadContext,
+    /// The policy and referring URL for the originator of this request
+    pub referrer_policy: Option<ReferrerPolicy>,
+    pub referrer_url: Option<Url>,
+
 }
 
 impl LoadData {
-    pub fn new(context: LoadContext, url: Url, id: Option<PipelineId>) -> LoadData {
+    pub fn new(context: LoadContext,
+               url: Url,
+               id: Option<PipelineId>,
+               referrer_policy: Option<ReferrerPolicy>,
+               referrer_url: Option<Url>) -> LoadData {
         LoadData {
             url: url,
             method: Method::Get,
@@ -101,7 +109,9 @@ impl LoadData {
             cors: None,
             pipeline_id: id,
             credentials_flag: true,
-            context: context
+            context: context,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url
         }
     }
 }
@@ -235,6 +245,8 @@ pub struct PendingAsyncLoad {
     pipeline: Option<PipelineId>,
     guard: PendingLoadGuard,
     context: LoadContext,
+    referrer_policy: Option<ReferrerPolicy>,
+    referrer_url: Option<Url>,
 }
 
 struct PendingLoadGuard {
@@ -256,21 +268,28 @@ impl Drop for PendingLoadGuard {
 }
 
 impl PendingAsyncLoad {
-    pub fn new(context: LoadContext, resource_thread: ResourceThread, url: Url, pipeline: Option<PipelineId>)
+    pub fn new(context: LoadContext,
+               resource_thread: ResourceThread,
+               url: Url,
+               pipeline: Option<PipelineId>,
+               referrer_policy: Option<ReferrerPolicy>,
+               referrer_url: Option<Url>)
                -> PendingAsyncLoad {
         PendingAsyncLoad {
             resource_thread: resource_thread,
             url: url,
             pipeline: pipeline,
             guard: PendingLoadGuard { loaded: false, },
-            context: context
+            context: context,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url
         }
     }
 
     /// Initiate the network request associated with this pending load, using the provided target.
     pub fn load_async(mut self, listener: AsyncResponseTarget) {
         self.guard.neuter();
-        let load_data = LoadData::new(self.context, self.url, self.pipeline);
+        let load_data = LoadData::new(self.context, self.url, self.pipeline, self.referrer_policy, self.referrer_url);
         let consumer = LoadConsumer::Listener(listener);
         self.resource_thread.send(ControlMsg::Load(load_data, consumer, None)).unwrap();
     }
@@ -387,7 +406,7 @@ pub fn load_whole_resource(context: LoadContext,
                            pipeline_id: Option<PipelineId>)
         -> Result<(Metadata, Vec<u8>), NetworkError> {
     let (start_chan, start_port) = ipc::channel().unwrap();
-    resource_thread.send(ControlMsg::Load(LoadData::new(context, url, pipeline_id),
+    resource_thread.send(ControlMsg::Load(LoadData::new(context, url, pipeline_id, None, None),
                        LoadConsumer::Channel(start_chan), None)).unwrap();
     let response = start_port.recv().unwrap();
 

components/script/document_loader.rs

@@ -127,16 +127,21 @@ impl DocumentLoader {
 
     /// Create a new pending network request, which can be initiated at some point in
     /// the future.
-    pub fn prepare_async_load(&mut self, load: LoadType) -> PendingAsyncLoad {
+    pub fn prepare_async_load(&mut self, load: LoadType, referrer: &Document) -> PendingAsyncLoad {
         let context = load.to_load_context();
         let url = load.url().clone();
         self.add_blocking_load(load);
-        PendingAsyncLoad::new(context, (*self.resource_thread).clone(), url, self.pipeline)
+        PendingAsyncLoad::new(context,
+                              (*self.resource_thread).clone(),
+                              url,
+                              self.pipeline,
+                              referrer.get_referrer_policy(),
+                              Some(referrer.url().clone()))
     }
 
     /// Create and initiate a new network request.
-    pub fn load_async(&mut self, load: LoadType, listener: AsyncResponseTarget) {
-        let pending = self.prepare_async_load(load);
+    pub fn load_async(&mut self, load: LoadType, listener: AsyncResponseTarget, referrer: &Document) {
+        let pending = self.prepare_async_load(load, referrer);
         pending.load_async(listener)
     }
 

components/script/dom/bindings/trace.rs

@@ -56,7 +56,7 @@ use js::rust::Runtime;
 use layout_interface::{LayoutChan, LayoutRPC};
 use libc;
 use msg::constellation_msg::ConstellationChan;
-use msg::constellation_msg::{PipelineId, SubpageId, WindowSizeData, WindowSizeType};
+use msg::constellation_msg::{PipelineId, SubpageId, WindowSizeData, WindowSizeType, ReferrerPolicy};
 use net_traits::image::base::{Image, ImageMetadata};
 use net_traits::image_cache_thread::{ImageCacheChan, ImageCacheThread};
 use net_traits::response::HttpsState;
@@ -325,6 +325,7 @@ no_jsmanaged_fields!(ElementSnapshot);
 no_jsmanaged_fields!(HttpsState);
 no_jsmanaged_fields!(SharedRt);
 no_jsmanaged_fields!(TouchpadPressurePhase);
+no_jsmanaged_fields!(ReferrerPolicy);
 
 impl JSTraceable for ConstellationChan<ScriptMsg> {
     #[inline]

components/script/dom/document.rs

@@ -88,7 +88,7 @@ use js::jsapi::{JSContext, JSObject, JSRuntime};
 use layout_interface::{LayoutChan, Msg, ReflowQueryType};
 use msg::constellation_msg::{ALT, CONTROL, SHIFT, SUPER};
 use msg::constellation_msg::{ConstellationChan, Key, KeyModifiers, KeyState};
-use msg::constellation_msg::{PipelineId, SubpageId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy, SubpageId};
 use net_traits::ControlMsg::{GetCookiesForUrl, SetCookiesForUrl};
 use net_traits::CookieSource::NonHTTP;
 use net_traits::response::HttpsState;
@@ -226,6 +226,8 @@ pub struct Document {
     touchpad_pressure_phase: Cell<TouchpadPressurePhase>,
     /// The document's origin.
     origin: Origin,
+    ///  https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states
+    referrer_policy: Option<ReferrerPolicy>,
 }
 
 #[derive(JSTraceable, HeapSizeOf)]
@@ -1326,12 +1328,12 @@ impl Document {
 
     pub fn prepare_async_load(&self, load: LoadType) -> PendingAsyncLoad {
         let mut loader = self.loader.borrow_mut();
-        loader.prepare_async_load(load)
+        loader.prepare_async_load(load, self)
     }
 
     pub fn load_async(&self, load: LoadType, listener: AsyncResponseTarget) {
         let mut loader = self.loader.borrow_mut();
-        loader.load_async(load, listener)
+        loader.load_async(load, listener, self)
     }
 
     pub fn finish_load(&self, load: LoadType) {
@@ -1684,6 +1686,8 @@ impl Document {
             https_state: Cell::new(HttpsState::None),
             touchpad_pressure_phase: Cell::new(TouchpadPressurePhase::BeforeClick),
             origin: origin,
+            //TODO - setting this for now so no Referer header set
+            referrer_policy: Some(ReferrerPolicy::NoReferrer),
         }
     }
 
@@ -1812,6 +1816,11 @@ impl Document {
             snapshot.attrs = Some(attrs);
         }
     }
+
+    //TODO - for now, returns no-referrer for all until reading in the value
+    pub fn get_referrer_policy(&self) -> Option<ReferrerPolicy> {
+        return self.referrer_policy.clone();
+    }
 }
 
 

components/script/dom/htmlformelement.rs

@@ -283,7 +283,7 @@ impl HTMLFormElement {
         let _target = submitter.target();
         // TODO: Handle browsing contexts, partially loaded documents (step 16-17)
 
-        let mut load_data = LoadData::new(action_components);
+        let mut load_data = LoadData::new(action_components, doc.get_referrer_policy(), Some(doc.url().clone()));
 
         let parsed_data = match enctype {
             FormEncType::UrlEncoded => {

components/script/dom/window.rs

@@ -1199,8 +1199,10 @@ impl Window {
 
     /// Commence a new URL load which will either replace this window or scroll to a fragment.
     pub fn load_url(&self, url: Url) {
+        let doc = self.Document();
         self.main_thread_script_chan().send(
-            MainThreadScriptMsg::Navigate(self.id, LoadData::new(url))).unwrap();
+            MainThreadScriptMsg::Navigate(self.id,
+                LoadData::new(url, doc.get_referrer_policy(), Some(doc.url().clone())))).unwrap();
     }
 
     pub fn handle_fire_timer(&self, timer_id: TimerEventId) {

components/script/dom/xmlhttprequest.rs

@@ -576,10 +576,13 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
         // Step 5
         let global = self.global();
         let pipeline_id = global.r().pipeline();
+        //TODO - set referrer_policy/referrer_url in load_data
         let mut load_data =
             LoadData::new(LoadContext::Browsing,
                           self.request_url.borrow().clone().unwrap(),
-                          Some(pipeline_id));
+                          Some(pipeline_id),
+                          None,
+                          None);
         if load_data.url.origin().ne(&global.r().get_url().origin()) {
             load_data.credentials_flag = self.WithCredentials();
         }

components/script/script_thread.rs

@@ -1887,6 +1887,8 @@ impl ScriptThread {
             cors: None,
             pipeline_id: Some(id),
             credentials_flag: true,
+            referrer_policy: load_data.referrer_policy,
+            referrer_url: load_data.referrer_url,
         }, LoadConsumer::Listener(response_target), None)).unwrap();
 
         self.incomplete_loads.borrow_mut().push(incomplete);

components/webdriver_server/lib.rs

@@ -322,7 +322,7 @@ impl Handler {
 
         let (sender, receiver) = ipc::channel().unwrap();
 
-        let load_data = LoadData::new(url);
+        let load_data = LoadData::new(url, None, None);
         let cmd_msg = WebDriverCommandMsg::LoadUrl(pipeline_id, load_data, sender.clone());
         self.constellation_chan.send(ConstellationMsg::WebDriverCommand(cmd_msg)).unwrap();