Auto merge of #10696 - rebstar6:referrerPolicy, r=jdm

Referer header

PR1 for https://github.com/servo/servo/issues/10311

This puts the code and data structures in place to set the Referer header based on the Referrer Policy for a given document. Note that document:: get_referrer_policy() always returns the 'No Referrer' option, so for now, this should have no impact on production code, and that policy requires that the Referer header is not added.

Later PRs will determine the policy and edit that get_referrer_policy() accordingly.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/10696)
<!-- Reviewable:end -->

components/compositing/compositor.rs

@@ -1305,7 +1305,7 @@ impl<Window: WindowMethods> IOCompositor<Window> {
             Ok(url) => {
                 self.window.set_page_url(url.clone());
                 let msg = match self.scene.root {
-                    Some(ref layer) => ConstellationMsg::LoadUrl(layer.pipeline_id(), LoadData::new(url)),
+                    Some(ref layer) => ConstellationMsg::LoadUrl(layer.pipeline_id(), LoadData::new(url, None, None)),
                     None => ConstellationMsg::InitLoadUrl(url)
                 };
                 if let Err(e) = self.constellation_chan.send(msg) {

components/compositing/constellation.rs

@@ -887,7 +887,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
                           parent_info,
                           window_size,
                           None,
-                          LoadData::new(Url::parse("about:failure").expect("infallible")));
+                          LoadData::new(Url::parse("about:failure").expect("infallible"), None, None));
 
         self.push_pending_frame(new_pipeline_id, Some(pipeline_id));
 
@@ -897,7 +897,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
         let window_size = self.window_size.visible_viewport;
         let root_pipeline_id = PipelineId::new();
         debug_assert!(PipelineId::fake_root_pipeline_id() == root_pipeline_id);
-        self.new_pipeline(root_pipeline_id, None, Some(window_size), None, LoadData::new(url.clone()));
+        self.new_pipeline(root_pipeline_id, None, Some(window_size), None, LoadData::new(url.clone(), None, None));
         self.handle_load_start_msg(&root_pipeline_id);
         self.push_pending_frame(root_pipeline_id, None);
         self.compositor_proxy.send(ToCompositorMsg::ChangePageUrl(root_pipeline_id, url));
@@ -1009,11 +1009,12 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
         };
 
         // Create the new pipeline, attached to the parent and push to pending frames
+        // TODO - loaddata here should have referrer info (not None, None)
         self.new_pipeline(load_info.new_pipeline_id,
                           Some((load_info.containing_pipeline_id, load_info.new_subpage_id)),
                           window_size,
                           script_chan,
-                          LoadData::new(new_url));
+                          LoadData::new(new_url, None, None));
 
         self.subpage_map.insert((load_info.containing_pipeline_id, load_info.new_subpage_id),
                                 load_info.new_pipeline_id);
@@ -1419,7 +1420,7 @@ impl<LTF: LayoutThreadFactory, STF: ScriptThreadFactory> Constellation<LTF, STF>
             },
             WebDriverCommandMsg::Refresh(pipeline_id, reply) => {
                 let load_data = match self.pipelines.get(&pipeline_id) {
-                    Some(pipeline) => LoadData::new(pipeline.url.clone()),
+                    Some(pipeline) => LoadData::new(pipeline.url.clone(), None, None),
                     None => return warn!("Pipeline {:?} Refresh after closure.", pipeline_id),
                 };
                 self.load_url_for_webdriver(pipeline_id, load_data, reply);

components/gfx/font_cache_thread.rs

@@ -170,6 +170,8 @@ impl FontCache {
                             let load = PendingAsyncLoad::new(LoadContext::Font,
                                                              self.resource_thread.clone(),
                                                              url.clone(),
+                                                             None,
+                                                             None,
                                                              None);
                             let (data_sender, data_receiver) = ipc::channel().unwrap();
                             let data_target = AsyncResponseTarget {

components/msg/constellation_msg.rs

@@ -246,15 +246,19 @@ pub struct LoadData {
     pub method: Method,
     pub headers: Headers,
     pub data: Option<Vec<u8>>,
+    pub referrer_policy: Option<ReferrerPolicy>,
+    pub referrer_url: Option<Url>,
 }
 
 impl LoadData {
-    pub fn new(url: Url) -> LoadData {
+    pub fn new(url: Url, referrer_policy: Option<ReferrerPolicy>, referrer_url: Option<Url>) -> LoadData {
         LoadData {
             url: url,
             method: Method::Get,
             headers: Headers::new(),
             data: None,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url,
         }
     }
 }
@@ -385,3 +389,15 @@ impl ConvertPipelineIdFromWebRender for webrender_traits::PipelineId {
         }
     }
 }
+
+/// [Policies](https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states)
+/// for providing a referrer header for a request
+#[derive(HeapSizeOf, Clone, Deserialize, Serialize)]
+pub enum ReferrerPolicy {
+    NoReferrer,
+    NoRefWhenDowngrade,
+    OriginOnly,
+    OriginWhenCrossOrigin,
+    UnsafeUrl,
+}
+

components/net/file_loader.rs

@@ -84,7 +84,7 @@ pub fn factory(load_data: LoadData,
                 // http://doc.rust-lang.org/std/fs/struct.OpenOptions.html#method.open
                 // but, we'll go for a "file not found!"
                 let url = Url::parse("about:not-found").unwrap();
-                let load_data_404 = LoadData::new(load_data.context, url, None);
+                let load_data_404 = LoadData::new(load_data.context, url, None, None, None);
                 about_loader::factory(load_data_404, senders, classifier, cancel_listener);
                 return;
             }

components/net/http_loader.rs

@@ -12,7 +12,7 @@ use flate2::read::{DeflateDecoder, GzDecoder};
 use hsts::{HstsEntry, HstsList, secure_url};
 use hyper::Error as HttpError;
 use hyper::client::{Pool, Request, Response};
-use hyper::header::{Accept, AcceptEncoding, ContentLength, ContentType, Host};
+use hyper::header::{Accept, AcceptEncoding, ContentLength, ContentType, Host, Referer};
 use hyper::header::{Authorization, Basic};
 use hyper::header::{ContentEncoding, Encoding, Header, Headers, Quality, QualityItem};
 use hyper::header::{Location, SetCookie, StrictTransportSecurity, UserAgent, qitem};
@@ -23,7 +23,7 @@ use hyper::net::{Fresh, HttpsConnector, Openssl};
 use hyper::status::{StatusClass, StatusCode};
 use log;
 use mime_classifier::MIMEClassifier;
-use msg::constellation_msg::{PipelineId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use net_traits::ProgressMsg::{Done, Payload};
 use net_traits::hosts::replace_hosts;
 use net_traits::response::HttpsState;
@@ -356,6 +356,49 @@ fn set_default_accept(headers: &mut Headers) {
     }
 }
 
+/// https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-state-no-referrer-when-downgrade
+fn no_ref_when_downgrade_header(referrer_url: Url, url: Url) -> Option<Url> {
+    if referrer_url.scheme() == "https" && url.scheme() != "https" {
+        return None;
+    }
+    return strip_url(referrer_url, false);
+}
+
+/// https://w3c.github.io/webappsec-referrer-policy/#strip-url
+fn strip_url(mut referrer_url: Url, origin_only: bool) -> Option<Url> {
+    if referrer_url.scheme() == "https" || referrer_url.scheme() == "http" {
+        referrer_url.set_username("").unwrap();
+        referrer_url.set_password(None).unwrap();
+        referrer_url.set_fragment(None);
+        if origin_only {
+            referrer_url.set_path("");
+            referrer_url.set_query(None);
+        }
+        return Some(referrer_url);
+    }
+    return None;
+}
+
+/// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
+fn determine_request_referrer(headers: &mut Headers,
+                              referrer_policy: Option<ReferrerPolicy>,
+                              referrer_url: Option<Url>,
+                              url: Url) -> Option<Url> {
+    //TODO - algorithm step 2 not addressed
+    assert!(!headers.has::<Referer>());
+    if let Some(ref_url) = referrer_url {
+        let cross_origin = ref_url.origin() != url.origin();
+        return match referrer_policy {
+            Some(ReferrerPolicy::NoReferrer) => None,
+            Some(ReferrerPolicy::OriginOnly) => strip_url(ref_url, true),
+            Some(ReferrerPolicy::UnsafeUrl) => strip_url(ref_url, false),
+            Some(ReferrerPolicy::OriginWhenCrossOrigin) => strip_url(ref_url, cross_origin),
+            Some(ReferrerPolicy::NoRefWhenDowngrade) | None => no_ref_when_downgrade_header(ref_url, url),
+        };
+    }
+    return None;
+}
+
 pub fn set_request_cookies(url: Url, headers: &mut Headers, cookie_jar: &Arc<RwLock<CookieStorage>>) {
     let mut cookie_jar = cookie_jar.write().unwrap();
     if let Some(cookie_list) = cookie_jar.cookies_for_url(&url, CookieSource::HTTP) {
@@ -539,6 +582,13 @@ pub fn modify_request_headers(headers: &mut Headers,
     set_default_accept(headers);
     set_default_accept_encoding(headers);
 
+    if let Some(referer_val) = determine_request_referrer(headers,
+                                                          load_data.referrer_policy.clone(),
+                                                          load_data.referrer_url.clone(),
+                                                          url.clone()) {
+        headers.set(Referer(referer_val.into_string()));
+    }
+
     // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
     if load_data.credentials_flag {
         set_request_cookies(url.clone(), headers, cookie_jar);

components/net/image_cache_thread.rs

@@ -517,7 +517,7 @@ impl ImageCache {
                     CacheResult::Miss => {
                         // A new load request! Request the load from
                         // the resource thread.
-                        let load_data = LoadData::new(LoadContext::Image, (*ref_url).clone(), None);
+                        let load_data = LoadData::new(LoadContext::Image, (*ref_url).clone(), None, None, None);
                         let (action_sender, action_receiver) = ipc::channel().unwrap();
                         let response_target = AsyncResponseTarget {
                             sender: action_sender,

components/net_traits/lib.rs

@@ -33,7 +33,7 @@ use hyper::http::RawStatus;
 use hyper::method::Method;
 use hyper::mime::{Attr, Mime};
 use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
-use msg::constellation_msg::{PipelineId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy};
 use serde::{Deserializer, Serializer};
 use std::sync::mpsc::Sender;
 use std::thread;
@@ -88,10 +88,18 @@ pub struct LoadData {
     // https://fetch.spec.whatwg.org/#concept-http-fetch step 4.3
     pub credentials_flag: bool,
     pub context: LoadContext,
+    /// The policy and referring URL for the originator of this request
+    pub referrer_policy: Option<ReferrerPolicy>,
+    pub referrer_url: Option<Url>,
+
 }
 
 impl LoadData {
-    pub fn new(context: LoadContext, url: Url, id: Option<PipelineId>) -> LoadData {
+    pub fn new(context: LoadContext,
+               url: Url,
+               id: Option<PipelineId>,
+               referrer_policy: Option<ReferrerPolicy>,
+               referrer_url: Option<Url>) -> LoadData {
         LoadData {
             url: url,
             method: Method::Get,
@@ -101,7 +109,9 @@ impl LoadData {
             cors: None,
             pipeline_id: id,
             credentials_flag: true,
-            context: context
+            context: context,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url
         }
     }
 }
@@ -235,6 +245,8 @@ pub struct PendingAsyncLoad {
     pipeline: Option<PipelineId>,
     guard: PendingLoadGuard,
     context: LoadContext,
+    referrer_policy: Option<ReferrerPolicy>,
+    referrer_url: Option<Url>,
 }
 
 struct PendingLoadGuard {
@@ -256,21 +268,28 @@ impl Drop for PendingLoadGuard {
 }
 
 impl PendingAsyncLoad {
-    pub fn new(context: LoadContext, resource_thread: ResourceThread, url: Url, pipeline: Option<PipelineId>)
+    pub fn new(context: LoadContext,
+               resource_thread: ResourceThread,
+               url: Url,
+               pipeline: Option<PipelineId>,
+               referrer_policy: Option<ReferrerPolicy>,
+               referrer_url: Option<Url>)
                -> PendingAsyncLoad {
         PendingAsyncLoad {
             resource_thread: resource_thread,
             url: url,
             pipeline: pipeline,
             guard: PendingLoadGuard { loaded: false, },
-            context: context
+            context: context,
+            referrer_policy: referrer_policy,
+            referrer_url: referrer_url
         }
     }
 
     /// Initiate the network request associated with this pending load, using the provided target.
     pub fn load_async(mut self, listener: AsyncResponseTarget) {
         self.guard.neuter();
-        let load_data = LoadData::new(self.context, self.url, self.pipeline);
+        let load_data = LoadData::new(self.context, self.url, self.pipeline, self.referrer_policy, self.referrer_url);
         let consumer = LoadConsumer::Listener(listener);
         self.resource_thread.send(ControlMsg::Load(load_data, consumer, None)).unwrap();
     }
@@ -387,7 +406,7 @@ pub fn load_whole_resource(context: LoadContext,
                            pipeline_id: Option<PipelineId>)
         -> Result<(Metadata, Vec<u8>), NetworkError> {
     let (start_chan, start_port) = ipc::channel().unwrap();
-    resource_thread.send(ControlMsg::Load(LoadData::new(context, url, pipeline_id),
+    resource_thread.send(ControlMsg::Load(LoadData::new(context, url, pipeline_id, None, None),
                        LoadConsumer::Channel(start_chan), None)).unwrap();
     let response = start_port.recv().unwrap();
 

components/script/document_loader.rs

@@ -127,16 +127,21 @@ impl DocumentLoader {
 
     /// Create a new pending network request, which can be initiated at some point in
     /// the future.
-    pub fn prepare_async_load(&mut self, load: LoadType) -> PendingAsyncLoad {
+    pub fn prepare_async_load(&mut self, load: LoadType, referrer: &Document) -> PendingAsyncLoad {
         let context = load.to_load_context();
         let url = load.url().clone();
         self.add_blocking_load(load);
-        PendingAsyncLoad::new(context, (*self.resource_thread).clone(), url, self.pipeline)
+        PendingAsyncLoad::new(context,
+                              (*self.resource_thread).clone(),
+                              url,
+                              self.pipeline,
+                              referrer.get_referrer_policy(),
+                              Some(referrer.url().clone()))
     }
 
     /// Create and initiate a new network request.
-    pub fn load_async(&mut self, load: LoadType, listener: AsyncResponseTarget) {
-        let pending = self.prepare_async_load(load);
+    pub fn load_async(&mut self, load: LoadType, listener: AsyncResponseTarget, referrer: &Document) {
+        let pending = self.prepare_async_load(load, referrer);
         pending.load_async(listener)
     }
 

components/script/dom/bindings/trace.rs

@@ -56,7 +56,7 @@ use js::rust::Runtime;
 use layout_interface::{LayoutChan, LayoutRPC};
 use libc;
 use msg::constellation_msg::ConstellationChan;
-use msg::constellation_msg::{PipelineId, SubpageId, WindowSizeData, WindowSizeType};
+use msg::constellation_msg::{PipelineId, SubpageId, WindowSizeData, WindowSizeType, ReferrerPolicy};
 use net_traits::image::base::{Image, ImageMetadata};
 use net_traits::image_cache_thread::{ImageCacheChan, ImageCacheThread};
 use net_traits::response::HttpsState;
@@ -325,6 +325,7 @@ no_jsmanaged_fields!(ElementSnapshot);
 no_jsmanaged_fields!(HttpsState);
 no_jsmanaged_fields!(SharedRt);
 no_jsmanaged_fields!(TouchpadPressurePhase);
+no_jsmanaged_fields!(ReferrerPolicy);
 
 impl JSTraceable for ConstellationChan<ScriptMsg> {
     #[inline]

components/script/dom/document.rs

@@ -90,7 +90,7 @@ use js::jsapi::{JSContext, JSObject, JSRuntime};
 use layout_interface::{LayoutChan, Msg, ReflowQueryType};
 use msg::constellation_msg::{ALT, CONTROL, SHIFT, SUPER};
 use msg::constellation_msg::{ConstellationChan, Key, KeyModifiers, KeyState};
-use msg::constellation_msg::{PipelineId, SubpageId};
+use msg::constellation_msg::{PipelineId, ReferrerPolicy, SubpageId};
 use net_traits::ControlMsg::{GetCookiesForUrl, SetCookiesForUrl};
 use net_traits::CookieSource::NonHTTP;
 use net_traits::response::HttpsState;
@@ -228,6 +228,8 @@ pub struct Document {
     touchpad_pressure_phase: Cell<TouchpadPressurePhase>,
     /// The document's origin.
     origin: Origin,
+    ///  https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-states
+    referrer_policy: Option<ReferrerPolicy>,
 }
 
 #[derive(JSTraceable, HeapSizeOf)]
@@ -1328,12 +1330,12 @@ impl Document {
 
     pub fn prepare_async_load(&self, load: LoadType) -> PendingAsyncLoad {
         let mut loader = self.loader.borrow_mut();
-        loader.prepare_async_load(load)
+        loader.prepare_async_load(load, self)
     }
 
     pub fn load_async(&self, load: LoadType, listener: AsyncResponseTarget) {
         let mut loader = self.loader.borrow_mut();
-        loader.load_async(load, listener)
+        loader.load_async(load, listener, self)
     }
 
     pub fn finish_load(&self, load: LoadType) {
@@ -1686,6 +1688,8 @@ impl Document {
             https_state: Cell::new(HttpsState::None),
             touchpad_pressure_phase: Cell::new(TouchpadPressurePhase::BeforeClick),
             origin: origin,
+            //TODO - setting this for now so no Referer header set
+            referrer_policy: Some(ReferrerPolicy::NoReferrer),
         }
     }
 
@@ -1814,6 +1818,11 @@ impl Document {
             snapshot.attrs = Some(attrs);
         }
     }
+
+    //TODO - for now, returns no-referrer for all until reading in the value
+    pub fn get_referrer_policy(&self) -> Option<ReferrerPolicy> {
+        return self.referrer_policy.clone();
+    }
 }
 
 

components/script/dom/htmlformelement.rs

@@ -283,7 +283,7 @@ impl HTMLFormElement {
         let _target = submitter.target();
         // TODO: Handle browsing contexts, partially loaded documents (step 16-17)
 
-        let mut load_data = LoadData::new(action_components);
+        let mut load_data = LoadData::new(action_components, doc.get_referrer_policy(), Some(doc.url().clone()));
 
         let parsed_data = match enctype {
             FormEncType::UrlEncoded => {

components/script/dom/window.rs

@@ -1199,8 +1199,10 @@ impl Window {
 
     /// Commence a new URL load which will either replace this window or scroll to a fragment.
     pub fn load_url(&self, url: Url) {
+        let doc = self.Document();
         self.main_thread_script_chan().send(
-            MainThreadScriptMsg::Navigate(self.id, LoadData::new(url))).unwrap();
+            MainThreadScriptMsg::Navigate(self.id,
+                LoadData::new(url, doc.get_referrer_policy(), Some(doc.url().clone())))).unwrap();
     }
 
     pub fn handle_fire_timer(&self, timer_id: TimerEventId) {

components/script/dom/xmlhttprequest.rs

@@ -576,10 +576,13 @@ impl XMLHttpRequestMethods for XMLHttpRequest {
         // Step 5
         let global = self.global();
         let pipeline_id = global.r().pipeline();
+        //TODO - set referrer_policy/referrer_url in load_data
         let mut load_data =
             LoadData::new(LoadContext::Browsing,
                           self.request_url.borrow().clone().unwrap(),
-                          Some(pipeline_id));
+                          Some(pipeline_id),
+                          None,
+                          None);
         if load_data.url.origin().ne(&global.r().get_url().origin()) {
             load_data.credentials_flag = self.WithCredentials();
         }

components/script/script_thread.rs

@@ -1887,6 +1887,8 @@ impl ScriptThread {
             cors: None,
             pipeline_id: Some(id),
             credentials_flag: true,
+            referrer_policy: load_data.referrer_policy,
+            referrer_url: load_data.referrer_url,
         }, LoadConsumer::Listener(response_target), None)).unwrap();
 
         self.incomplete_loads.borrow_mut().push(incomplete);

components/webdriver_server/lib.rs

@@ -322,7 +322,7 @@ impl Handler {
 
         let (sender, receiver) = ipc::channel().unwrap();
 
-        let load_data = LoadData::new(url);
+        let load_data = LoadData::new(url, None, None);
         let cmd_msg = WebDriverCommandMsg::LoadUrl(pipeline_id, load_data, sender.clone());
         self.constellation_chan.send(ConstellationMsg::WebDriverCommand(cmd_msg)).unwrap();