Auto merge of #11413 - nox:content-blocker, r=nox+jdm+Ms2ger

Support content blocking

This is @jdm's rebased work from https://github.com/servo/servo/issues/9749#issuecomment-211115746 with a small addition with regard to cross-origin blocking. Mainly opening this for review and for further discussions.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/11413)
<!-- Reviewable:end -->

components/net/Cargo.toml

@@ -11,6 +11,7 @@ path = "lib.rs"
 [dependencies]
 bitflags = "0.7"
 brotli = {git = "https://github.com/ende76/brotli-rs"}
+content-blocker = "0.2"
 cookie = {version = "0.2.4", features = ["serialize-rustc"]}
 device = {git = "https://github.com/servo/devices"}
 devtools_traits = {path = "../devtools_traits"}
@@ -18,6 +19,7 @@ flate2 = "0.2.0"
 hyper = {version = "0.9", features = ["serde-serialization"]}
 immeta = "0.3.1"
 ipc-channel = {git = "https://github.com/servo/ipc-channel"}
+lazy_static = "0.2"
 log = "0.3.5"
 matches = "0.1"
 mime = "0.2.0"

components/net/content_blocker.rs

@@ -0,0 +1,31 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use content_blocker_parser::{RuleList, parse_list};
+use std::str;
+use std::sync::Arc;
+use util::resource_files::read_resource_file;
+
+lazy_static! {
+    pub static ref BLOCKED_CONTENT_RULES: Arc<Option<RuleList>> = Arc::new(create_rule_list());
+}
+
+fn create_rule_list() -> Option<RuleList> {
+    let contents = match read_resource_file("blocked-content.json") {
+        Ok(c) => c,
+        Err(_) => return None,
+    };
+
+    let str_contents = match str::from_utf8(&contents) {
+        Ok(c) => c,
+        Err(_) => return None,
+    };
+
+    let list = match parse_list(&str_contents) {
+        Ok(l) => l,
+        Err(_) => return None,
+    };
+
+    Some(list)
+}

components/net/http_loader.rs

@@ -2,9 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-
 use brotli::Decompressor;
 use connector::Connector;
+use content_blocker_parser::{LoadType, Reaction, Request as CBRequest, ResourceType};
+use content_blocker_parser::{RuleList, process_rules_for_request};
 use cookie;
 use cookie_storage::CookieStorage;
 use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, HttpRequest as DevtoolsHttpRequest};
@@ -104,6 +105,7 @@ pub struct HttpState {
     pub hsts_list: Arc<RwLock<HstsList>>,
     pub cookie_jar: Arc<RwLock<CookieStorage>>,
     pub auth_cache: Arc<RwLock<AuthCache>>,
+    pub blocked_content: Arc<Option<RuleList>>,
 }
 
 impl HttpState {
@@ -112,6 +114,7 @@ impl HttpState {
             hsts_list: Arc::new(RwLock::new(HstsList::new())),
             cookie_jar: Arc::new(RwLock::new(CookieStorage::new())),
             auth_cache: Arc::new(RwLock::new(AuthCache::new())),
+            blocked_content: Arc::new(None),
         }
     }
 }
@@ -327,6 +330,7 @@ pub enum LoadErrorType {
     Cancelled,
     Connection { reason: String },
     ConnectionAborted { reason: String },
+    ContentBlocked,
     // Preflight fetch inconsistent with main fetch
     CorsPreflightFetchInconsistent,
     Decoding { reason: String },
@@ -349,6 +353,7 @@ impl Error for LoadErrorType {
             LoadErrorType::Cancelled => "load cancelled",
             LoadErrorType::Connection { ref reason } => reason,
             LoadErrorType::ConnectionAborted { ref reason } => reason,
+            LoadErrorType::ContentBlocked => "content blocked",
             LoadErrorType::CorsPreflightFetchInconsistent => "preflight fetch inconsistent with main fetch",
             LoadErrorType::Decoding { ref reason } => reason,
             LoadErrorType::InvalidRedirect { ref reason } => reason,
@@ -590,7 +595,8 @@ pub fn modify_request_headers(headers: &mut Headers,
                               user_agent: &str,
                               cookie_jar: &Arc<RwLock<CookieStorage>>,
                               auth_cache: &Arc<RwLock<AuthCache>>,
-                              load_data: &LoadData) {
+                              load_data: &LoadData,
+                              block_cookies: bool) {
     // Ensure that the host header is set from the original url
     let host = Host {
         hostname: url.host_str().unwrap().to_owned(),
@@ -620,7 +626,9 @@ pub fn modify_request_headers(headers: &mut Headers,
 
     // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
     if load_data.credentials_flag {
-        set_request_cookies(url.clone(), headers, cookie_jar);
+        if !block_cookies {
+            set_request_cookies(url.clone(), headers, cookie_jar);
+        }
 
         // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12
         set_auth_header(headers, url, auth_cache);
@@ -852,6 +860,29 @@ pub fn load<A, B>(load_data: &LoadData,
             return Err(LoadError::new(doc_url, LoadErrorType::Cancelled));
         }
 
+        let mut block_cookies = false;
+        if let Some(ref rules) = *http_state.blocked_content {
+            let same_origin =
+                load_data.referrer_url.as_ref()
+                         .map(|url| url.origin() == doc_url.origin())
+                         .unwrap_or(false);
+            let load_type = if same_origin { LoadType::FirstParty } else { LoadType::ThirdParty };
+            let actions = process_rules_for_request(rules, &CBRequest {
+                url: &doc_url,
+                resource_type: to_resource_type(&load_data.context),
+                load_type: load_type,
+            });
+            for action in actions {
+                match action {
+                    Reaction::Block => {
+                        return Err(LoadError::new(doc_url, LoadErrorType::ContentBlocked));
+                    },
+                    Reaction::BlockCookies => block_cookies = true,
+                    Reaction::HideMatchingElements(_) => (),
+                }
+            }
+        }
+
         info!("requesting {}", doc_url);
 
         // Avoid automatically preserving request headers when redirects occur.
@@ -870,7 +901,7 @@ pub fn load<A, B>(load_data: &LoadData,
 
         modify_request_headers(&mut request_headers, &doc_url,
                                &user_agent, &http_state.cookie_jar,
-                               &http_state.auth_cache, &load_data);
+                               &http_state.auth_cache, &load_data, block_cookies);
 
         //if there is a new auth header then set the request headers with it
         if let Some(ref auth_header) = new_auth_header {
@@ -1032,3 +1063,17 @@ fn is_cert_verify_error(error: &OpensslError) -> bool {
         }
     }
 }
+
+fn to_resource_type(context: &LoadContext) -> ResourceType {
+    match *context {
+        LoadContext::Browsing => ResourceType::Document,
+        LoadContext::Image => ResourceType::Image,
+        LoadContext::AudioVideo => ResourceType::Media,
+        LoadContext::Plugin => ResourceType::Raw,
+        LoadContext::Style => ResourceType::StyleSheet,
+        LoadContext::Script => ResourceType::Script,
+        LoadContext::Font => ResourceType::Font,
+        LoadContext::TextTrack => ResourceType::Media,
+        LoadContext::CacheManifest => ResourceType::Raw,
+    }
+}

components/net/lib.rs

@@ -16,6 +16,7 @@
 #[macro_use]
 extern crate bitflags;
 extern crate brotli;
+extern crate content_blocker as content_blocker_parser;
 extern crate cookie as cookie_rs;
 extern crate device;
 extern crate devtools_traits;
@@ -23,6 +24,7 @@ extern crate flate2;
 extern crate hyper;
 extern crate immeta;
 extern crate ipc_channel;
+#[macro_use] extern crate lazy_static;
 #[macro_use] extern crate log;
 #[macro_use] #[no_link] extern crate matches;
 #[macro_use]
@@ -49,6 +51,7 @@ pub mod about_loader;
 pub mod bluetooth_thread;
 pub mod chrome_loader;
 pub mod connector;
+pub mod content_blocker;
 pub mod cookie;
 pub mod cookie_storage;
 pub mod data_loader;

components/net/resource_thread.rs

@@ -6,6 +6,7 @@
 use about_loader;
 use chrome_loader;
 use connector::{Connector, create_http_connector};
+use content_blocker::BLOCKED_CONTENT_RULES;
 use cookie;
 use cookie_storage::CookieStorage;
 use data_loader;
@@ -453,7 +454,8 @@ impl CoreResourceManager {
                 let http_state = HttpState {
                     hsts_list: self.hsts_list.clone(),
                     cookie_jar: self.cookie_jar.clone(),
-                    auth_cache: self.auth_cache.clone()
+                    auth_cache: self.auth_cache.clone(),
+                    blocked_content: BLOCKED_CONTENT_RULES.clone(),
                 };
                 http_loader::factory(self.user_agent.clone(),
                                      http_state,

components/servo/Cargo.lock

@@ -363,6 +363,16 @@ dependencies = [
  "webrender_traits 0.1.0 (git+https://github.com/servo/webrender_traits)",
 ]
 
+[[package]]
+name = "content-blocker"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "cookie"
 version = "0.2.4"
@@ -1357,6 +1367,7 @@ version = "0.0.1"
 dependencies = [
  "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "brotli 0.3.23 (git+https://github.com/ende76/brotli-rs)",
+ "content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "device 0.0.1 (git+https://github.com/servo/devices)",
  "devtools_traits 0.0.1",
@@ -1364,6 +1375,7 @@ dependencies = [
  "hyper 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "immeta 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.2.2 (git+https://github.com/servo/ipc-channel)",
+ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1402,6 +1414,7 @@ dependencies = [
 name = "net_tests"
 version = "0.0.1"
 dependencies = [
+ "content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "devtools_traits 0.0.1",
  "flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",