Auto merge of #11413 - nox:content-blocker, r=nox+jdm+Ms2ger

Support content blocking

This is @jdm's rebased work from https://github.com/servo/servo/issues/9749#issuecomment-211115746 with a small addition with regard to cross-origin blocking. Mainly opening this for review and for further discussions.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/11413)
<!-- Reviewable:end -->

components/net/Cargo.toml

@@ -11,6 +11,7 @@ path = "lib.rs"
 [dependencies]
 bitflags = "0.7"
 brotli = {git = "https://github.com/ende76/brotli-rs"}
+content-blocker = "0.2"
 cookie = {version = "0.2.4", features = ["serialize-rustc"]}
 device = {git = "https://github.com/servo/devices"}
 devtools_traits = {path = "../devtools_traits"}
@@ -18,6 +19,7 @@ flate2 = "0.2.0"
 hyper = {version = "0.9", features = ["serde-serialization"]}
 immeta = "0.3.1"
 ipc-channel = {git = "https://github.com/servo/ipc-channel"}
+lazy_static = "0.2"
 log = "0.3.5"
 matches = "0.1"
 mime = "0.2.0"

components/net/content_blocker.rs

@@ -0,0 +1,31 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+use content_blocker_parser::{RuleList, parse_list};
+use std::str;
+use std::sync::Arc;
+use util::resource_files::read_resource_file;
+
+lazy_static! {
+    pub static ref BLOCKED_CONTENT_RULES: Arc<Option<RuleList>> = Arc::new(create_rule_list());
+}
+
+fn create_rule_list() -> Option<RuleList> {
+    let contents = match read_resource_file("blocked-content.json") {
+        Ok(c) => c,
+        Err(_) => return None,
+    };
+
+    let str_contents = match str::from_utf8(&contents) {
+        Ok(c) => c,
+        Err(_) => return None,
+    };
+
+    let list = match parse_list(&str_contents) {
+        Ok(l) => l,
+        Err(_) => return None,
+    };
+
+    Some(list)
+}

components/net/http_loader.rs

@@ -2,9 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-
 use brotli::Decompressor;
 use connector::Connector;
+use content_blocker_parser::{LoadType, Reaction, Request as CBRequest, ResourceType};
+use content_blocker_parser::{RuleList, process_rules_for_request};
 use cookie;
 use cookie_storage::CookieStorage;
 use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, HttpRequest as DevtoolsHttpRequest};
@@ -104,6 +105,7 @@ pub struct HttpState {
     pub hsts_list: Arc<RwLock<HstsList>>,
     pub cookie_jar: Arc<RwLock<CookieStorage>>,
     pub auth_cache: Arc<RwLock<AuthCache>>,
+    pub blocked_content: Arc<Option<RuleList>>,
 }
 
 impl HttpState {
@@ -112,6 +114,7 @@ impl HttpState {
             hsts_list: Arc::new(RwLock::new(HstsList::new())),
             cookie_jar: Arc::new(RwLock::new(CookieStorage::new())),
             auth_cache: Arc::new(RwLock::new(AuthCache::new())),
+            blocked_content: Arc::new(None),
         }
     }
 }
@@ -327,6 +330,7 @@ pub enum LoadErrorType {
     Cancelled,
     Connection { reason: String },
     ConnectionAborted { reason: String },
+    ContentBlocked,
     // Preflight fetch inconsistent with main fetch
     CorsPreflightFetchInconsistent,
     Decoding { reason: String },
@@ -349,6 +353,7 @@ impl Error for LoadErrorType {
             LoadErrorType::Cancelled => "load cancelled",
             LoadErrorType::Connection { ref reason } => reason,
             LoadErrorType::ConnectionAborted { ref reason } => reason,
+            LoadErrorType::ContentBlocked => "content blocked",
             LoadErrorType::CorsPreflightFetchInconsistent => "preflight fetch inconsistent with main fetch",
             LoadErrorType::Decoding { ref reason } => reason,
             LoadErrorType::InvalidRedirect { ref reason } => reason,
@@ -590,7 +595,8 @@ pub fn modify_request_headers(headers: &mut Headers,
                               user_agent: &str,
                               cookie_jar: &Arc<RwLock<CookieStorage>>,
                               auth_cache: &Arc<RwLock<AuthCache>>,
-                              load_data: &LoadData) {
+                              load_data: &LoadData,
+                              block_cookies: bool) {
     // Ensure that the host header is set from the original url
     let host = Host {
         hostname: url.host_str().unwrap().to_owned(),
@@ -620,7 +626,9 @@ pub fn modify_request_headers(headers: &mut Headers,
 
     // https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
     if load_data.credentials_flag {
+        if !block_cookies {
             set_request_cookies(url.clone(), headers, cookie_jar);
+        }
 
         // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12
         set_auth_header(headers, url, auth_cache);
@@ -852,6 +860,29 @@ pub fn load<A, B>(load_data: &LoadData,
             return Err(LoadError::new(doc_url, LoadErrorType::Cancelled));
         }
 
+        let mut block_cookies = false;
+        if let Some(ref rules) = *http_state.blocked_content {
+            let same_origin =
+                load_data.referrer_url.as_ref()
+                         .map(|url| url.origin() == doc_url.origin())
+                         .unwrap_or(false);
+            let load_type = if same_origin { LoadType::FirstParty } else { LoadType::ThirdParty };
+            let actions = process_rules_for_request(rules, &CBRequest {
+                url: &doc_url,
+                resource_type: to_resource_type(&load_data.context),
+                load_type: load_type,
+            });
+            for action in actions {
+                match action {
+                    Reaction::Block => {
+                        return Err(LoadError::new(doc_url, LoadErrorType::ContentBlocked));
+                    },
+                    Reaction::BlockCookies => block_cookies = true,
+                    Reaction::HideMatchingElements(_) => (),
+                }
+            }
+        }
+
         info!("requesting {}", doc_url);
 
         // Avoid automatically preserving request headers when redirects occur.
@@ -870,7 +901,7 @@ pub fn load<A, B>(load_data: &LoadData,
 
         modify_request_headers(&mut request_headers, &doc_url,
                                &user_agent, &http_state.cookie_jar,
-                               &http_state.auth_cache, &load_data);
+                               &http_state.auth_cache, &load_data, block_cookies);
 
         //if there is a new auth header then set the request headers with it
         if let Some(ref auth_header) = new_auth_header {
@@ -1032,3 +1063,17 @@ fn is_cert_verify_error(error: &OpensslError) -> bool {
         }
     }
 }
+
+fn to_resource_type(context: &LoadContext) -> ResourceType {
+    match *context {
+        LoadContext::Browsing => ResourceType::Document,
+        LoadContext::Image => ResourceType::Image,
+        LoadContext::AudioVideo => ResourceType::Media,
+        LoadContext::Plugin => ResourceType::Raw,
+        LoadContext::Style => ResourceType::StyleSheet,
+        LoadContext::Script => ResourceType::Script,
+        LoadContext::Font => ResourceType::Font,
+        LoadContext::TextTrack => ResourceType::Media,
+        LoadContext::CacheManifest => ResourceType::Raw,
+    }
+}

components/net/lib.rs

@@ -16,6 +16,7 @@
 #[macro_use]
 extern crate bitflags;
 extern crate brotli;
+extern crate content_blocker as content_blocker_parser;
 extern crate cookie as cookie_rs;
 extern crate device;
 extern crate devtools_traits;
@@ -23,6 +24,7 @@ extern crate flate2;
 extern crate hyper;
 extern crate immeta;
 extern crate ipc_channel;
+#[macro_use] extern crate lazy_static;
 #[macro_use] extern crate log;
 #[macro_use] #[no_link] extern crate matches;
 #[macro_use]
@@ -49,6 +51,7 @@ pub mod about_loader;
 pub mod bluetooth_thread;
 pub mod chrome_loader;
 pub mod connector;
+pub mod content_blocker;
 pub mod cookie;
 pub mod cookie_storage;
 pub mod data_loader;

components/net/resource_thread.rs

@@ -6,6 +6,7 @@
 use about_loader;
 use chrome_loader;
 use connector::{Connector, create_http_connector};
+use content_blocker::BLOCKED_CONTENT_RULES;
 use cookie;
 use cookie_storage::CookieStorage;
 use data_loader;
@@ -453,7 +454,8 @@ impl CoreResourceManager {
                 let http_state = HttpState {
                     hsts_list: self.hsts_list.clone(),
                     cookie_jar: self.cookie_jar.clone(),
-                    auth_cache: self.auth_cache.clone()
+                    auth_cache: self.auth_cache.clone(),
+                    blocked_content: BLOCKED_CONTENT_RULES.clone(),
                 };
                 http_loader::factory(self.user_agent.clone(),
                                      http_state,

components/servo/Cargo.lock

@@ -363,6 +363,16 @@ dependencies = [
  "webrender_traits 0.1.0 (git+https://github.com/servo/webrender_traits)",
 ]
 
+[[package]]
+name = "content-blocker"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "cookie"
 version = "0.2.4"
@@ -1357,6 +1367,7 @@ version = "0.0.1"
 dependencies = [
  "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "brotli 0.3.23 (git+https://github.com/ende76/brotli-rs)",
+ "content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "device 0.0.1 (git+https://github.com/servo/devices)",
  "devtools_traits 0.0.1",
@@ -1364,6 +1375,7 @@ dependencies = [
  "hyper 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "immeta 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.2.2 (git+https://github.com/servo/ipc-channel)",
+ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1402,6 +1414,7 @@ dependencies = [
 name = "net_tests"
 version = "0.0.1"
 dependencies = [
+ "content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "devtools_traits 0.0.1",
  "flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",

ports/cef/Cargo.lock

@@ -325,6 +325,16 @@ dependencies = [
  "webrender_traits 0.1.0 (git+https://github.com/servo/webrender_traits)",
 ]
 
+[[package]]
+name = "content-blocker"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "cookie"
 version = "0.2.4"
@@ -1262,6 +1272,7 @@ version = "0.0.1"
 dependencies = [
  "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "brotli 0.3.23 (git+https://github.com/ende76/brotli-rs)",
+ "content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "device 0.0.1 (git+https://github.com/servo/devices)",
  "devtools_traits 0.0.1",
@@ -1269,6 +1280,7 @@ dependencies = [
  "hyper 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "immeta 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "ipc-channel 0.2.2 (git+https://github.com/servo/ipc-channel)",
+ "lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",

tests/unit/net/Cargo.toml

@@ -9,17 +9,18 @@ path = "lib.rs"
 doctest = false
 
 [dependencies]
+content-blocker = "0.2"
+cookie = "0.2"
+devtools_traits = {path = "../../../components/devtools_traits"}
+flate2 = "0.2.0"
+hyper = "0.9"
+ipc-channel = {git = "https://github.com/servo/ipc-channel"}
+msg = {path = "../../../components/msg"}
 net = {path = "../../../components/net"}
 net_traits = {path = "../../../components/net_traits"}
-util = {path = "../../../components/util"}
-msg = {path = "../../../components/msg"}
 plugins = {path = "../../../components/plugins"}
 profile_traits = {path = "../../../components/profile_traits"}
-devtools_traits = {path = "../../../components/devtools_traits"}
-ipc-channel = {git = "https://github.com/servo/ipc-channel"}
-cookie = "0.2"
-hyper = "0.9"
-url = {version = "1.0.0", features = ["heap_size"]}
 time = "0.1"
-flate2 = "0.2.0"
 unicase = "1.0"
+url = {version = "1.0.0", features = ["heap_size"]}
+util = {path = "../../../components/util"}

tests/unit/net/http_loader.rs

@@ -2,6 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+use content_blocker::parse_list;
 use cookie_rs::Cookie as CookiePair;
 use devtools_traits::HttpRequest as DevtoolsHttpRequest;
 use devtools_traits::HttpResponse as DevtoolsHttpResponse;
@@ -1858,3 +1859,62 @@ fn test_custom_response_from_worker() {
     assert_eq!(metadata.status, Some(RawStatus(200, Cow::Borrowed("OK"))));
     assert_eq!(body, String::from_utf8(expected_body).unwrap());
 }
+
+#[test]
+fn test_content_blocked() {
+    struct Factory;
+    impl HttpRequestFactory for Factory {
+        type R = MockRequest;
+
+        fn create(&self, _url: Url, _method: Method, _: Headers) -> Result<MockRequest, LoadError> {
+            Ok(MockRequest::new(ResponseType::Text(<[_]>::to_vec("Yay!".as_bytes()))))
+        }
+    }
+
+    let blocked_url = Url::parse("http://mozilla.com").unwrap();
+    let url_without_cookies = Url::parse("http://mozilla2.com").unwrap();
+    let mut http_state = HttpState::new();
+
+    let blocked_content_list = "[{ \"trigger\": { \"url-filter\": \"https?://mozilla.com\" }, \
+                                   \"action\": { \"type\": \"block\" } },\
+                                 { \"trigger\": { \"url-filter\": \"https?://mozilla2.com\" }, \
+                                   \"action\": { \"type\": \"block-cookies\" } }]";
+    http_state.blocked_content = Arc::new(parse_list(blocked_content_list).ok());
+    assert!(http_state.blocked_content.is_some());
+
+    {
+        let mut cookie_jar = http_state.cookie_jar.write().unwrap();
+        let cookie = Cookie::new_wrapped(
+            CookiePair::parse("mozillaIs=theBest;").unwrap(),
+            &url_without_cookies,
+            CookieSource::HTTP
+        ).unwrap();
+        cookie_jar.push(cookie, CookieSource::HTTP);
+    }
+
+    let ui_provider = TestProvider::new();
+
+    let load_data = LoadData::new(LoadContext::Browsing, url_without_cookies, &HttpTest);
+
+    let response = load(
+        &load_data, &ui_provider, &http_state,
+        None, &AssertMustNotIncludeHeadersRequestFactory {
+            headers_not_expected: vec!["Cookie".to_owned()],
+            body: b"hi".to_vec(),
+        }, DEFAULT_USER_AGENT.to_owned(), &CancellationListener::new(None));
+    match response {
+        Ok(_) => {},
+        _ => panic!("request should have succeeded without cookies"),
+    }
+
+    let load_data = LoadData::new(LoadContext::Browsing, blocked_url, &HttpTest);
+
+    let response = load(
+        &load_data, &ui_provider, &http_state,
+        None, &Factory,
+        DEFAULT_USER_AGENT.to_owned(), &CancellationListener::new(None));
+    match response {
+        Err(LoadError { error: LoadErrorType::ContentBlocked, .. }) => {},
+        _ => panic!("request should have been blocked"),
+    }
+}

tests/unit/net/lib.rs

@@ -5,6 +5,7 @@
 #![feature(plugin)]
 #![plugin(plugins)]
 
+extern crate content_blocker;
 extern crate cookie as cookie_rs;
 extern crate devtools_traits;
 extern crate flate2;