Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-30 03:53:37 +01:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #11413 - nox:content-blocker, r=nox+jdm+Ms2ger

Browse source 
Support content blocking

This is @jdm's rebased work from https://github.com/servo/servo/issues/9749#issuecomment-211115746 with a small addition with regard to cross-origin blocking. Mainly opening this for review and for further discussions.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/11413)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2016-05-30 06:39:36 -05:00
commit e63ded6d72
10 changed files with 183 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

2
components/net/Cargo.toml
View file

@ -11,6 +11,7 @@ path = "lib.rs"
[dependencies]
bitflags = "0.7"
brotli = {git = "https://github.com/ende76/brotli-rs"}
content-blocker = "0.2"
cookie = {version = "0.2.4", features = ["serialize-rustc"]}
device = {git = "https://github.com/servo/devices"}
devtools_traits = {path = "../devtools_traits"}
@ -18,6 +19,7 @@ flate2 = "0.2.0"
hyper = {version = "0.9", features = ["serde-serialization"]}
immeta = "0.3.1"
ipc-channel = {git = "https://github.com/servo/ipc-channel"}
lazy_static = "0.2"
log = "0.3.5"
matches = "0.1"
mime = "0.2.0"

31
components/net/content_blocker.rs Normal file
View file

@ -0,0 +1,31 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use content_blocker_parser::{RuleList, parse_list};
use std::str;
use std::sync::Arc;
use util::resource_files::read_resource_file;
lazy_static! {
pub static ref BLOCKED_CONTENT_RULES: Arc<Option<RuleList>> = Arc::new(create_rule_list());
}
fn create_rule_list() -> Option<RuleList> {
let contents = match read_resource_file("blocked-content.json") {
Ok(c) => c,
Err(_) => return None,
};
let str_contents = match str::from_utf8(&contents) {
Ok(c) => c,
Err(_) => return None,
};
let list = match parse_list(&str_contents) {
Ok(l) => l,
Err(_) => return None,
};
Some(list)
}

51
components/net/http_loader.rs
View file

@ -2,9 +2,10 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use brotli::Decompressor;
use connector::Connector;
use content_blocker_parser::{LoadType, Reaction, Request as CBRequest, ResourceType};
use content_blocker_parser::{RuleList, process_rules_for_request};
use cookie;
use cookie_storage::CookieStorage;
use devtools_traits::{ChromeToDevtoolsControlMsg, DevtoolsControlMsg, HttpRequest as DevtoolsHttpRequest};
@ -104,6 +105,7 @@ pub struct HttpState {
pub hsts_list: Arc<RwLock<HstsList>>,
pub cookie_jar: Arc<RwLock<CookieStorage>>,
pub auth_cache: Arc<RwLock<AuthCache>>,
pub blocked_content: Arc<Option<RuleList>>,
}
impl HttpState {
@ -112,6 +114,7 @@ impl HttpState {
hsts_list: Arc::new(RwLock::new(HstsList::new())),
cookie_jar: Arc::new(RwLock::new(CookieStorage::new())),
auth_cache: Arc::new(RwLock::new(AuthCache::new())),
blocked_content: Arc::new(None),
}
}
}
@ -327,6 +330,7 @@ pub enum LoadErrorType {
Cancelled,
Connection { reason: String },
ConnectionAborted { reason: String },
ContentBlocked,
// Preflight fetch inconsistent with main fetch
CorsPreflightFetchInconsistent,
Decoding { reason: String },
@ -349,6 +353,7 @@ impl Error for LoadErrorType {
LoadErrorType::Cancelled => "load cancelled",
LoadErrorType::Connection { ref reason } => reason,
LoadErrorType::ConnectionAborted { ref reason } => reason,
LoadErrorType::ContentBlocked => "content blocked",
LoadErrorType::CorsPreflightFetchInconsistent => "preflight fetch inconsistent with main fetch",
LoadErrorType::Decoding { ref reason } => reason,
LoadErrorType::InvalidRedirect { ref reason } => reason,
@ -590,7 +595,8 @@ pub fn modify_request_headers(headers: &mut Headers,
user_agent: &str,
cookie_jar: &Arc<RwLock<CookieStorage>>,
auth_cache: &Arc<RwLock<AuthCache>>,
load_data: &LoadData) {
load_data: &LoadData,
block_cookies: bool) {
// Ensure that the host header is set from the original url
let host = Host {
hostname: url.host_str().unwrap().to_owned(),
@ -620,7 +626,9 @@ pub fn modify_request_headers(headers: &mut Headers,
// https://fetch.spec.whatwg.org/#concept-http-network-or-cache-fetch step 11
if load_data.credentials_flag {
if !block_cookies {
set_request_cookies(url.clone(), headers, cookie_jar);
}
// https://fetch.spec.whatwg.org/#http-network-or-cache-fetch step 12
set_auth_header(headers, url, auth_cache);
@ -852,6 +860,29 @@ pub fn load<A, B>(load_data: &LoadData,
return Err(LoadError::new(doc_url, LoadErrorType::Cancelled));
}
let mut block_cookies = false;
if let Some(ref rules) = *http_state.blocked_content {
let same_origin =
load_data.referrer_url.as_ref()
.map(|url| url.origin() == doc_url.origin())
.unwrap_or(false);
let load_type = if same_origin { LoadType::FirstParty } else { LoadType::ThirdParty };
let actions = process_rules_for_request(rules, &CBRequest {
url: &doc_url,
resource_type: to_resource_type(&load_data.context),
load_type: load_type,
});
for action in actions {
match action {
Reaction::Block => {
return Err(LoadError::new(doc_url, LoadErrorType::ContentBlocked));
},
Reaction::BlockCookies => block_cookies = true,
Reaction::HideMatchingElements(_) => (),
}
}
}
info!("requesting {}", doc_url);
// Avoid automatically preserving request headers when redirects occur.
@ -870,7 +901,7 @@ pub fn load<A, B>(load_data: &LoadData,
modify_request_headers(&mut request_headers, &doc_url,
&user_agent, &http_state.cookie_jar,
&http_state.auth_cache, &load_data);
&http_state.auth_cache, &load_data, block_cookies);
//if there is a new auth header then set the request headers with it
if let Some(ref auth_header) = new_auth_header {
@ -1032,3 +1063,17 @@ fn is_cert_verify_error(error: &OpensslError) -> bool {
}
}
}
fn to_resource_type(context: &LoadContext) -> ResourceType {
match *context {
LoadContext::Browsing => ResourceType::Document,
LoadContext::Image => ResourceType::Image,
LoadContext::AudioVideo => ResourceType::Media,
LoadContext::Plugin => ResourceType::Raw,
LoadContext::Style => ResourceType::StyleSheet,
LoadContext::Script => ResourceType::Script,
LoadContext::Font => ResourceType::Font,
LoadContext::TextTrack => ResourceType::Media,
LoadContext::CacheManifest => ResourceType::Raw,
}
}

3
components/net/lib.rs
View file

@ -16,6 +16,7 @@
#[macro_use]
extern crate bitflags;
extern crate brotli;
extern crate content_blocker as content_blocker_parser;
extern crate cookie as cookie_rs;
extern crate device;
extern crate devtools_traits;
@ -23,6 +24,7 @@ extern crate flate2;
extern crate hyper;
extern crate immeta;
extern crate ipc_channel;
#[macro_use] extern crate lazy_static;
#[macro_use] extern crate log;
#[macro_use] #[no_link] extern crate matches;
#[macro_use]
@ -49,6 +51,7 @@ pub mod about_loader;
pub mod bluetooth_thread;
pub mod chrome_loader;
pub mod connector;
pub mod content_blocker;
pub mod cookie;
pub mod cookie_storage;
pub mod data_loader;

4
components/net/resource_thread.rs
View file

@ -6,6 +6,7 @@
use about_loader;
use chrome_loader;
use connector::{Connector, create_http_connector};
use content_blocker::BLOCKED_CONTENT_RULES;
use cookie;
use cookie_storage::CookieStorage;
use data_loader;
@ -453,7 +454,8 @@ impl CoreResourceManager {
let http_state = HttpState {
hsts_list: self.hsts_list.clone(),
cookie_jar: self.cookie_jar.clone(),
auth_cache: self.auth_cache.clone()
auth_cache: self.auth_cache.clone(),
blocked_content: BLOCKED_CONTENT_RULES.clone(),
};
http_loader::factory(self.user_agent.clone(),
http_state,

13
components/servo/Cargo.lock generated
View file

@ -363,6 +363,16 @@ dependencies = [
"webrender_traits 0.1.0 (git+https://github.com/servo/webrender_traits)",
]
[[package]]
name = "content-blocker"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "cookie"
version = "0.2.4"
@ -1357,6 +1367,7 @@ version = "0.0.1"
dependencies = [
"bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"brotli 0.3.23 (git+https://github.com/ende76/brotli-rs)",
"content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"device 0.0.1 (git+https://github.com/servo/devices)",
"devtools_traits 0.0.1",
@ -1364,6 +1375,7 @@ dependencies = [
"hyper 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
"immeta 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"ipc-channel 0.2.2 (git+https://github.com/servo/ipc-channel)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1402,6 +1414,7 @@ dependencies = [
name = "net_tests"
version = "0.0.1"
dependencies = [
"content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"devtools_traits 0.0.1",
"flate2 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)",

12
ports/cef/Cargo.lock generated
View file

@ -325,6 +325,16 @@ dependencies = [
"webrender_traits 0.1.0 (git+https://github.com/servo/webrender_traits)",
]
[[package]]
name = "content-blocker"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"regex 0.1.71 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "cookie"
version = "0.2.4"
@ -1262,6 +1272,7 @@ version = "0.0.1"
dependencies = [
"bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"brotli 0.3.23 (git+https://github.com/ende76/brotli-rs)",
"content-blocker 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"cookie 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"device 0.0.1 (git+https://github.com/servo/devices)",
"devtools_traits 0.0.1",
@ -1269,6 +1280,7 @@ dependencies = [
"hyper 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
"immeta 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"ipc-channel 0.2.2 (git+https://github.com/servo/ipc-channel)",
"lazy_static 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
"matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"mime 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",

17
tests/unit/net/Cargo.toml
View file

@ -9,17 +9,18 @@ path = "lib.rs"
doctest = false
[dependencies]
content-blocker = "0.2"
cookie = "0.2"
devtools_traits = {path = "../../../components/devtools_traits"}
flate2 = "0.2.0"
hyper = "0.9"
ipc-channel = {git = "https://github.com/servo/ipc-channel"}
msg = {path = "../../../components/msg"}
net = {path = "../../../components/net"}
net_traits = {path = "../../../components/net_traits"}
util = {path = "../../../components/util"}
msg = {path = "../../../components/msg"}
plugins = {path = "../../../components/plugins"}
profile_traits = {path = "../../../components/profile_traits"}
devtools_traits = {path = "../../../components/devtools_traits"}
ipc-channel = {git = "https://github.com/servo/ipc-channel"}
cookie = "0.2"
hyper = "0.9"
url = {version = "1.0.0", features = ["heap_size"]}
time = "0.1"
flate2 = "0.2.0"
unicase = "1.0"
url = {version = "1.0.0", features = ["heap_size"]}
util = {path = "../../../components/util"}

60
tests/unit/net/http_loader.rs
View file

@ -2,6 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
use content_blocker::parse_list;
use cookie_rs::Cookie as CookiePair;
use devtools_traits::HttpRequest as DevtoolsHttpRequest;
use devtools_traits::HttpResponse as DevtoolsHttpResponse;
@ -1858,3 +1859,62 @@ fn test_custom_response_from_worker() {
assert_eq!(metadata.status, Some(RawStatus(200, Cow::Borrowed("OK"))));
assert_eq!(body, String::from_utf8(expected_body).unwrap());
}
#[test]
fn test_content_blocked() {
struct Factory;
impl HttpRequestFactory for Factory {
type R = MockRequest;
fn create(&self, _url: Url, _method: Method, _: Headers) -> Result<MockRequest, LoadError> {
Ok(MockRequest::new(ResponseType::Text(<[_]>::to_vec("Yay!".as_bytes()))))
}
}
let blocked_url = Url::parse("http://mozilla.com").unwrap();
let url_without_cookies = Url::parse("http://mozilla2.com").unwrap();
let mut http_state = HttpState::new();
let blocked_content_list = "[{ \"trigger\": { \"url-filter\": \"https?://mozilla.com\" }, \
\"action\": { \"type\": \"block\" } },\
{ \"trigger\": { \"url-filter\": \"https?://mozilla2.com\" }, \
\"action\": { \"type\": \"block-cookies\" } }]";
http_state.blocked_content = Arc::new(parse_list(blocked_content_list).ok());
assert!(http_state.blocked_content.is_some());
{
let mut cookie_jar = http_state.cookie_jar.write().unwrap();
let cookie = Cookie::new_wrapped(
CookiePair::parse("mozillaIs=theBest;").unwrap(),
&url_without_cookies,
CookieSource::HTTP
).unwrap();
cookie_jar.push(cookie, CookieSource::HTTP);
}
let ui_provider = TestProvider::new();
let load_data = LoadData::new(LoadContext::Browsing, url_without_cookies, &HttpTest);
let response = load(
&load_data, &ui_provider, &http_state,
None, &AssertMustNotIncludeHeadersRequestFactory {
headers_not_expected: vec!["Cookie".to_owned()],
body: b"hi".to_vec(),
}, DEFAULT_USER_AGENT.to_owned(), &CancellationListener::new(None));
match response {
Ok(_) => {},
_ => panic!("request should have succeeded without cookies"),
}
let load_data = LoadData::new(LoadContext::Browsing, blocked_url, &HttpTest);
let response = load(
&load_data, &ui_provider, &http_state,
None, &Factory,
DEFAULT_USER_AGENT.to_owned(), &CancellationListener::new(None));
match response {
Err(LoadError { error: LoadErrorType::ContentBlocked, .. }) => {},
_ => panic!("request should have been blocked"),
}
}

1
tests/unit/net/lib.rs
View file

@ -5,6 +5,7 @@
#![feature(plugin)]
#![plugin(plugins)]
extern crate content_blocker;
extern crate cookie as cookie_rs;
extern crate devtools_traits;
extern crate flate2;