Auto merge of #5005 - jdm:mime-sniffing, r=jdm

This rebases and integrates #4209, removing the sniffer task (turns out it wasn't a great idea), and adds a `--sniff-mime-types` command line flag to enable sniffing for file:// and http:// resources. Tested against a random picture file on my harddrive. The actual MIME sniffing implementation can be extracted into a separate library separately.

components/net/about_loader.rs

@@ -4,7 +4,8 @@
 
 use net_traits::{LoadData, Metadata};
 use net_traits::ProgressMsg::Done;
-use resource_task::{TargetedLoadResponse, start_sending, ResponseSenders};
+use mime_classifier::MIMEClassifier;
+use resource_task::start_sending;
 use file_loader;
 
 use url::Url;
@@ -13,16 +14,13 @@ use util::resource_files::resources_dir_path;
 
 use std::borrow::IntoCow;
 use std::fs::PathExt;
-use std::sync::mpsc::Sender;
+use std::sync::Arc;
 
-pub fn factory(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>) {
+pub fn factory(mut load_data: LoadData, classifier: Arc<MIMEClassifier>) {
-    let senders = ResponseSenders {
-        immediate_consumer: start_chan.clone(),
-        eventual_consumer: load_data.consumer.clone(),
-    };
     match load_data.url.non_relative_scheme_data().unwrap() {
         "blank" => {
-            let chan = start_sending(senders, Metadata {
+            let start_chan = load_data.consumer;
+            let chan = start_sending(start_chan, Metadata {
                 final_url: load_data.url,
                 content_type: Some(("text".to_string(), "html".to_string())),
                 charset: Some("utf-8".to_string()),
@@ -40,10 +38,11 @@ pub fn factory(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>
             load_data.url = Url::from_file_path(&*path).unwrap();
         }
         _ => {
-            start_sending(senders, Metadata::default(load_data.url))
+            let start_chan = load_data.consumer;
+            start_sending(start_chan, Metadata::default(load_data.url))
                 .send(Done(Err("Unknown about: URL.".to_string()))).unwrap();
             return
         }
     };
-    file_loader::factory(load_data, start_chan)
+    file_loader::factory(load_data, classifier)
 }

components/net/data_loader.rs

@@ -4,34 +4,30 @@
 
 use net_traits::{LoadData, Metadata};
 use net_traits::ProgressMsg::{Payload, Done};
-use resource_task::{TargetedLoadResponse, start_sending, ResponseSenders};
+use mime_classifier::MIMEClassifier;
+use resource_task::start_sending;
 
 use rustc_serialize::base64::FromBase64;
 
 use hyper::mime::Mime;
+use std::sync::Arc;
 use url::{percent_decode, SchemeData};
 
-use std::sync::mpsc::Sender;
+pub fn factory(load_data: LoadData, _classifier: Arc<MIMEClassifier>) {
-
-pub fn factory(load_data: LoadData, start_chan: Sender<TargetedLoadResponse>) {
     // NB: we don't spawn a new task.
     // Hypothesis: data URLs are too small for parallel base64 etc. to be worth it.
     // Should be tested at some point.
     // Left in separate function to allow easy moving to a task, if desired.
-    load(load_data, start_chan)
+    load(load_data)
 }
 
-fn load(load_data: LoadData, start_chan: Sender<TargetedLoadResponse>) {
+fn load(load_data: LoadData) {
+    let start_chan = load_data.consumer;
     let url = load_data.url;
     assert!(&*url.scheme == "data");
 
     let mut metadata = Metadata::default(url.clone());
 
-    let senders = ResponseSenders {
-        immediate_consumer: start_chan,
-        eventual_consumer: load_data.consumer,
-    };
-
     // Split out content type and data.
     let mut scheme_data = match url.scheme_data {
         SchemeData::NonRelative(scheme_data) => scheme_data,
@@ -46,7 +42,7 @@ fn load(load_data: LoadData, start_chan: Sender<TargetedLoadResponse>) {
     }
     let parts: Vec<&str> = scheme_data.splitn(1, ',').collect();
     if parts.len() != 2 {
-        start_sending(senders, metadata).send(Done(Err("invalid data uri".to_string()))).unwrap();
+        start_sending(start_chan, metadata).send(Done(Err("invalid data uri".to_string()))).unwrap();
         return;
     }
 
@@ -64,7 +60,7 @@ fn load(load_data: LoadData, start_chan: Sender<TargetedLoadResponse>) {
     let content_type: Option<Mime> = ct_str.parse().ok();
     metadata.set_content_type(content_type.as_ref());
 
-    let progress_chan = start_sending(senders, metadata);
+    let progress_chan = start_sending(start_chan, metadata);
     let bytes = percent_decode(parts[1].as_bytes());
 
     if is_base64 {
@@ -93,11 +89,9 @@ fn assert_parse(url:          &'static str,
                 data:         Option<Vec<u8>>) {
     use std::sync::mpsc::channel;
     use url::Url;
-    use sniffer_task;
 
     let (start_chan, start_port) = channel();
-    let sniffer_task = sniffer_task::new_sniffer_task();
+    load(LoadData::new(Url::parse(url).unwrap(), start_chan));
-    load(LoadData::new(Url::parse(url).unwrap(), start_chan), sniffer_task);
 
     let response = start_port.recv().unwrap();
     assert_eq!(&response.metadata.content_type, &content_type);

components/net/file_loader.rs

@@ -4,55 +4,78 @@
 
 use net_traits::{LoadData, Metadata, ProgressMsg};
 use net_traits::ProgressMsg::{Payload, Done};
-use resource_task::{start_sending, TargetedLoadResponse, ResponseSenders};
+use mime_classifier::MIMEClassifier;
+use resource_task::{start_sending, start_sending_sniffed};
 
 use std::borrow::ToOwned;
 use std::io;
 use std::fs::File;
 use std::path::PathBuf;
+use std::sync::Arc;
 use std::sync::mpsc::Sender;
 use util::task::spawn_named;
 
 static READ_SIZE: uint = 8192;
 
+enum ReadStatus {
+    Partial(Vec<u8>),
+    EOF,
+}
+
+fn read_block(reader: &mut io::Read) -> Result<ReadStatus, String> {
+    let mut buf = vec![0; READ_SIZE];
+    match reader.read(buf.as_mut_slice()) {
+        Ok(0) => return Ok(ReadStatus::EOF),
+        Ok(n) => {
+            buf.truncate(n);
+            Ok(ReadStatus::Partial(buf))
+        }
+        Err(e) => Err(e.description().to_string()),
+    }
+}
+
 fn read_all(reader: &mut io::Read, progress_chan: &Sender<ProgressMsg>)
             -> Result<(), String> {
     loop {
-        let mut buf = vec![0; READ_SIZE];
+        match try!(read_block(reader)) {
-        match reader.read(buf.as_mut_slice()) {
+            ReadStatus::Partial(buf) => progress_chan.send(Payload(buf)).unwrap(),
-            Ok(0) => return Ok(()),
+            ReadStatus::EOF => return Ok(()),
-            Ok(n) => {
-                buf.truncate(n);
-                progress_chan.send(Payload(buf)).unwrap();
-            },
-            Err(e) => return Err(e.description().to_string()),
         }
     }
 }
 
-pub fn factory(load_data: LoadData, start_chan: Sender<TargetedLoadResponse>) {
+pub fn factory(load_data: LoadData, classifier: Arc<MIMEClassifier>) {
     let url = load_data.url;
+    let start_chan = load_data.consumer;
     assert!(&*url.scheme == "file");
-    let senders = ResponseSenders {
-        immediate_consumer: start_chan,
-        eventual_consumer: load_data.consumer,
-    };
-    let progress_chan = start_sending(senders, Metadata::default(url.clone()));
     spawn_named("file_loader".to_owned(), move || {
+        let metadata = Metadata::default(url.clone());
         let file_path: Result<PathBuf, ()> = url.to_file_path();
         match file_path {
             Ok(file_path) => {
                 match File::open(&file_path) {
                     Ok(ref mut reader) => {
-                        let res = read_all(reader, &progress_chan);
+                        let res = read_block(reader);
+                        let (res, progress_chan) = match res {
+                            Ok(ReadStatus::Partial(buf)) => {
+                                let progress_chan = start_sending_sniffed(start_chan, metadata,
+                                                                          classifier, &buf);
+                                progress_chan.send(Payload(buf)).unwrap();
+                                (read_all(reader, &progress_chan), progress_chan)
+                            }
+                            Ok(ReadStatus::EOF) | Err(_) =>
+                                (res.map(|_| ()), start_sending(start_chan, metadata)),
+                        };
                         progress_chan.send(Done(res)).unwrap();
                     }
                     Err(e) => {
+                        let progress_chan = start_sending(start_chan, metadata);
                         progress_chan.send(Done(Err(e.description().to_string()))).unwrap();
                     }
                 }
             }
             Err(_) => {
+                let progress_chan = start_sending(start_chan, metadata);
                 progress_chan.send(Done(Err(url.to_string()))).unwrap();
             }
         }

components/net/http_loader.rs

@@ -2,10 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use net_traits::{ControlMsg, CookieSource, LoadData, Metadata};
+use net_traits::{ControlMsg, CookieSource, LoadData, LoadResponse, Metadata};
-use net_traits::ProgressMsg;
 use net_traits::ProgressMsg::{Payload, Done};
-use resource_task::{TargetedLoadResponse, start_sending_opt, ResponseSenders};
+use mime_classifier::MIMEClassifier;
+use resource_task::{start_sending_opt, start_sending_sniffed_opt};
 
 use log;
 use std::collections::HashSet;
@@ -21,6 +21,7 @@ use hyper::status::{StatusCode, StatusClass};
 use std::error::Error;
 use openssl::ssl::{SslContext, SslVerifyMode};
 use std::io::{self, Read, Write};
+use std::sync::Arc;
 use std::sync::mpsc::{Sender, channel};
 use std::thunk::Invoke;
 use util::task::spawn_named;
@@ -31,36 +32,50 @@ use url::{Url, UrlParser};
 use std::borrow::ToOwned;
 
 pub fn factory(cookies_chan: Sender<ControlMsg>)
-               -> Box<Invoke<(LoadData, Sender<TargetedLoadResponse>)> + Send> {
+               -> Box<Invoke<(LoadData, Arc<MIMEClassifier>)> + Send> {
-    box move |(load_data, start_chan)| {
+    box move |(load_data, classifier)| {
-        spawn_named("http_loader".to_owned(), move || load(load_data, start_chan, cookies_chan))
+        spawn_named("http_loader".to_owned(), move || load(load_data, classifier, cookies_chan))
     }
 }
 
-fn send_error(url: Url, err: String, senders: ResponseSenders) {
+fn send_error(url: Url, err: String, start_chan: Sender<LoadResponse>) {
     let mut metadata: Metadata = Metadata::default(url);
     metadata.status = None;
 
-    match start_sending_opt(senders, metadata) {
+    match start_sending_opt(start_chan, metadata) {
         Ok(p) => p.send(Done(Err(err))).unwrap(),
         _ => {}
     };
 }
 
-fn load(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>, cookies_chan: Sender<ControlMsg>) {
+enum ReadResult {
+    Payload(Vec<u8>),
+    EOF,
+}
+
+fn read_block<R: Read>(reader: &mut R) -> Result<ReadResult, ()> {
+    let mut buf = vec![0; 1024];
+
+    match reader.read(buf.as_mut_slice()) {
+        Ok(len) if len > 0 => {
+            unsafe { buf.set_len(len); }
+            Ok(ReadResult::Payload(buf))
+        }
+        Ok(_) => Ok(ReadResult::EOF),
+        Err(_) => Err(()),
+    }
+}
+
+fn load(mut load_data: LoadData, classifier: Arc<MIMEClassifier>, cookies_chan: Sender<ControlMsg>) {
     // FIXME: At the time of writing this FIXME, servo didn't have any central
     //        location for configuration. If you're reading this and such a
     //        repository DOES exist, please update this constant to use it.
     let max_redirects = 50u;
     let mut iters = 0u;
+    let start_chan = load_data.consumer;
     let mut url = load_data.url.clone();
     let mut redirected_to = HashSet::new();
 
-    let senders = ResponseSenders {
-        immediate_consumer: start_chan,
-        eventual_consumer: load_data.consumer
-    };
-
     // If the URL is a view-source scheme then the scheme data contains the
     // real URL that should be used for which the source is to be viewed.
     // Change our existing URL to that and keep note that we are viewing
@@ -73,7 +88,7 @@ fn load(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>, cooki
             "http" | "https" => {}
             _ => {
                 let s = format!("The {} scheme with view-source is not supported", url.scheme);
-                send_error(url, s, senders);
+                send_error(url, s, start_chan);
                 return;
             }
         };
@@ -84,7 +99,7 @@ fn load(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>, cooki
         iters = iters + 1;
 
         if iters > max_redirects {
-            send_error(url, "too many redirects".to_string(), senders);
+            send_error(url, "too many redirects".to_string(), start_chan);
             return;
         }
 
@@ -92,7 +107,7 @@ fn load(mut load_data: LoadData, start_chan: Sender<TargetedLoadResponse>, cooki
             "http" | "https" => {}
             _ => {
                 let s = format!("{} request, but we don't support that scheme", url.scheme);
-                send_error(url, s, senders);
+                send_error(url, s, start_chan);
                 return;
             }
         }
@@ -125,13 +140,13 @@ reason: \"certificate verify failed\" }]";
             ) => {
                 let mut image = resources_dir_path();
                 image.push("badcert.html");
-                let load_data = LoadData::new(Url::from_file_path(&*image).unwrap(), senders.eventual_consumer);
+                let load_data = LoadData::new(Url::from_file_path(&*image).unwrap(), start_chan);
-                file_loader::factory(load_data, senders.immediate_consumer);
+                file_loader::factory(load_data, classifier);
                 return;
             },
             Err(e) => {
                 println!("{:?}", e);
-                send_error(url, e.description().to_string(), senders);
+                send_error(url, e.description().to_string(), start_chan);
                 return;
             }
         };
@@ -179,13 +194,13 @@ reason: \"certificate verify failed\" }]";
                 let mut writer = match req.start() {
                     Ok(w) => w,
                     Err(e) => {
-                        send_error(url, e.description().to_string(), senders);
+                        send_error(url, e.description().to_string(), start_chan);
                         return;
                     }
                 };
                 match writer.write_all(&*data) {
                     Err(e) => {
-                        send_error(url, e.description().to_string(), senders);
+                        send_error(url, e.description().to_string(), start_chan);
                         return;
                     }
                     _ => {}
@@ -200,7 +215,7 @@ reason: \"certificate verify failed\" }]";
                 match req.start() {
                     Ok(w) => w,
                     Err(e) => {
-                        send_error(url, e.description().to_string(), senders);
+                        send_error(url, e.description().to_string(), start_chan);
                         return;
                     }
                 }
@@ -209,7 +224,7 @@ reason: \"certificate verify failed\" }]";
         let mut response = match writer.send() {
             Ok(r) => r,
             Err(e) => {
-                send_error(url, e.description().to_string(), senders);
+                send_error(url, e.description().to_string(), start_chan);
                 return;
             }
         };
@@ -240,7 +255,7 @@ reason: \"certificate verify failed\" }]";
                         Some(ref c) => {
                             if c.preflight {
                                 // The preflight lied
-                                send_error(url, "Preflight fetch inconsistent with main fetch".to_string(), senders);
+                                send_error(url, "Preflight fetch inconsistent with main fetch".to_string(), start_chan);
                                 return;
                             } else {
                                 // XXXManishearth There are some CORS-related steps here,
@@ -252,7 +267,7 @@ reason: \"certificate verify failed\" }]";
                     let new_url = match UrlParser::new().base_url(&url).parse(&new_url) {
                         Ok(u) => u,
                         Err(e) => {
-                            send_error(url, e.to_string(), senders);
+                            send_error(url, e.to_string(), start_chan);
                             return;
                         }
                     };
@@ -268,7 +283,7 @@ reason: \"certificate verify failed\" }]";
                     }
 
                     if redirected_to.contains(&url) {
-                        send_error(url, "redirect loop".to_string(), senders);
+                        send_error(url, "redirect loop".to_string(), start_chan);
                         return;
                     }
 
@@ -291,11 +306,6 @@ reason: \"certificate verify failed\" }]";
         metadata.headers = Some(adjusted_headers);
         metadata.status = Some(response.status_raw().clone());
 
-        let progress_chan = match start_sending_opt(senders, metadata) {
-            Ok(p) => p,
-            _ => return
-        };
-
         let mut encoding_str: Option<String> = None;
         //FIXME: Implement Content-Encoding Header https://github.com/hyperium/hyper/issues/391
         if let Some(encodings) = response.headers.get_raw("content-encoding") {
@@ -313,14 +323,14 @@ reason: \"certificate verify failed\" }]";
             Some(encoding) => {
                 if encoding == "gzip" {
                     let mut response_decoding = GzDecoder::new(response).unwrap();
-                    send_data(&mut response_decoding, progress_chan);
+                    send_data(&mut response_decoding, start_chan, metadata, classifier);
                 } else if encoding == "deflate" {
                     let mut response_decoding = DeflateDecoder::new(response);
-                    send_data(&mut response_decoding, progress_chan);
+                    send_data(&mut response_decoding, start_chan, metadata, classifier);
                 }
             },
             None => {
-                send_data(&mut response, progress_chan);
+                send_data(&mut response, start_chan, metadata, classifier);
             }
         }
 
@@ -329,25 +339,35 @@ reason: \"certificate verify failed\" }]";
     }
 }
 
-fn send_data<R: Read>(reader: &mut R, progress_chan: Sender<ProgressMsg>) {
+fn send_data<R: Read>(reader: &mut R,
-    loop {
+                      start_chan: Sender<LoadResponse>,
-        let mut buf = Vec::with_capacity(1024);
+                      metadata: Metadata,
+                      classifier: Arc<MIMEClassifier>) {
+    let (progress_chan, mut chunk) = {
+        let buf = match read_block(reader) {
+            Ok(ReadResult::Payload(buf)) => buf,
+            _ => vec!(),
+        };
+        let p = match start_sending_sniffed_opt(start_chan, metadata, classifier, &buf) {
+            Ok(p) => p,
+            _ => return
+        };
+        (p, buf)
+    };
 
-        unsafe { buf.set_len(1024); }
+    loop {
-        match reader.read(buf.as_mut_slice()) {
+        if progress_chan.send(Payload(chunk)).is_err() {
-            Ok(len) if len > 0 => {
-                unsafe { buf.set_len(len); }
-                if progress_chan.send(Payload(buf)).is_err() {
             // The send errors when the receiver is out of scope,
             // which will happen if the fetch has timed out (or has been aborted)
             // so we don't need to continue with the loading of the file here.
             return;
         }
+
+        chunk = match read_block(reader) {
+            Ok(ReadResult::Payload(buf)) => buf,
+            Ok(ReadResult::EOF) | Err(_) => break,
+        };
     }
-            Ok(_) | Err(_) => {
+
     let _ = progress_chan.send(Done(Ok(())));
-                break;
-            }
-        }
-    }
 }

components/net/image_cache_task.rs

@@ -428,11 +428,10 @@ mod tests {
     use net_traits::image_cache_task::ImageResponseMsg::*;
     use net_traits::image_cache_task::Msg::*;
 
-    use resource_task::{start_sending, ResponseSenders};
+    use resource_task::start_sending;
     use net_traits::{ControlMsg, Metadata, ProgressMsg, ResourceTask};
     use net_traits::image_cache_task::{ImageCacheTask, ImageResponseMsg, Msg};
     use net_traits::ProgressMsg::{Payload, Done};
-    use sniffer_task;
     use profile::time;
     use std::sync::mpsc::{Sender, channel, Receiver};
     use url::Url;
@@ -534,12 +533,7 @@ mod tests {
             loop {
                 match port.recv().unwrap() {
                     ControlMsg::Load(response) => {
-                        let sniffer_task = sniffer_task::new_sniffer_task();
+                        let chan = start_sending(response.consumer, Metadata::default(
-                        let senders = ResponseSenders {
-                            immediate_consumer: sniffer_task,
-                            eventual_consumer: response.consumer.clone(),
-                        };
-                        let chan = start_sending(senders, Metadata::default(
                             Url::parse("file:///fake").unwrap()));
                         on_load.invoke(chan);
                     }
@@ -709,12 +703,7 @@ mod tests {
             loop {
                 match port.recv().unwrap() {
                     ControlMsg::Load(response) => {
-                        let sniffer_task = sniffer_task::new_sniffer_task();
+                        let chan = start_sending(response.consumer, Metadata::default(
-                        let senders = ResponseSenders {
-                            immediate_consumer: sniffer_task,
-                            eventual_consumer: response.consumer.clone(),
-                        };
-                        let chan = start_sending(senders, Metadata::default(
                             Url::parse("file:///fake").unwrap()));
                         chan.send(Payload(test_image_bin()));
                         chan.send(Done(Ok(())));
@@ -763,12 +752,7 @@ mod tests {
             loop {
                 match port.recv().unwrap() {
                     ControlMsg::Load(response) => {
-                        let sniffer_task = sniffer_task::new_sniffer_task();
+                        let chan = start_sending(response.consumer, Metadata::default(
-                        let senders = ResponseSenders {
-                            immediate_consumer: sniffer_task,
-                            eventual_consumer: response.consumer.clone(),
-                        };
-                        let chan = start_sending(senders, Metadata::default(
                             Url::parse("file:///fake").unwrap()));
                         chan.send(Payload(test_image_bin()));
                         chan.send(Done(Err("".to_string())));

components/net/lib.rs

@@ -46,7 +46,7 @@ pub mod image_cache_task;
 pub mod pub_domains;
 pub mod resource_task;
 pub mod storage_task;
-mod sniffer_task;
+mod mime_classifier;
 
 /// An implementation of the [Fetch spec](http://fetch.spec.whatwg.org/)
 pub mod fetch {

components/net/resource_task.rs

@@ -8,14 +8,14 @@ use about_loader;
 use data_loader;
 use file_loader;
 use http_loader;
-use sniffer_task;
-use sniffer_task::SnifferTask;
 use cookie_storage::CookieStorage;
 use cookie;
+use mime_classifier::MIMEClassifier;
 
 use net_traits::{ControlMsg, LoadData, LoadResponse};
 use net_traits::{Metadata, ProgressMsg, ResourceTask};
 use net_traits::ProgressMsg::Done;
+use util::opts;
 use util::task::spawn_named;
 
 use hyper::header::UserAgent;
@@ -29,6 +29,7 @@ use std::collections::HashMap;
 use std::env;
 use std::fs::File;
 use std::io::{BufReader, Read};
+use std::sync::Arc;
 use std::sync::mpsc::{channel, Receiver, Sender};
 use std::thunk::Invoke;
 
@@ -58,32 +59,41 @@ pub fn global_init() {
     }
 }
 
-/// A LoadResponse directed at a particular consumer
+/// For use by loaders in responding to a Load message.
-pub struct TargetedLoadResponse {
+pub fn start_sending(start_chan: Sender<LoadResponse>, metadata: Metadata) -> Sender<ProgressMsg> {
-    pub load_response: LoadResponse,
+    start_sending_opt(start_chan, metadata).ok().unwrap()
-    pub consumer: Sender<LoadResponse>,
 }
 
-// Data structure containing ports
+/// For use by loaders in responding to a Load message that allows content sniffing.
-pub struct ResponseSenders {
+pub fn start_sending_sniffed(start_chan: Sender<LoadResponse>, metadata: Metadata,
-    pub immediate_consumer: Sender<TargetedLoadResponse>,
+                             classifier: Arc<MIMEClassifier>, partial_body: &Vec<u8>)
-    pub eventual_consumer: Sender<LoadResponse>,
+                             -> Sender<ProgressMsg> {
+    start_sending_sniffed_opt(start_chan, metadata, classifier, partial_body).ok().unwrap()
+}
+
+/// For use by loaders in responding to a Load message that allows content sniffing.
+pub fn start_sending_sniffed_opt(start_chan: Sender<LoadResponse>, mut metadata: Metadata,
+                                 classifier: Arc<MIMEClassifier>, partial_body: &Vec<u8>)
+                                 -> Result<Sender<ProgressMsg>, ()> {
+    if opts::get().sniff_mime_types {
+        // TODO: should be calculated in the resource loader, from pull requeset #4094
+        let nosniff = false;
+        let check_for_apache_bug = false;
+
+        metadata.content_type = classifier.classify(nosniff, check_for_apache_bug,
+                                                    &metadata.content_type, &partial_body);
+
+    }
+
+    start_sending_opt(start_chan, metadata)
 }
 
 /// For use by loaders in responding to a Load message.
-pub fn start_sending(senders: ResponseSenders, metadata: Metadata) -> Sender<ProgressMsg> {
+pub fn start_sending_opt(start_chan: Sender<LoadResponse>, metadata: Metadata) -> Result<Sender<ProgressMsg>, ()> {
-    start_sending_opt(senders, metadata).ok().unwrap()
-}
-
-/// For use by loaders in responding to a Load message.
-pub fn start_sending_opt(senders: ResponseSenders, metadata: Metadata) -> Result<Sender<ProgressMsg>, ()> {
     let (progress_chan, progress_port) = channel();
-    let result = senders.immediate_consumer.send(TargetedLoadResponse {
+    let result = start_chan.send(LoadResponse {
-        load_response: LoadResponse {
         metadata:      metadata,
         progress_port: progress_port,
-        },
-        consumer: senders.eventual_consumer
     });
     match result {
         Ok(_) => Ok(progress_chan),
@@ -94,10 +104,9 @@ pub fn start_sending_opt(senders: ResponseSenders, metadata: Metadata) -> Result
 /// Create a ResourceTask
 pub fn new_resource_task(user_agent: Option<String>) -> ResourceTask {
     let (setup_chan, setup_port) = channel();
-    let sniffer_task = sniffer_task::new_sniffer_task();
     let setup_chan_clone = setup_chan.clone();
     spawn_named("ResourceManager".to_owned(), move || {
-        ResourceManager::new(setup_port, user_agent, sniffer_task, setup_chan_clone).start();
+        ResourceManager::new(setup_port, user_agent, setup_chan_clone).start();
     });
     setup_chan
 }
@@ -139,20 +148,20 @@ pub fn replace_hosts(mut load_data: LoadData, host_table: *mut HashMap<String, S
 struct ResourceManager {
     from_client: Receiver<ControlMsg>,
     user_agent: Option<String>,
-    sniffer_task: SnifferTask,
     cookie_storage: CookieStorage,
     resource_task: Sender<ControlMsg>,
+    mime_classifier: Arc<MIMEClassifier>,
 }
 
 impl ResourceManager {
-    fn new(from_client: Receiver<ControlMsg>, user_agent: Option<String>, sniffer_task: SnifferTask,
+    fn new(from_client: Receiver<ControlMsg>, user_agent: Option<String>,
            resource_task: Sender<ControlMsg>) -> ResourceManager {
         ResourceManager {
             from_client: from_client,
             user_agent: user_agent,
-            sniffer_task: sniffer_task,
             cookie_storage: CookieStorage::new(),
             resource_task: resource_task,
+            mime_classifier: Arc::new(MIMEClassifier::new()),
         }
     }
 }
@@ -193,15 +202,11 @@ impl ResourceManager {
         }
 
         self.user_agent.as_ref().map(|ua| load_data.headers.set(UserAgent(ua.clone())));
-        let senders = ResponseSenders {
-            immediate_consumer: self.sniffer_task.clone(),
-            eventual_consumer: load_data.consumer.clone(),
-        };
 
-        fn from_factory(factory: fn(LoadData, Sender<TargetedLoadResponse>))
+        fn from_factory(factory: fn(LoadData, Arc<MIMEClassifier>))
-                        -> Box<Invoke<(LoadData, Sender<TargetedLoadResponse>)> + Send> {
+                        -> Box<Invoke<(LoadData, Arc<MIMEClassifier>)> + Send> {
-            box move |(load_data, start_chan)| {
+            box move |(load_data, classifier)| {
-                factory(load_data, start_chan)
+                factory(load_data, classifier)
             }
         }
 
@@ -212,14 +217,14 @@ impl ResourceManager {
             "about" => from_factory(about_loader::factory),
             _ => {
                 debug!("resource_task: no loader for scheme {}", load_data.url.scheme);
-                start_sending(senders, Metadata::default(load_data.url))
+                start_sending(load_data.consumer, Metadata::default(load_data.url))
                     .send(ProgressMsg::Done(Err("no loader for scheme".to_string()))).unwrap();
                 return
             }
         };
         debug!("resource_task: loading url: {}", load_data.url.serialize());
 
-        loader.invoke((load_data, self.sniffer_task.clone()));
+        loader.invoke((load_data, self.mime_classifier.clone()));
     }
 }
 

components/net/sniffer_task.rs

@@ -1,44 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-//! A task that sniffs data
-use std::sync::mpsc::{channel, Receiver, Sender};
-use std::thread::Builder;
-use resource_task::{TargetedLoadResponse};
-
-pub type SnifferTask = Sender<TargetedLoadResponse>;
-
-pub fn new_sniffer_task() -> SnifferTask {
-    let(sen, rec) = channel();
-    let builder = Builder::new().name("SnifferManager".to_string());
-    builder.spawn(move || {
-        SnifferManager::new(rec).start();
-    }).unwrap();
-    sen
-}
-
-struct SnifferManager {
-    data_receiver: Receiver<TargetedLoadResponse>,
-}
-
-impl SnifferManager {
-    fn new(data_receiver: Receiver <TargetedLoadResponse>) -> SnifferManager {
-        SnifferManager {
-            data_receiver: data_receiver,
-        }
-    }
-}
-
-impl SnifferManager {
-    fn start(self) {
-        loop {
-            match self.data_receiver.recv() {
-                Ok(snif_data) => {
-                    let _ = snif_data.consumer.send(snif_data.load_response);
-                }
-                Err(_) => break,
-            }
-        }
-    }
-}

components/util/opts.rs

@@ -132,6 +132,9 @@ pub struct Opts {
 
     /// A specific path to find required resources (such as user-agent.css).
     pub resources_path: Option<String>,
+
+    /// Whether MIME sniffing should be used
+    pub sniff_mime_types: bool,
 }
 
 fn print_usage(app: &str, opts: &[getopts::OptGroup]) {
@@ -212,6 +215,7 @@ pub fn default_opts() -> Opts {
         validate_display_list_geometry: false,
         profile_tasks: false,
         resources_path: None,
+        sniff_mime_types: false,
     }
 }
 
@@ -244,6 +248,7 @@ pub fn from_cmdline_args(args: &[String]) -> bool {
         getopts::optflag("h", "help", "Print this message"),
         getopts::optopt("r", "render-api", "Set the rendering API to use", "gl|mesa"),
         getopts::optopt("", "resources-path", "Path to find static resources", "/home/servo/resources"),
+        getopts::optflag("", "sniff-mime-types" , "Enable MIME sniffing"),
     );
 
     let opt_match = match getopts::getopts(args, opts.as_slice()) {
@@ -371,6 +376,7 @@ pub fn from_cmdline_args(args: &[String]) -> bool {
         relayout_event: debug_options.contains(&"relayout-event"),
         validate_display_list_geometry: debug_options.contains(&"validate-display-list-geometry"),
         resources_path: opt_match.opt_str("resources-path"),
+        sniff_mime_types: opt_match.opt_present("sniff-mime-types"),
     };
 
     set_opts(opts);

tests/content/parsable_mime/application/font-woff/test.wof

@@ -0,0 +1 @@
+wOFF

tests/content/parsable_mime/application/pdf/test.pdf

@@ -0,0 +1,157 @@
+%PDF-1.2 
+%âãÏÓ
+ 
+9 0 obj
+<<
+/Length 10 0 R
+/Filter /FlateDecode 
+>>
+stream
+H‰Í<EFBFBD>ÑJÃ0†Ÿ ïð{§²fç$M“ínÒ-<14>‚<EFBFBD>[&jeŠâÛÛ¤ñ~‚$ÉÉÿ}ÉÉ…¬Ij«¬ÌsÀ—‚Ç~€XÖ-],÷‚$Y—÷Ó)ü'N«u­1!œ„ÀVÙ?ŸÁ?
+žb1RbbœÒ‰ÉH²[¹™TD:#ž&Ø­ÙÌX®¦øiç»$qnf¬ƒ¿†¶]»ÀõËîãaÿ¶{ÿÂØ£‰›×q|JªLs]™QÒI¸¬jî„%¯Œ9Øé`ß঺¼ÅU»ite<74>zÛ$›’Ú¿OeBÆÄ’Ò¯á¸Råþ@zÜ—úóÿgª¼ø<õ¡ª
+endstream
+endobj
+10 0 obj
+246
+endobj
+4 0 obj
+<<
+/Type /Page
+/Parent 5 0 R
+/Resources <<
+/Font <<
+/F0 6 0 R 
+/F1 7 0 R 
+>>
+/ProcSet 2 0 R
+>>
+/Contents 9 0 R
+>>
+endobj
+6 0 obj
+<<
+/Type /Font
+/Subtype /TrueType
+/Name /F0
+/BaseFont /Arial
+/Encoding /WinAnsiEncoding
+>>
+endobj
+7 0 obj
+<<
+/Type /Font
+/Subtype /TrueType
+/Name /F1
+/BaseFont /BookAntiqua,Bold
+/FirstChar 31
+/LastChar 255
+/Widths [ 750 250 278 402 606 500 889 833 227 333 333 444 606 250 333 250 
+296 500 500 500 500 500 500 500 500 500 500 250 250 606 606 606 
+444 747 778 667 722 833 611 556 833 833 389 389 778 611 1000 833 
+833 611 833 722 611 667 778 778 1000 667 667 667 333 606 333 606 
+500 333 500 611 444 611 500 389 556 611 333 333 611 333 889 611 
+556 611 611 389 444 333 611 556 833 500 556 500 310 606 310 606 
+750 500 750 333 500 500 1000 500 500 333 1000 611 389 1000 750 750 
+750 750 278 278 500 500 606 500 1000 333 998 444 389 833 750 750 
+667 250 278 500 500 606 500 606 500 333 747 438 500 606 333 747 
+500 400 549 361 361 333 576 641 250 333 361 488 500 889 890 889 
+444 778 778 778 778 778 778 1000 722 611 611 611 611 389 389 389 
+389 833 833 833 833 833 833 833 606 833 778 778 778 778 667 611 
+611 500 500 500 500 500 500 778 444 500 500 500 500 333 333 333 
+333 556 611 556 556 556 556 556 549 556 611 611 611 611 556 611 
+556 ]
+/Encoding /WinAnsiEncoding
+/FontDescriptor 8 0 R
+>>
+endobj
+8 0 obj
+<<
+/Type /FontDescriptor
+/FontName /BookAntiqua,Bold
+/Flags 16418
+/FontBBox [ -250 -260 1236 930 ]
+/MissingWidth 750
+/StemV 146
+/StemH 146
+/ItalicAngle 0
+/CapHeight 930
+/XHeight 651
+/Ascent 930
+/Descent 260
+/Leading 210
+/MaxWidth 1030
+/AvgWidth 460
+>>
+endobj
+2 0 obj
+[ /PDF /Text  ]
+endobj
+5 0 obj
+<<
+/Kids [4 0 R ]
+/Count 1
+/Type /Pages
+/MediaBox [ 0 0 612 792 ]
+>>
+endobj
+1 0 obj
+<<
+/Creator (1725.fm)
+/CreationDate (1-Jan-3 18:15PM)
+/Title (1725.PDF)
+/Author (Unknown)
+/Producer (Acrobat PDFWriter 3.02 for Windows)
+/Keywords ()
+/Subject ()
+>>
+endobj
+3 0 obj
+<<
+/Pages 5 0 R
+/Type /Catalog
+/DefaultGray 11 0 R
+/DefaultRGB  12 0 R
+>>
+endobj
+11 0 obj
+[/CalGray
+<<
+/WhitePoint [0.9505 1 1.0891 ]
+/Gamma 0.2468 
+>>
+]
+endobj
+12 0 obj
+[/CalRGB
+<<
+/WhitePoint [0.9505 1 1.0891 ]
+/Gamma [0.2468 0.2468 0.2468 ]
+/Matrix [0.4361 0.2225 0.0139 0.3851 0.7169 0.0971 0.1431 0.0606 0.7141 ]
+>>
+]
+endobj
+xref
+0 13
+0000000000 65535 f
+0000002172 00000 n
+0000002046 00000 n
+0000002363 00000 n
+0000000375 00000 n
+0000002080 00000 n
+0000000518 00000 n
+0000000633 00000 n
+0000001760 00000 n
+0000000021 00000 n
+0000000352 00000 n
+0000002460 00000 n
+0000002548 00000 n
+trailer
+<<
+/Size 13
+/Root 3 0 R
+/Info 1 0 R
+/ID [<47149510433dd4882f05f8c124223734><47149510433dd4882f05f8c124223734>]
+>>
+startxref
+2726
+%%EOF

tests/content/parsable_mime/application/postscript/test.ps

@@ -0,0 +1 @@
+%!PS-Adobe-

tests/content/parsable_mime/application/zip/test.zip

@@ -0,0 +1 @@
+PK

tests/content/parsable_mime/text/html/text_html_a_20.html

@@ -0,0 +1,3 @@
+
+
+  <A 

tests/content/parsable_mime/text/html/text_html_a_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <a 

tests/content/parsable_mime/text/html/text_html_a_3e.html

@@ -0,0 +1,3 @@
+
+
+  <A>

tests/content/parsable_mime/text/html/text_html_a_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <a>

tests/content/parsable_mime/text/html/text_html_b_20.html

@@ -0,0 +1,3 @@
+
+
+  <B 

tests/content/parsable_mime/text/html/text_html_b_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <b 

tests/content/parsable_mime/text/html/text_html_b_3e.html

@@ -0,0 +1,3 @@
+
+
+  <B>

tests/content/parsable_mime/text/html/text_html_b_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <b>

tests/content/parsable_mime/text/html/text_html_body_20.html

@@ -0,0 +1,3 @@
+
+
+  <BODY 

tests/content/parsable_mime/text/html/text_html_body_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <body 

tests/content/parsable_mime/text/html/text_html_body_3e.html

@@ -0,0 +1,3 @@
+
+
+  <BODY>

tests/content/parsable_mime/text/html/text_html_body_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <body>

tests/content/parsable_mime/text/html/text_html_br_20.html

@@ -0,0 +1,3 @@
+
+
+  <BR 

tests/content/parsable_mime/text/html/text_html_br_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <br 

tests/content/parsable_mime/text/html/text_html_br_3e.html

@@ -0,0 +1,3 @@
+
+
+  <BR>

tests/content/parsable_mime/text/html/text_html_br_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <br>

tests/content/parsable_mime/text/html/text_html_comment_20.html

@@ -0,0 +1,3 @@
+
+
+  <!-- 

tests/content/parsable_mime/text/html/text_html_comment_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <!-- 

tests/content/parsable_mime/text/html/text_html_comment_3e.html

@@ -0,0 +1,3 @@
+
+
+  <!-->

tests/content/parsable_mime/text/html/text_html_comment_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <!-->

tests/content/parsable_mime/text/html/text_html_div_20.html

@@ -0,0 +1,3 @@
+
+
+  <DIV 

tests/content/parsable_mime/text/html/text_html_div_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <div 

tests/content/parsable_mime/text/html/text_html_div_3e.html

@@ -0,0 +1,3 @@
+
+
+  <DIV>

tests/content/parsable_mime/text/html/text_html_div_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <div>

tests/content/parsable_mime/text/html/text_html_doctype_20.html

@@ -0,0 +1,3 @@
+
+
+  <!DOCTYPE HTML 

tests/content/parsable_mime/text/html/text_html_doctype_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <!doctype html 

tests/content/parsable_mime/text/html/text_html_doctype_3e.html

@@ -0,0 +1,4 @@
+
+
+  <!DOCTYPE HTML>
+

tests/content/parsable_mime/text/html/text_html_doctype_3e_u.html

@@ -0,0 +1,4 @@
+
+
+  <!doctype html>
+

tests/content/parsable_mime/text/html/text_html_font_20.html

@@ -0,0 +1,3 @@
+
+
+  <FONT 

tests/content/parsable_mime/text/html/text_html_font_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <font 

tests/content/parsable_mime/text/html/text_html_font_3e.html

@@ -0,0 +1,3 @@
+
+
+  <FONT>

tests/content/parsable_mime/text/html/text_html_font_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <font>

tests/content/parsable_mime/text/html/text_html_h1_20.html

@@ -0,0 +1,3 @@
+
+
+  <H1 

tests/content/parsable_mime/text/html/text_html_h1_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <h1 

tests/content/parsable_mime/text/html/text_html_h1_3e.html

@@ -0,0 +1,3 @@
+
+
+  <H1>

tests/content/parsable_mime/text/html/text_html_h1_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <h1>

tests/content/parsable_mime/text/html/text_html_head_20.html

@@ -0,0 +1,3 @@
+
+
+  <HEAD 

tests/content/parsable_mime/text/html/text_html_head_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <head 

tests/content/parsable_mime/text/html/text_html_head_3e.html

@@ -0,0 +1,3 @@
+
+
+  <HEAD>

tests/content/parsable_mime/text/html/text_html_head_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <head>

tests/content/parsable_mime/text/html/text_html_iframe_20.html

@@ -0,0 +1,3 @@
+
+
+  <IFRAME 

tests/content/parsable_mime/text/html/text_html_iframe_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <iframe 

tests/content/parsable_mime/text/html/text_html_iframe_3e.html

@@ -0,0 +1,3 @@
+
+
+  <IFRAME>

tests/content/parsable_mime/text/html/text_html_iframe_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <iframe>

tests/content/parsable_mime/text/html/text_html_p_20.html

@@ -0,0 +1,3 @@
+
+
+  <P 

tests/content/parsable_mime/text/html/text_html_p_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <p 

tests/content/parsable_mime/text/html/text_html_p_3e.html

@@ -0,0 +1,3 @@
+
+
+  <P>

tests/content/parsable_mime/text/html/text_html_p_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <p>

tests/content/parsable_mime/text/html/text_html_page_20.html

@@ -0,0 +1,3 @@
+
+
+  <HTML 

tests/content/parsable_mime/text/html/text_html_page_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <html 

tests/content/parsable_mime/text/html/text_html_page_3e.html

@@ -0,0 +1,3 @@
+
+
+  <HTML>

tests/content/parsable_mime/text/html/text_html_page_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <html>

tests/content/parsable_mime/text/html/text_html_script_20.html

@@ -0,0 +1,3 @@
+
+
+  <SCRIPT 

tests/content/parsable_mime/text/html/text_html_script_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <script 

tests/content/parsable_mime/text/html/text_html_script_3e.html

@@ -0,0 +1,3 @@
+
+
+  <SCRIPT>

tests/content/parsable_mime/text/html/text_html_script_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <script>

tests/content/parsable_mime/text/html/text_html_style_20.html

@@ -0,0 +1,3 @@
+
+
+  <STYLE 

tests/content/parsable_mime/text/html/text_html_style_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <style 

tests/content/parsable_mime/text/html/text_html_style_3e.html

@@ -0,0 +1,3 @@
+
+
+  <STYLE>

tests/content/parsable_mime/text/html/text_html_style_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <style>

tests/content/parsable_mime/text/html/text_html_table_20.html

@@ -0,0 +1,3 @@
+
+
+  <TABLE 

tests/content/parsable_mime/text/html/text_html_table_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <table 

tests/content/parsable_mime/text/html/text_html_table_3e.html

@@ -0,0 +1,3 @@
+
+
+  <TABLE>

tests/content/parsable_mime/text/html/text_html_table_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <table>

tests/content/parsable_mime/text/html/text_html_title_20.html

@@ -0,0 +1,3 @@
+
+
+  <TITLE 

tests/content/parsable_mime/text/html/text_html_title_20_u.html

@@ -0,0 +1,3 @@
+
+
+  <title 

tests/content/parsable_mime/text/html/text_html_title_3e.html

@@ -0,0 +1,3 @@
+
+
+  <TITLE>

tests/content/parsable_mime/text/html/text_html_title_3e_u.html

@@ -0,0 +1,3 @@
+
+
+  <title>