From f674cba6125cecddc9a54182f66963238b5bdee1 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Wed, 12 Nov 2014 14:08:13 -0500 Subject: [PATCH 1/7] Beginnings of a http cache Doom cache entries based on the initial response, and prevent matching against doomed cache enties. Evict cache entries that have passed their expiry date instead of matching them. Document the cache. Refactor incomplete entries to lessen Option-itis. Revalidate expired cache entries instead of unconditionally evicting them. Forbid missing docs in cache code. Revalidate must-revalidate entries. Fetch content tests from a local HTTP server. Track requests made to the test HTTP server. Add a simple test that a cached resource with no expiry is not revalidated. Correct inverted expiry check in revalidation code. Fix incorrect revalidation logic that dropped the consumer channels on the floor. Ensure that requests are cached based on their request headers. Run a separate http server instance for each test to avoid intermittent failures due to concurrent cache tests. Add a test for uncacheable responses. Address review comments. --- components/net/http_cache.rs | 500 ++++++++++++++++++ components/net/lib.rs | 1 + components/net/resource_task.rs | 291 ++++++++++ components/script/parse/html.rs | 268 ++++++++++ components/util/time.rs | 297 +++++++++++ tests/content/harness.js | 106 ++++ tests/content/netharness.js | 25 + tests/content/resources/helper.html | 2 + .../resources/helper_must_revalidate.html | 2 + .../helper_must_revalidate.html^headers | 2 + tests/content/resources/helper_nocache.html | 2 + .../resources/helper_nocache.html^headers | 2 + tests/content/test_cached_headers_differ.html | 14 + tests/content/test_cached_request.html | 14 + tests/content/test_document_url.html | 30 ++ tests/content/test_nocache.html | 14 + tests/content/test_revalidate.html | 14 + tests/contenttest.rs | 194 +++++++ tests/httpserver.py | 115 ++++ 19 files changed, 1893 insertions(+) create mode 100644 components/net/http_cache.rs create mode 100644 components/net/resource_task.rs create mode 100644 components/script/parse/html.rs create mode 100644 components/util/time.rs create mode 100644 tests/content/harness.js create mode 100644 tests/content/netharness.js create mode 100644 tests/content/resources/helper.html create mode 100644 tests/content/resources/helper_must_revalidate.html create mode 100644 tests/content/resources/helper_must_revalidate.html^headers create mode 100644 tests/content/resources/helper_nocache.html create mode 100644 tests/content/resources/helper_nocache.html^headers create mode 100644 tests/content/test_cached_headers_differ.html create mode 100644 tests/content/test_cached_request.html create mode 100644 tests/content/test_document_url.html create mode 100644 tests/content/test_nocache.html create mode 100644 tests/content/test_revalidate.html create mode 100644 tests/contenttest.rs create mode 100644 tests/httpserver.py diff --git a/components/net/http_cache.rs b/components/net/http_cache.rs new file mode 100644 index 00000000000..cfb2c8acc01 --- /dev/null +++ b/components/net/http_cache.rs @@ -0,0 +1,500 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#![deny(missing_docs)] + +//! A non-validating memory cache that only evicts expired entries and grows +//! without bound. Implements the logic specified in http://tools.ietf.org/html/rfc7234 +//! and http://tools.ietf.org/html/rfc7232. + +use http_loader::send_error_direct; +use resource_task::{Metadata, ProgressMsg, LoadResponse, LoadData, Payload, Done, start_sending_opt}; + +use servo_util::time::parse_http_timestamp; + +use http::headers::etag::EntityTag; +use http::headers::HeaderEnum; +use http::headers::response::HeaderCollection as ResponseHeaderCollection; +use http::method::Get; +use http::status::Ok as StatusOk; + +use std::collections::HashMap; +use std::comm::Sender; +use std::iter::Map; +use std::mem; +use std::num::{Bounded, FromStrRadix}; +use std::str::CharSplits; +use std::sync::{Arc, Mutex}; +use std::time::duration::{MAX, Duration}; +use time; +use time::{Tm, Timespec}; +use url::Url; + +//TODO: Store an Arc> instead? +//TODO: Cache HEAD requests +//TODO: Doom responses with network errors +//TODO: Send Err responses for doomed entries +//TODO: Enable forced eviction of a request instead of retrieving the cached response +//TODO: Doom incomplete entries +//TODO: Cache-Control: must-revalidate +//TODO: Last-Modified +//TODO: Range requests +//TODO: Revalidation rules for query strings +//TODO: Vary header + +/// The key used to differentiate requests in the cache. +#[deriving(Clone, Hash, PartialEq, Eq)] +pub struct CacheKey { + url: Url, + request_headers: Vec<(String, String)>, +} + +impl CacheKey { + fn new(load_data: LoadData) -> CacheKey { + CacheKey { + url: load_data.url.clone(), + request_headers: load_data.headers + .iter() + .map(|header| (header.header_name(), header.header_value())) + .collect(), + } + } + + /// Retrieve the URL associated with this key + pub fn url(&self) -> Url { + self.url.clone() + } +} + +/// The list of consumers waiting on this requests's response. +enum PendingConsumers { + /// Consumers awaiting the initial response metadata + AwaitingHeaders(Vec>), + /// Consumers awaiting the remaining response body. Incomplete body stored as Vec. + AwaitingBody(Metadata, Vec, Vec>), +} + +/// An unfulfilled request representing both the consumers waiting for the initial +/// metadata and the subsequent response body. If doomed, the entry will be removed +/// after the final payload. +struct PendingResource { + consumers: PendingConsumers, + expires: Duration, + last_validated: Tm, + doomed: bool, +} + +/// A complete cached resource. +struct CachedResource { + metadata: Metadata, + body: Vec, + expires: Duration, + last_validated: Tm, + revalidating_consumers: Vec>, +} + +/// A memory cache that tracks incomplete and complete responses, differentiated by +/// the initial request. +pub struct MemoryCache { + /// Complete cached responses. + complete_entries: HashMap, + /// Incomplete cached responses. + pending_entries: HashMap, + /// The time at which this cache was created for use by expiry checks. + base_time: Timespec, +} + +/// Abstraction over the concept of a single target for HTTP response messages. +pub enum ResourceResponseTarget { + /// A response is being streamed into the cache. + CachedPendingResource(CacheKey, Arc>), + /// A response is being streamed directly to a consumer and skipping the cache. + UncachedPendingResource(Sender), +} + +/// Abstraction over the concept of a single target for HTTP response payload messages. +pub enum ResourceProgressTarget { + /// A response is being streamed into the cache. + CachedInProgressResource(CacheKey, Arc>), + /// A response is being streamed directly to a consumer and skipping the cache. + UncachedInProgressResource(Sender), +} + +/// The result of matching a request against an HTTP cache. +pub enum CacheOperationResult { + /// The request cannot be cached for a given reason. + Uncacheable(&'static str), + /// The request is in the cache and the response data is forthcoming. + CachedContentPending, + /// The request is not present in the cache but will be cached with the given key. + NewCacheEntry(CacheKey), + /// The request is in the cache but requires revalidation. + Revalidate(CacheKey, RevalidationMethod), +} + +/// The means by which to revalidate stale cached content +pub enum RevalidationMethod { + /// The result of a stored Last-Modified or Expires header + ExpiryDate(Tm), + /// The result of a stored Etag header + Etag(EntityTag), +} + +/// Tokenize a header value. +fn split_header(header: &str) -> Map<&str, &str, CharSplits> { + header.split(',') + .map(|v| v.trim()) +} + +/// Match any header value token. +fn any_token_matches(header: &str, tokens: &[&str]) -> bool { + split_header(header).any(|token| tokens.iter().any(|&s| s == token)) +} + +/// Determine if a given response is cacheable based on the initial metadata received. +/// Based on http://tools.ietf.org/html/rfc7234#section-5 +fn response_is_cacheable(metadata: &Metadata) -> bool { + if metadata.status != StatusOk { + return false; + } + + if metadata.headers.is_none() { + return true; + } + + let headers = metadata.headers.as_ref().unwrap(); + match headers.cache_control { + Some(ref cache_control) => { + if any_token_matches(cache_control[], &["no-cache", "no-store", "max-age=0"]) { + return false; + } + } + None => () + } + + match headers.pragma { + Some(ref pragma) => { + if any_token_matches(pragma[], &["no-cache"]) { + return false; + } + } + None => () + } + + return true; +} + +/// Determine the expiry date of the given response headers. +/// Returns a far-future date if the response does not expire. +fn get_response_expiry_from_headers(headers: &ResponseHeaderCollection) -> Duration { + headers.cache_control.as_ref().and_then(|cache_control| { + for token in split_header(cache_control[]) { + let mut parts = token.split('='); + if parts.next() == Some("max-age") { + return parts.next() + .and_then(|val| FromStrRadix::from_str_radix(val, 10)) + .map(|secs| Duration::seconds(secs)); + } + } + None + }).or_else(|| { + headers.expires.as_ref().and_then(|expires| { + parse_http_timestamp(expires[]).map(|t| { + // store the period of time from now until expiry + let desired = t.to_timespec(); + let current = time::now().to_timespec(); + if desired > current { + desired - current + } else { + Bounded::min_value() + } + }) + }) + }).unwrap_or(Bounded::max_value()) +} + +/// Determine the expiry date of the given response. +/// Returns a far-future date if this response does not expire. +fn get_response_expiry(metadata: &Metadata) -> Duration { + metadata.headers.as_ref().map(|headers| { + get_response_expiry_from_headers(headers) + }).unwrap_or(Bounded::max_value()) +} + +impl MemoryCache { + /// Create a new memory cache instance. + pub fn new() -> MemoryCache { + MemoryCache { + complete_entries: HashMap::new(), + pending_entries: HashMap::new(), + base_time: time::now().to_timespec(), + } + } + + /// Process a revalidation that returned new content for an expired entry. + pub fn process_revalidation_failed(&mut self, key: &CacheKey) { + debug!("recreating entry for {} (cache entry expired)", key.url); + let resource = self.complete_entries.remove(key).unwrap(); + self.add_pending_cache_entry(key.clone(), resource.revalidating_consumers); + } + + /// Mark an incomplete cached request as doomed. Any waiting consumers will immediately + /// receive an error message or a final body payload. The cache entry is immediately + /// removed. + pub fn doom_request(&mut self, key: &CacheKey, err: String) { + debug!("dooming entry for {} ({})", key.url, err); + + assert!(!self.complete_entries.contains_key(key)); + + let resource = self.pending_entries.remove(key).unwrap(); + match resource.consumers { + AwaitingHeaders(ref consumers) => { + for consumer in consumers.iter() { + send_error_direct(key.url.clone(), err.clone(), consumer.clone()); + } + } + AwaitingBody(_, _, ref consumers) => { + for consumer in consumers.iter() { + let _ = consumer.send_opt(Done(Ok(()))); + } + } + } + } + + /// Handle a 304 response to a revalidation request. Updates the cached response + /// metadata with any new expiration data. + pub fn process_not_modified(&mut self, key: &CacheKey, headers: &ResponseHeaderCollection) { + debug!("updating metadata for {}", key.url); + let resource = self.complete_entries.get_mut(key).unwrap(); + resource.expires = get_response_expiry_from_headers(headers); + + for consumer in mem::replace(&mut resource.revalidating_consumers, vec!()).into_iter() { + MemoryCache::send_complete_resource(resource, consumer); + } + } + + /// Handle the initial response metadata for an incomplete cached request. + /// If the response should not be cached, the entry will be doomed and any + /// subsequent requests will not see the cached request. All waiting consumers + /// will see the new metadata. + pub fn process_metadata(&mut self, key: &CacheKey, metadata: Metadata) { + debug!("storing metadata for {}", key.url); + let resource = self.pending_entries.get_mut(key).unwrap(); + let chans: Vec>; + match resource.consumers { + AwaitingHeaders(ref consumers) => { + chans = consumers.iter() + .map(|chan| start_sending_opt(chan.clone(), metadata.clone())) + .take_while(|chan| chan.is_ok()) + .map(|chan| chan.unwrap()) + .collect(); + } + AwaitingBody(..) => panic!("obtained headers for {} but awaiting body?", key.url) + } + + if !response_is_cacheable(&metadata) { + resource.doomed = true; + } + + resource.expires = get_response_expiry(&metadata); + resource.last_validated = time::now(); + resource.consumers = AwaitingBody(metadata, vec!(), chans); + } + + /// Handle a repsonse body payload for an incomplete cached response. + /// All waiting consumers will see the new payload addition. + pub fn process_payload(&mut self, key: &CacheKey, payload: Vec) { + debug!("storing partial response for {}", key.url); + let resource = self.pending_entries.get_mut(key).unwrap(); + match resource.consumers { + AwaitingBody(_, ref mut body, ref consumers) => { + body.push_all(payload.as_slice()); + for consumer in consumers.iter() { + //FIXME: maybe remove consumer on failure to avoid extra clones? + let _ = consumer.send_opt(Payload(payload.clone())); + } + } + AwaitingHeaders(_) => panic!("obtained body for {} but awaiting headers?", key.url) + } + } + + /// Handle a response body final payload for an incomplete cached response. + /// All waiting consumers will see the new message. If the cache entry is + /// doomed, it will not be transferred to the set of complete cache entries. + pub fn process_done(&mut self, key: &CacheKey) { + debug!("finished fetching {}", key.url); + let resource = self.pending_entries.remove(key).unwrap(); + match resource.consumers { + AwaitingHeaders(_) => panic!("saw Done for {} but awaiting headers?", key.url), + AwaitingBody(_, _, ref consumers) => { + for consumer in consumers.iter() { + let _ = consumer.send_opt(Done(Ok(()))); + } + } + } + + if resource.doomed { + debug!("completing dooming of {}", key.url); + return; + } + + let (metadata, body) = match resource.consumers { + AwaitingBody(metadata, body, _) => (metadata, body), + _ => panic!("expected consumer list awaiting bodies"), + }; + + let complete = CachedResource { + metadata: metadata, + body: body, + expires: resource.expires, + last_validated: resource.last_validated, + revalidating_consumers: vec!(), + }; + self.complete_entries.insert(key.clone(), complete); + } + + /// Match a new request against the set of incomplete and complete cached requests. + /// If the request matches an existing, non-doomed entry, any existing response data will + /// be synchronously streamed to the consumer. If the request does not match but can be + /// cached, a new cache entry will be created and the request will be responsible for + /// notifying the cache of the subsequent HTTP response. If the request does not match + /// and cannot be cached, the request is responsible for handling its own response and + /// consumer. + pub fn process_pending_request(&mut self, load_data: &LoadData, start_chan: Sender) + -> CacheOperationResult { + fn revalidate(resource: &mut CachedResource, + key: &CacheKey, + start_chan: Sender, + method: RevalidationMethod) -> CacheOperationResult { + // Ensure that at most one revalidation is taking place at a time for a + // cached resource. + resource.revalidating_consumers.push(start_chan); + if resource.revalidating_consumers.len() > 1 { + CachedContentPending + } else { + Revalidate(key.clone(), method) + } + } + + if load_data.method != Get { + return Uncacheable("Only GET requests can be cached."); + } + + let key = CacheKey::new(load_data.clone()); + match self.complete_entries.get_mut(&key) { + Some(resource) => { + if self.base_time + resource.expires < time::now().to_timespec() { + debug!("entry for {} has expired", key.url()); + let expiry = time::at(self.base_time + resource.expires); + return revalidate(resource, &key, start_chan, ExpiryDate(expiry)); + } + + let must_revalidate = resource.metadata.headers.as_ref().and_then(|headers| { + headers.cache_control.as_ref().map(|header| { + any_token_matches(header[], &["must-revalidate"]) + }) + }).unwrap_or(false); + + if must_revalidate { + debug!("entry for {} must be revalidated", key.url()); + let last_validated = resource.last_validated; + return revalidate(resource, &key, start_chan, ExpiryDate(last_validated)); + } + + let etag = resource.metadata.headers.as_ref().and_then(|headers| headers.etag.clone()); + match etag { + Some(etag) => { + debug!("entry for {} has an Etag", key.url()); + return revalidate(resource, &key, start_chan, Etag(etag.clone())); + } + None => () + } + + //TODO: Revalidate once per session for response with no explicit expiry + } + + None => () + } + + if self.complete_entries.contains_key(&key) { + self.send_complete_entry(key, start_chan); + return CachedContentPending; + } + + let new_entry = match self.pending_entries.get(&key) { + Some(resource) if resource.doomed => return Uncacheable("Cache entry already doomed"), + Some(_) => false, + None => true, + }; + + if new_entry { + self.add_pending_cache_entry(key.clone(), vec!(start_chan)); + NewCacheEntry(key) + } else { + self.send_partial_entry(key, start_chan); + CachedContentPending + } + } + + /// Add a new pending request to the set of incomplete cache entries. + fn add_pending_cache_entry(&mut self, key: CacheKey, consumers: Vec>) { + let resource = PendingResource { + consumers: AwaitingHeaders(consumers), + expires: MAX, + last_validated: time::now(), + doomed: false, + }; + debug!("creating cache entry for {}", key.url); + self.pending_entries.insert(key, resource); + } + + /// Synchronously send the entire cached response body to the given consumer. + fn send_complete_resource(resource: &CachedResource, start_chan: Sender) { + let progress_chan = start_sending_opt(start_chan, resource.metadata.clone()); + match progress_chan { + Ok(chan) => { + let _ = chan.send_opt(Payload(resource.body.clone())); + let _ = chan.send_opt(Done(Ok(()))); + } + Err(_) => () + } + } + + /// Synchronously send the entire cached response body to the given consumer. + fn send_complete_entry(&self, key: CacheKey, start_chan: Sender) { + debug!("returning full cache body for {}", key.url); + let resource = self.complete_entries.get(&key).unwrap(); + MemoryCache::send_complete_resource(resource, start_chan) + } + + /// Synchronously send all partial stored response data for a cached request to the + /// given consumer. + fn send_partial_entry(&mut self, key: CacheKey, start_chan: Sender) { + debug!("returning partial cache data for {}", key.url); + + let resource = self.pending_entries.get_mut(&key).unwrap(); + + match resource.consumers { + AwaitingHeaders(ref mut consumers) => { + consumers.push(start_chan); + } + AwaitingBody(ref metadata, ref body, ref mut consumers) => { + debug!("headers available for {}", key.url); + let progress_chan = start_sending_opt(start_chan, metadata.clone()); + match progress_chan { + Ok(chan) => { + consumers.push(chan.clone()); + + if !body.is_empty() { + debug!("partial body available for {}", key.url); + let _ = chan.send_opt(Payload(body.clone())); + } + } + + Err(_) => () + } + } + } + } +} diff --git a/components/net/lib.rs b/components/net/lib.rs index 7869ff1bb06..c77fce6c509 100644 --- a/components/net/lib.rs +++ b/components/net/lib.rs @@ -48,6 +48,7 @@ mod data_loader; pub mod filemanager_thread; mod hosts; pub mod hsts; +pub mod http_cache; pub mod http_loader; pub mod image_cache; pub mod mime_classifier; diff --git a/components/net/resource_task.rs b/components/net/resource_task.rs new file mode 100644 index 00000000000..ec6d49f065a --- /dev/null +++ b/components/net/resource_task.rs @@ -0,0 +1,291 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! A task that takes a URL and streams back the binary data. + +use about_loader; +use data_loader; +use file_loader; +use http_cache::MemoryCache; +use http_loader; +use sniffer_task; + +use std::comm::{channel, Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use http::headers::content_type::MediaType; +use http::headers::response::HeaderCollection as ResponseHeaderCollection; +use http::headers::request::HeaderCollection as RequestHeaderCollection; +use http::method::{Method, Get}; +use url::Url; + +use http::status::Ok as StatusOk; +use http::status::Status; + +use servo_util::task::spawn_named; + +pub enum ControlMsg { + /// Request the data associated with a particular URL + Load(LoadData, Sender), + Exit +} + +#[deriving(Clone)] +pub struct LoadData { + pub url: Url, + pub method: Method, + pub headers: RequestHeaderCollection, + pub data: Option>, + pub cors: Option +} + +impl LoadData { + pub fn new(url: Url) -> LoadData { + LoadData { + url: url, + method: Get, + headers: RequestHeaderCollection::new(), + data: None, + cors: None + } + } +} + +#[deriving(Clone)] +pub struct ResourceCORSData { + /// CORS Preflight flag + pub preflight: bool, + /// Origin of CORS Request + pub origin: Url +} + +/// Metadata about a loaded resource, such as is obtained from HTTP headers. +#[deriving(Clone)] +pub struct Metadata { + /// Final URL after redirects. + pub final_url: Url, + + /// MIME type / subtype. + pub content_type: Option<(String, String)>, + + /// Character set. + pub charset: Option, + + /// Headers + pub headers: Option, + + /// HTTP Status + pub status: Status +} + +impl Metadata { + /// Metadata with defaults for everything optional. + pub fn default(url: Url) -> Metadata { + Metadata { + final_url: url, + content_type: None, + charset: None, + headers: None, + status: StatusOk // http://fetch.spec.whatwg.org/#concept-response-status-message + } + } + + /// Extract the parts of a MediaType that we care about. + pub fn set_content_type(&mut self, content_type: &Option) { + match *content_type { + None => (), + Some(MediaType { ref type_, + ref subtype, + ref parameters }) => { + self.content_type = Some((type_.clone(), subtype.clone())); + for &(ref k, ref v) in parameters.iter() { + if "charset" == k.as_slice() { + self.charset = Some(v.clone()); + } + } + } + } + } +} + +/// Message sent in response to `Load`. Contains metadata, and a port +/// for receiving the data. +/// +/// Even if loading fails immediately, we send one of these and the +/// progress_port will provide the error. +pub struct LoadResponse { + /// Metadata, such as from HTTP headers. + pub metadata: Metadata, + /// Port for reading data. + pub progress_port: Receiver, +} + +/// Messages sent in response to a `Load` message +#[deriving(PartialEq,Show)] +pub enum ProgressMsg { + /// Binary data - there may be multiple of these + Payload(Vec), + /// Indicates loading is complete, either successfully or not + Done(Result<(), String>) +} + +/// For use by loaders in responding to a Load message. +pub fn start_sending(start_chan: Sender, metadata: Metadata) -> Sender { + start_sending_opt(start_chan, metadata).ok().unwrap() +} + +/// For use by loaders in responding to a Load message. +pub fn start_sending_opt(start_chan: Sender, metadata: Metadata) -> Result, ()> { + let (progress_chan, progress_port) = channel(); + let result = start_chan.send_opt(LoadResponse { + metadata: metadata, + progress_port: progress_port, + }); + match result { + Ok(_) => Ok(progress_chan), + Err(_) => Err(()) + } +} + +/// Convenience function for synchronously loading a whole resource. +pub fn load_whole_resource(resource_task: &ResourceTask, url: Url) + -> Result<(Metadata, Vec), String> { + let (start_chan, start_port) = channel(); + resource_task.send(Load(LoadData::new(url), start_chan)); + let response = start_port.recv(); + + let mut buf = vec!(); + loop { + match response.progress_port.recv() { + Payload(data) => buf.push_all(data.as_slice()), + Done(Ok(())) => return Ok((response.metadata, buf)), + Done(Err(e)) => return Err(e) + } + } +} + +/// Handle to a resource task +pub type ResourceTask = Sender; + +/// Create a ResourceTask +pub fn new_resource_task(user_agent: Option) -> ResourceTask { + let (setup_chan, setup_port) = channel(); + spawn_named("ResourceManager", proc() { + ResourceManager::new(setup_port, user_agent).start(); + }); + setup_chan +} + +struct ResourceManager { + from_client: Receiver, + user_agent: Option, + memory_cache: Arc>, +} + +impl ResourceManager { + fn new(from_client: Receiver, user_agent: Option) -> ResourceManager { + ResourceManager { + from_client: from_client, + user_agent: user_agent, + memory_cache: Arc::new(Mutex::new(MemoryCache::new())), + } + } +} + + +impl ResourceManager { + fn start(&self) { + loop { + match self.from_client.recv() { + Load(load_data, start_chan) => { + self.load(load_data, start_chan) + } + Exit => { + break + } + } + } + } + + fn load(&self, load_data: LoadData, start_chan: Sender) { + let mut load_data = load_data; + load_data.headers.user_agent = self.user_agent.clone(); + + // Create new communication channel, create new sniffer task, + // send all the data to the new sniffer task with the send + // end of the pipe, receive all the data. + + let sniffer_task = sniffer_task::new_sniffer_task(start_chan.clone()); + + fn from_factory<'a>(factory: fn(LoadData, Sender)) + -> proc(LoadData, Sender): 'a { + proc(load_data: LoadData, start_chan: Sender) { + factory(load_data, start_chan) + } + } + + let loader = match load_data.url.scheme.as_slice() { + "file" => from_factory(file_loader::factory), + "http" | "https" => http_loader::factory(self.memory_cache.clone()), + "data" => from_factory(data_loader::factory), + "about" => from_factory(about_loader::factory), + _ => { + debug!("resource_task: no loader for scheme {:s}", load_data.url.scheme); + start_sending(start_chan, Metadata::default(load_data.url)) + .send(Done(Err("no loader for scheme".to_string()))); + return + } + }; + debug!("resource_task: loading url: {:s}", load_data.url.serialize()); + + loader(load_data, sniffer_task); + } +} + +/// Load a URL asynchronously and iterate over chunks of bytes from the response. +pub fn load_bytes_iter(resource_task: &ResourceTask, url: Url) -> (Metadata, ProgressMsgPortIterator) { + let (input_chan, input_port) = channel(); + resource_task.send(Load(LoadData::new(url), input_chan)); + + let response = input_port.recv(); + let iter = ProgressMsgPortIterator { progress_port: response.progress_port }; + (response.metadata, iter) +} + +/// Iterator that reads chunks of bytes from a ProgressMsg port +pub struct ProgressMsgPortIterator { + progress_port: Receiver +} + +impl Iterator> for ProgressMsgPortIterator { + fn next(&mut self) -> Option> { + match self.progress_port.recv() { + Payload(data) => Some(data), + Done(Ok(())) => None, + Done(Err(e)) => { + error!("error receiving bytes: {}", e); + None + } + } + } +} + +#[test] +fn test_exit() { + let resource_task = new_resource_task(None); + resource_task.send(Exit); +} + +#[test] +fn test_bad_scheme() { + let resource_task = new_resource_task(None); + let (start_chan, start) = channel(); + let url = Url::parse("bogus://whatever").unwrap(); + resource_task.send(Load(LoadData::new(url), start_chan)); + let response = start.recv(); + match response.progress_port.recv() { + Done(result) => { assert!(result.is_err()) } + _ => panic!("bleh") + } + resource_task.send(Exit); +} diff --git a/components/script/parse/html.rs b/components/script/parse/html.rs new file mode 100644 index 00000000000..b69d60da8d1 --- /dev/null +++ b/components/script/parse/html.rs @@ -0,0 +1,268 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use dom::attr::AttrHelpers; +use dom::bindings::codegen::Bindings::NodeBinding::NodeMethods; +use dom::bindings::codegen::InheritTypes::{NodeCast, ElementCast, HTMLScriptElementCast}; +use dom::bindings::js::{JS, JSRef, Temporary, OptionalRootable, Root}; +use dom::comment::Comment; +use dom::document::{Document, DocumentHelpers}; +use dom::documenttype::DocumentType; +use dom::element::{Element, AttributeHandlers, ElementHelpers, ParserCreated}; +use dom::htmlscriptelement::HTMLScriptElement; +use dom::htmlscriptelement::HTMLScriptElementHelpers; +use dom::node::{Node, NodeHelpers, TrustedNodeAddress}; +use dom::servohtmlparser; +use dom::servohtmlparser::ServoHTMLParser; +use dom::text::Text; +use page::Page; +use parse::Parser; + +use encoding::all::UTF_8; +use encoding::types::{Encoding, DecodeReplace}; + +use servo_net::resource_task::{Load, LoadData, Payload, Done, ResourceTask}; +use servo_msg::constellation_msg::LoadData as MsgLoadData; +use servo_util::task_state; +use servo_util::task_state::IN_HTML_PARSER; +use servo_util::time::parse_http_timestamp; +use std::ascii::AsciiExt; +use std::comm::channel; +use std::str::MaybeOwned; +use url::Url; +use http::headers::HeaderEnum; +use html5ever::Attribute; +use html5ever::tree_builder::{TreeSink, QuirksMode, NodeOrText, AppendNode, AppendText}; +use string_cache::QualName; + +pub enum HTMLInput { + InputString(String), + InputUrl(Url), +} + +// Parses an RFC 2616 compliant date/time string, and returns a localized +// date/time string in a format suitable for document.lastModified. +fn parse_last_modified(timestamp: &str) -> String { + parse_http_timestamp(timestamp).map(|t| { + t.to_local().strftime("%m/%d/%Y %H:%M:%S").unwrap() + }).unwrap_or(String::new()) +} + +trait SinkHelpers { + fn get_or_create(&self, child: NodeOrText) -> Temporary; +} + +impl SinkHelpers for servohtmlparser::Sink { + fn get_or_create(&self, child: NodeOrText) -> Temporary { + match child { + AppendNode(n) => Temporary::new(unsafe { JS::from_trusted_node_address(n) }), + AppendText(t) => { + let doc = self.document.root(); + let text = Text::new(t, *doc); + NodeCast::from_temporary(text) + } + } + } +} + +impl<'a> TreeSink for servohtmlparser::Sink { + fn get_document(&mut self) -> TrustedNodeAddress { + let doc = self.document.root(); + let node: JSRef = NodeCast::from_ref(*doc); + node.to_trusted_node_address() + } + + fn same_node(&self, x: TrustedNodeAddress, y: TrustedNodeAddress) -> bool { + x == y + } + + fn elem_name(&self, target: TrustedNodeAddress) -> QualName { + let node: Root = unsafe { JS::from_trusted_node_address(target).root() }; + let elem: JSRef = ElementCast::to_ref(*node) + .expect("tried to get name of non-Element in HTML parsing"); + QualName { + ns: elem.get_namespace().clone(), + local: elem.get_local_name().clone(), + } + } + + fn create_element(&mut self, name: QualName, attrs: Vec) + -> TrustedNodeAddress { + let doc = self.document.root(); + let elem = Element::create(name, None, *doc, ParserCreated).root(); + + for attr in attrs.into_iter() { + elem.set_attribute_from_parser(attr.name, attr.value, None); + } + + let node: JSRef = NodeCast::from_ref(*elem); + node.to_trusted_node_address() + } + + fn create_comment(&mut self, text: String) -> TrustedNodeAddress { + let doc = self.document.root(); + let comment = Comment::new(text, *doc); + let node: Root = NodeCast::from_temporary(comment).root(); + node.to_trusted_node_address() + } + + fn append_before_sibling(&mut self, + sibling: TrustedNodeAddress, + new_node: NodeOrText) -> Result<(), NodeOrText> { + // If there is no parent, return the node to the parser. + let sibling: Root = unsafe { JS::from_trusted_node_address(sibling).root() }; + let parent = match sibling.parent_node() { + Some(p) => p.root(), + None => return Err(new_node), + }; + + let child = self.get_or_create(new_node).root(); + assert!(parent.InsertBefore(*child, Some(*sibling)).is_ok()); + Ok(()) + } + + fn parse_error(&mut self, msg: MaybeOwned<'static>) { + debug!("Parse error: {:s}", msg); + } + + fn set_quirks_mode(&mut self, mode: QuirksMode) { + let doc = self.document.root(); + doc.set_quirks_mode(mode); + } + + fn append(&mut self, parent: TrustedNodeAddress, child: NodeOrText) { + let parent: Root = unsafe { JS::from_trusted_node_address(parent).root() }; + let child = self.get_or_create(child).root(); + + // FIXME(#3701): Use a simpler algorithm and merge adjacent text nodes + assert!(parent.AppendChild(*child).is_ok()); + } + + fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) { + let doc = self.document.root(); + let doc_node: JSRef = NodeCast::from_ref(*doc); + let doctype = DocumentType::new(name, Some(public_id), Some(system_id), *doc); + let node: Root = NodeCast::from_temporary(doctype).root(); + + assert!(doc_node.AppendChild(*node).is_ok()); + } + + fn add_attrs_if_missing(&mut self, target: TrustedNodeAddress, attrs: Vec) { + let node: Root = unsafe { JS::from_trusted_node_address(target).root() }; + let elem: JSRef = ElementCast::to_ref(*node) + .expect("tried to set attrs on non-Element in HTML parsing"); + for attr in attrs.into_iter() { + elem.set_attribute_from_parser(attr.name, attr.value, None); + } + } + + fn remove_from_parent(&mut self, _target: TrustedNodeAddress) { + error!("remove_from_parent not implemented!"); + } + + fn mark_script_already_started(&mut self, node: TrustedNodeAddress) { + let node: Root = unsafe { JS::from_trusted_node_address(node).root() }; + let script: Option> = HTMLScriptElementCast::to_ref(*node); + script.map(|script| script.mark_already_started()); + } + + fn complete_script(&mut self, node: TrustedNodeAddress) { + let node: Root = unsafe { JS::from_trusted_node_address(node).root() }; + let script: Option> = HTMLScriptElementCast::to_ref(*node); + script.map(|script| script.prepare()); + } +} + +// The url from msg_load_data is ignored here +pub fn parse_html(page: &Page, + document: JSRef, + input: HTMLInput, + resource_task: ResourceTask, + msg_load_data: Option) { + let (base_url, load_response) = match input { + InputUrl(ref url) => { + // Wait for the LoadResponse so that the parser knows the final URL. + let (input_chan, input_port) = channel(); + let mut load_data = LoadData::new(url.clone()); + msg_load_data.map(|m| { + load_data.headers = m.headers; + load_data.method = m.method; + load_data.data = m.data; + }); + resource_task.send(Load(load_data, input_chan)); + + let load_response = input_port.recv(); + + load_response.metadata.headers.as_ref().map(|headers| { + let header = headers.iter().find(|h| + h.header_name().as_slice().to_ascii_lower() == "last-modified".to_string() + ); + + match header { + Some(h) => document.set_last_modified( + parse_last_modified(h.header_value().as_slice())), + None => {}, + }; + }); + + let base_url = load_response.metadata.final_url.clone(); + + { + // Store the final URL before we start parsing, so that DOM routines + // (e.g. HTMLImageElement::update_image) can resolve relative URLs + // correctly. + *page.mut_url() = Some((base_url.clone(), true)); + } + + (Some(base_url), Some(load_response)) + }, + InputString(_) => { + match *page.url() { + Some((ref page_url, _)) => (Some(page_url.clone()), None), + None => (None, None), + } + }, + }; + + let parser = ServoHTMLParser::new(base_url.clone(), document).root(); + let parser: JSRef = *parser; + + task_state::enter(IN_HTML_PARSER); + + match input { + InputString(s) => { + parser.parse_chunk(s); + } + InputUrl(url) => { + let load_response = load_response.unwrap(); + match load_response.metadata.content_type { + Some((ref t, _)) if t.as_slice().eq_ignore_ascii_case("image") => { + let page = format!("", base_url.as_ref().unwrap().serialize()); + parser.parse_chunk(page); + }, + _ => { + for msg in load_response.progress_port.iter() { + match msg { + Payload(data) => { + // FIXME: use Vec (html5ever #34) + let data = UTF_8.decode(data.as_slice(), DecodeReplace).unwrap(); + parser.parse_chunk(data); + } + Done(Err(err)) => { + panic!("Failed to load page URL {:s}, error: {:s}", url.serialize(), err); + } + Done(Ok(())) => break, + } + } + } + } + } + } + + parser.finish(); + + task_state::exit(IN_HTML_PARSER); + + debug!("finished parsing"); +} diff --git a/components/util/time.rs b/components/util/time.rs new file mode 100644 index 00000000000..ffc537b4db2 --- /dev/null +++ b/components/util/time.rs @@ -0,0 +1,297 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Timing functions. + +use collections::TreeMap; +use std::comm::{Sender, channel, Receiver}; +use std::f64; +use std::io::timer::sleep; +use std::iter::AdditiveIterator; +use std::time::duration::Duration; +use std_time::{Tm, precise_time_ns, strptime}; +use task::{spawn_named}; +use url::Url; + +// front-end representation of the profiler used to communicate with the profiler +#[deriving(Clone)] +pub struct TimeProfilerChan(pub Sender); + +impl TimeProfilerChan { + pub fn send(&self, msg: TimeProfilerMsg) { + let TimeProfilerChan(ref c) = *self; + c.send(msg); + } +} + +#[deriving(PartialEq, Clone, PartialOrd, Eq, Ord)] +pub struct TimerMetadata { + url: String, + iframe: bool, + incremental: bool, +} + +pub trait Formatable { + fn format(&self) -> String; +} + +impl Formatable for Option { + fn format(&self) -> String { + match self { + // TODO(cgaebel): Center-align in the format strings as soon as rustc supports it. + &Some(ref meta) => { + let url = meta.url.as_slice(); + let url = if url.len() > 30 { + url.slice_to(30) + } else { + url + }; + let incremental = if meta.incremental { " yes" } else { " no " }; + let iframe = if meta.iframe { " yes" } else { " no " }; + format!(" {:14} {:9} {:30}", incremental, iframe, url) + }, + &None => + format!(" {:14} {:9} {:30}", " N/A", " N/A", " N/A") + } + } +} + +#[deriving(Clone)] +pub enum TimeProfilerMsg { + /// Normal message used for reporting time + TimeMsg((TimeProfilerCategory, Option), f64), + /// Message used to force print the profiling metrics + PrintMsg, + /// Tells the profiler to shut down. + ExitMsg, +} + +#[repr(u32)] +#[deriving(PartialEq, Clone, PartialOrd, Eq, Ord)] +pub enum TimeProfilerCategory { + CompositingCategory, + LayoutPerformCategory, + LayoutStyleRecalcCategory, + LayoutRestyleDamagePropagation, + LayoutNonIncrementalReset, + LayoutSelectorMatchCategory, + LayoutTreeBuilderCategory, + LayoutDamagePropagateCategory, + LayoutMainCategory, + LayoutParallelWarmupCategory, + LayoutShapingCategory, + LayoutDispListBuildCategory, + PaintingPerTileCategory, + PaintingPrepBuffCategory, + PaintingCategory, +} + +impl Formatable for TimeProfilerCategory { + // some categories are subcategories of LayoutPerformCategory + // and should be printed to indicate this + fn format(&self) -> String { + let padding = match *self { + LayoutStyleRecalcCategory | + LayoutRestyleDamagePropagation | + LayoutNonIncrementalReset | + LayoutMainCategory | + LayoutDispListBuildCategory | + LayoutShapingCategory | + LayoutDamagePropagateCategory | + PaintingPerTileCategory | + PaintingPrepBuffCategory => "+ ", + LayoutParallelWarmupCategory | + LayoutSelectorMatchCategory | + LayoutTreeBuilderCategory => "| + ", + _ => "" + }; + let name = match *self { + CompositingCategory => "Compositing", + LayoutPerformCategory => "Layout", + LayoutStyleRecalcCategory => "Style Recalc", + LayoutRestyleDamagePropagation => "Restyle Damage Propagation", + LayoutNonIncrementalReset => "Non-incremental reset (temporary)", + LayoutSelectorMatchCategory => "Selector Matching", + LayoutTreeBuilderCategory => "Tree Building", + LayoutDamagePropagateCategory => "Damage Propagation", + LayoutMainCategory => "Primary Layout Pass", + LayoutParallelWarmupCategory => "Parallel Warmup", + LayoutShapingCategory => "Shaping", + LayoutDispListBuildCategory => "Display List Construction", + PaintingPerTileCategory => "Painting Per Tile", + PaintingPrepBuffCategory => "Buffer Prep", + PaintingCategory => "Painting", + }; + format!("{:s}{}", padding, name) + } +} + +type TimeProfilerBuckets = TreeMap<(TimeProfilerCategory, Option), Vec>; + +// back end of the profiler that handles data aggregation and performance metrics +pub struct TimeProfiler { + pub port: Receiver, + buckets: TimeProfilerBuckets, + pub last_msg: Option, +} + +impl TimeProfiler { + pub fn create(period: Option) -> TimeProfilerChan { + let (chan, port) = channel(); + match period { + Some(period) => { + let period = Duration::milliseconds((period * 1000f64) as i64); + let chan = chan.clone(); + spawn_named("Time profiler timer", proc() { + loop { + sleep(period); + if chan.send_opt(PrintMsg).is_err() { + break; + } + } + }); + // Spawn the time profiler. + spawn_named("Time profiler", proc() { + let mut profiler = TimeProfiler::new(port); + profiler.start(); + }); + } + None => { + // No-op to handle messages when the time profiler is inactive. + spawn_named("Time profiler", proc() { + loop { + match port.recv_opt() { + Err(_) | Ok(ExitMsg) => break, + _ => {} + } + } + }); + } + } + + TimeProfilerChan(chan) + } + + pub fn new(port: Receiver) -> TimeProfiler { + TimeProfiler { + port: port, + buckets: TreeMap::new(), + last_msg: None, + } + } + + pub fn start(&mut self) { + loop { + let msg = self.port.recv_opt(); + match msg { + Ok(msg) => { + if !self.handle_msg(msg) { + break + } + } + _ => break + } + } + } + + fn find_or_insert(&mut self, k: (TimeProfilerCategory, Option), t: f64) { + match self.buckets.get_mut(&k) { + None => {}, + Some(v) => { v.push(t); return; }, + } + + self.buckets.insert(k, vec!(t)); + } + + fn handle_msg(&mut self, msg: TimeProfilerMsg) -> bool { + match msg.clone() { + TimeMsg(k, t) => self.find_or_insert(k, t), + PrintMsg => match self.last_msg { + // only print if more data has arrived since the last printout + Some(TimeMsg(..)) => self.print_buckets(), + _ => () + }, + ExitMsg => return false, + }; + self.last_msg = Some(msg); + true + } + + fn print_buckets(&mut self) { + println!("{:35s} {:14} {:9} {:30} {:15s} {:15s} {:-15s} {:-15s} {:-15s}", + "_category_", "_incremental?_", "_iframe?_", + " _url_", " _mean (ms)_", " _median (ms)_", + " _min (ms)_", " _max (ms)_", " _events_"); + for (&(ref category, ref meta), ref mut data) in self.buckets.iter_mut() { + data.sort_by(|a, b| { + if a < b { + Less + } else { + Greater + } + }); + let data_len = data.len(); + if data_len > 0 { + let (mean, median, min, max) = + (data.iter().map(|&x|x).sum() / (data_len as f64), + data.as_slice()[data_len / 2], + data.iter().fold(f64::INFINITY, |a, &b| a.min(b)), + data.iter().fold(-f64::INFINITY, |a, &b| a.max(b))); + println!("{:-35s}{} {:15.4f} {:15.4f} {:15.4f} {:15.4f} {:15u}", + category.format(), meta.format(), mean, median, min, max, data_len); + } + } + println!(""); + } +} + + +pub fn profile(category: TimeProfilerCategory, + // url, iframe?, first reflow? + meta: Option<(&Url, bool, bool)>, + time_profiler_chan: TimeProfilerChan, + callback: || -> T) + -> T { + let start_time = precise_time_ns(); + let val = callback(); + let end_time = precise_time_ns(); + let ms = (end_time - start_time) as f64 / 1000000f64; + let meta = meta.map(|(url, iframe, first_reflow)| + TimerMetadata { + url: url.serialize(), + iframe: iframe, + incremental: !first_reflow, + }); + time_profiler_chan.send(TimeMsg((category, meta), ms)); + return val; +} + +pub fn time(msg: &str, callback: || -> T) -> T{ + let start_time = precise_time_ns(); + let val = callback(); + let end_time = precise_time_ns(); + let ms = (end_time - start_time) as f64 / 1000000f64; + if ms >= 5f64 { + debug!("{:s} took {} ms", msg, ms); + } + return val; +} + +// Parses an RFC 2616 compliant date/time string +pub fn parse_http_timestamp(timestamp: &str) -> Option { + // RFC 822, updated by RFC 1123 + match strptime(timestamp, "%a, %d %b %Y %T %Z") { + Ok(t) => return Some(t), + Err(_) => () + } + + // RFC 850, obsoleted by RFC 1036 + match strptime(timestamp, "%A, %d-%b-%y %T %Z") { + Ok(t) => return Some(t), + Err(_) => () + } + + // ANSI C's asctime() format + strptime(timestamp, "%c").ok() +} diff --git a/tests/content/harness.js b/tests/content/harness.js new file mode 100644 index 00000000000..452c72fa67d --- /dev/null +++ b/tests/content/harness.js @@ -0,0 +1,106 @@ +function _oneline(x) { + var i = x.indexOf("\n"); + return (i == -1) ? x : (x.slice(0, i) + "..."); +} + +var _expectations = 0; +var _tests = 0; +function expect(num) { + _expectations = num; +} + +function _fail(s, m) { + _tests++; + // string split to avoid problems with tests that end up printing the value of window._fail. + window.alert(_oneline("TEST-UNEXPECTED" + "-FAIL | " + s + ": " + m)); +} + +function _pass(s, m) { + _tests++; + window.alert(_oneline("TEST-PASS | " + s + ": " + m)); +} + +function _printer(opstr, op) { + return function (a, b, msg) { + var f = op(a,b) ? _pass : _fail; + if (!msg) msg = ""; + f(a + " " + opstr + " " + b, msg); + }; +} + +var is = _printer("===", function (a,b) { return a === b; }); +var is_not = _printer("!==", function (a,b) { return a !== b; }); +var is_a = _printer("is a", function (a,b) { return a instanceof b; }); +var is_not_a = _printer("is not a", function (a,b) { return !(a instanceof b); }); +var is_in = _printer("is in", function (a,b) { return a in b; }); +var is_not_in = _printer("is not in", function (a,b) { return !(a in b); }); +var as_str_is = _printer("as string is", function (a,b) { return String(a) == b; }); +var lt = _printer("<", function (a,b) { return a < b; }); +var gt = _printer(">", function (a,b) { return a > b; }); +var leq = _printer("<=", function (a,b) { return a <= b; }); +var geq = _printer(">=", function (a,b) { return a >= b; }); +var starts_with = _printer("starts with", function (a,b) { return a.indexOf(b) == 0; }); + +function is_function(val, name) { + starts_with(String(val), "function " + name + "("); +} + +function should_throw(f) { + try { + f(); + _fail("operation should have thrown but did not"); + } catch (x) { + _pass("operation successfully threw an exception", x.toString()); + } +} + +function should_not_throw(f) { + try { + f(); + _pass("operation did not throw an exception"); + } catch (x) { + _fail("operation should have not thrown", x.toString()); + } +} + +function check_selector(elem, selector, matches) { + is(elem.matches(selector), matches); +} + +function check_disabled_selector(elem, disabled) { + check_selector(elem, ":disabled", disabled); + check_selector(elem, ":enabled", !disabled); +} + +var _test_complete = false; +var _test_timeout = 10000; //10 seconds +function finish() { + if (_test_complete) { + _fail('finish called multiple times'); + } + if (_expectations > _tests) { + _fail('expected ' + _expectations + ' tests, fullfilled ' + _tests); + } + _test_complete = true; + window.close(); +} + +function _test_timed_out() { + if (!_test_complete) { + _fail('test timed out (' + _test_timeout/1000 + 's)'); + finish(); + } +} + +setTimeout(_test_timed_out, _test_timeout); + +var _needs_finish = false; +function waitForExplicitFinish() { + _needs_finish = true; +} + +addEventListener('load', function() { + if (!_needs_finish) { + finish(); + } +}); diff --git a/tests/content/netharness.js b/tests/content/netharness.js new file mode 100644 index 00000000000..cc18fe1300d --- /dev/null +++ b/tests/content/netharness.js @@ -0,0 +1,25 @@ +function assert_requests_made(url, n) { + var x = new XMLHttpRequest(); + x.open('GET', 'stats?' + url, false); + x.send(); + is(parseInt(x.responseText), n, '# of requests for ' + url + ' should be ' + n); +} + +function reset_stats() { + var x = new XMLHttpRequest(); + x.open('POST', 'reset', false); + x.send(); + is(x.status, 200, 'resetting stats should succeed'); +} + +function fetch(url, headers) { + var x = new XMLHttpRequest(); + x.open('GET', url, false); + if (headers) { + for (var i = 0; i < headers.length; i++) { + x.setRequestHeader(headers[i][0], headers[i][1]); + } + } + x.send(); + is(x.status, 200, 'fetching ' + url + ' should succeed '); +} diff --git a/tests/content/resources/helper.html b/tests/content/resources/helper.html new file mode 100644 index 00000000000..90531a4b3ed --- /dev/null +++ b/tests/content/resources/helper.html @@ -0,0 +1,2 @@ + + diff --git a/tests/content/resources/helper_must_revalidate.html b/tests/content/resources/helper_must_revalidate.html new file mode 100644 index 00000000000..90531a4b3ed --- /dev/null +++ b/tests/content/resources/helper_must_revalidate.html @@ -0,0 +1,2 @@ + + diff --git a/tests/content/resources/helper_must_revalidate.html^headers b/tests/content/resources/helper_must_revalidate.html^headers new file mode 100644 index 00000000000..5f4c23137e1 --- /dev/null +++ b/tests/content/resources/helper_must_revalidate.html^headers @@ -0,0 +1,2 @@ +200 +Cache-Control: must-revalidate \ No newline at end of file diff --git a/tests/content/resources/helper_nocache.html b/tests/content/resources/helper_nocache.html new file mode 100644 index 00000000000..90531a4b3ed --- /dev/null +++ b/tests/content/resources/helper_nocache.html @@ -0,0 +1,2 @@ + + diff --git a/tests/content/resources/helper_nocache.html^headers b/tests/content/resources/helper_nocache.html^headers new file mode 100644 index 00000000000..e510c1a6f9a --- /dev/null +++ b/tests/content/resources/helper_nocache.html^headers @@ -0,0 +1,2 @@ +200 +Cache-Control: no-cache \ No newline at end of file diff --git a/tests/content/test_cached_headers_differ.html b/tests/content/test_cached_headers_differ.html new file mode 100644 index 00000000000..ba0e005a8c9 --- /dev/null +++ b/tests/content/test_cached_headers_differ.html @@ -0,0 +1,14 @@ + + + + + + + + + diff --git a/tests/content/test_cached_request.html b/tests/content/test_cached_request.html new file mode 100644 index 00000000000..978e783f220 --- /dev/null +++ b/tests/content/test_cached_request.html @@ -0,0 +1,14 @@ + + + + + + + + + diff --git a/tests/content/test_document_url.html b/tests/content/test_document_url.html new file mode 100644 index 00000000000..99b2a602b36 --- /dev/null +++ b/tests/content/test_document_url.html @@ -0,0 +1,30 @@ + + + + + + + + + + diff --git a/tests/content/test_nocache.html b/tests/content/test_nocache.html new file mode 100644 index 00000000000..d360841b5e2 --- /dev/null +++ b/tests/content/test_nocache.html @@ -0,0 +1,14 @@ + + + + + + + + + diff --git a/tests/content/test_revalidate.html b/tests/content/test_revalidate.html new file mode 100644 index 00000000000..1caa1562b4a --- /dev/null +++ b/tests/content/test_revalidate.html @@ -0,0 +1,14 @@ + + + + + + + + + diff --git a/tests/contenttest.rs b/tests/contenttest.rs new file mode 100644 index 00000000000..95e017ee768 --- /dev/null +++ b/tests/contenttest.rs @@ -0,0 +1,194 @@ +// Copyright 2013 The Servo Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![deny(unused_imports)] +#![deny(unused_variables)] + +extern crate getopts; +extern crate regex; +extern crate test; + +use test::{AutoColor, TestOpts, run_tests_console, TestDesc, TestDescAndFn, DynTestFn, DynTestName}; +use getopts::{getopts, reqopt}; +use std::comm::channel; +use std::from_str::FromStr; +use std::{os, str}; +use std::io::fs; +use std::io::Reader; +use std::io::process::{Command, Ignored, CreatePipe, InheritFd, ExitStatus}; +use std::task; +use regex::Regex; + +#[deriving(Clone)] +struct Config { + source_dir: String, + filter: Option +} + +fn main() { + let args = os::args(); + let config = parse_config(args.into_iter().collect()); + let opts = test_options(&config); + let tests = find_tests(&config); + match run_tests_console(&opts, tests) { + Ok(false) => os::set_exit_status(1), // tests failed + Err(_) => os::set_exit_status(2), // I/O-related failure + _ => (), + } +} + +enum ServerMsg { + IsAlive(Sender), + Exit, +} + +fn run_http_server(source_dir: String) -> (Sender, u16) { + let (tx, rx) = channel(); + let (port_sender, port_receiver) = channel(); + task::spawn(proc() { + let mut prc = Command::new("python") + .args(["../httpserver.py"]) + .stdin(Ignored) + .stdout(CreatePipe(false, true)) + .stderr(Ignored) + .cwd(&Path::new(source_dir)) + .spawn() + .ok() + .expect("Unable to spawn server."); + + let mut bytes = vec!(); + loop { + let byte = prc.stdout.as_mut().unwrap().read_byte().unwrap(); + if byte == '\n' as u8 { + break; + } else { + bytes.push(byte); + } + } + + let mut words = str::from_utf8(bytes.as_slice()).unwrap().split(' '); + let port = FromStr::from_str(words.last().unwrap()).unwrap(); + port_sender.send(port); + + loop { + match rx.recv() { + IsAlive(reply) => reply.send(prc.signal(0).is_ok()), + Exit => { + let _ = prc.signal_exit(); + break; + } + } + } + }); + (tx, port_receiver.recv()) +} + +fn parse_config(args: Vec) -> Config { + let args = args.tail(); + let opts = vec!(reqopt("s", "source-dir", "source-dir", "source-dir")); + let matches = match getopts(args, opts.as_slice()) { + Ok(m) => m, + Err(f) => panic!(format!("{}", f)) + }; + + Config { + source_dir: matches.opt_str("source-dir").unwrap(), + filter: matches.free.as_slice().head().map(|s| Regex::new(s.as_slice()).unwrap()) + } +} + +fn test_options(config: &Config) -> TestOpts { + TestOpts { + filter: config.filter.clone(), + run_ignored: false, + run_tests: true, + run_benchmarks: false, + ratchet_metrics: None, + ratchet_noise_percent: None, + save_metrics: None, + test_shard: None, + logfile: None, + nocapture: false, + color: AutoColor + } +} + +fn find_tests(config: &Config) -> Vec { + let files_res = fs::readdir(&Path::new(config.source_dir.clone())); + let mut files = match files_res { + Ok(files) => files, + _ => panic!("Error reading directory."), + }; + files.retain(|file| file.extension_str() == Some("html") ); + return files.iter().map(|file| make_test(format!("{}", file.display()), + config.source_dir.clone())).collect(); +} + +fn make_test(file: String, source_dir: String) -> TestDescAndFn { + TestDescAndFn { + desc: TestDesc { + name: DynTestName(file.clone()), + ignore: false, + should_fail: false + }, + testfn: DynTestFn(proc() { run_test(file, source_dir) }) + } +} + +fn run_test(file: String, source_dir: String) { + let (server, port) = run_http_server(source_dir); + + let path = os::make_absolute(&Path::new(file)); + // FIXME (#1094): not the right way to transform a path + let infile = format!("http://localhost:{}/{}", port, path.filename_display()); + let stdout = CreatePipe(false, true); + let stderr = InheritFd(2); + let args = ["-z", "-f", infile.as_slice()]; + + let (tx, rx) = channel(); + server.send(IsAlive(tx)); + assert!(rx.recv(), "HTTP server must be running."); + + let mut prc = match Command::new("target/servo") + .args(args) + .stdin(Ignored) + .stdout(stdout) + .stderr(stderr) + .spawn() + { + Ok(p) => p, + _ => panic!("Unable to spawn process."), + }; + let mut output = Vec::new(); + loop { + let byte = prc.stdout.as_mut().unwrap().read_byte(); + match byte { + Ok(byte) => { + print!("{}", byte as char); + output.push(byte); + } + _ => break + } + } + + server.send(Exit); + + let out = str::from_utf8(output.as_slice()); + let lines: Vec<&str> = out.unwrap().split('\n').collect(); + for &line in lines.iter() { + if line.contains("TEST-UNEXPECTED-FAIL") { + panic!(line.to_string()); + } + } + + let retval = prc.wait(); + if retval != Ok(ExitStatus(0)) { + panic!("Servo exited with non-zero status {}", retval); + } +} diff --git a/tests/httpserver.py b/tests/httpserver.py new file mode 100644 index 00000000000..689d29bef91 --- /dev/null +++ b/tests/httpserver.py @@ -0,0 +1,115 @@ +from SimpleHTTPServer import SimpleHTTPRequestHandler +import SocketServer +import os +import sys +from collections import defaultdict + +PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 0 + +requests = defaultdict(int) + +class CountingRequestHandler(SimpleHTTPRequestHandler): + def __init__(self, req, client_addr, server): + SimpleHTTPRequestHandler.__init__(self, req, client_addr, server) + + def do_POST(self): + global requests + parts = self.path.split('/') + + if parts[1] == 'reset': + requests = defaultdict(int) + self.send_response(200) + self.send_header('Content-Type', 'text/plain') + self.send_header('Content-Length', 0) + self.end_headers() + self.wfile.write('') + return + + def do_GET(self): + global requests + parts = self.path.split('?') + if parts[0] == '/stats': + self.send_response(200) + self.send_header('Content-Type', 'text/plain') + if len(parts) > 1: + body = str(requests['/' + parts[1]]) + else: + body = '' + for key, value in requests.iteritems(): + body += key + ': ' + str(value) + '\n' + self.send_header('Content-Length', len(body)) + self.end_headers() + self.wfile.write(body) + return + + header_list = [] + status = None + + path = self.translate_path(self.path) + headers = path + '^headers' + + if os.path.isfile(headers): + try: + h = open(headers, 'rb') + except IOError: + self.send_error(404, "Header file not found") + return + + header_lines = h.readlines() + status = int(header_lines[0]) + for header in header_lines[1:]: + parts = map(lambda x: x.strip(), header.split(':')) + header_list += [parts] + + if self.headers.get('If-Modified-Since'): + self.send_response(304) + self.end_headers() + return + + if not status or status == 200: + requests[self.path] += 1 + + if status or header_list: + ctype = self.guess_type(path) + try: + # Always read in binary mode. Opening files in text mode may cause + # newline translations, making the actual size of the content + # transmitted *less* than the content-length! + f = open(path, 'rb') + except IOError: + self.send_error(404, "File not found") + return + + try: + self.send_response(status or 200) + self.send_header("Content-type", ctype) + fs = os.fstat(f.fileno()) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + + for header in header_list: + self.send_header(header[0], header[1]) + + self.end_headers() + + try: + self.copyfile(f, self.wfile) + finally: + f.close() + except: + f.close() + raise + else: + SimpleHTTPRequestHandler.do_GET(self) + +class MyTCPServer(SocketServer.TCPServer): + request_queue_size = 2000 + allow_reuse_address = True + +httpd = MyTCPServer(("", PORT), CountingRequestHandler) +if not PORT: + ip, PORT = httpd.server_address + +print "serving at port", PORT +sys.stdout.flush() +httpd.serve_forever() From d196f3de621770ecf38a6349bc8cd00c0aa7aa07 Mon Sep 17 00:00:00 2001 From: Gregory Terzian Date: Sun, 12 Nov 2017 14:11:50 +0800 Subject: [PATCH 2/7] copy headers from script request to net request --- components/script/dom/headers.rs | 6 + components/script/dom/request.rs | 3 + .../fetch/api/basic/accept-header.any.js.ini | 17 -- .../api/cors/cors-preflight-star.any.js.ini | 6 - .../fetch/api/cors/cors-preflight.any.js.ini | 18 -- .../cors/cors-redirect-preflight.any.js.ini | 185 ------------------ .../authentication-basic.any.js.ini | 23 --- 7 files changed, 9 insertions(+), 249 deletions(-) delete mode 100644 tests/wpt/metadata/fetch/api/basic/accept-header.any.js.ini delete mode 100644 tests/wpt/metadata/fetch/api/cors/cors-redirect-preflight.any.js.ini delete mode 100644 tests/wpt/metadata/fetch/api/credentials/authentication-basic.any.js.ini diff --git a/components/script/dom/headers.rs b/components/script/dom/headers.rs index 5d152ab93bb..b351e2b7f13 100644 --- a/components/script/dom/headers.rs +++ b/components/script/dom/headers.rs @@ -234,6 +234,12 @@ impl Headers { *self.header_list.borrow_mut() = hyper_headers; } + pub fn get_headers_list(&self) -> HyperHeaders { + let mut headers = HyperHeaders::new(); + headers.extend(self.header_list.borrow_mut().iter()); + headers + } + // https://fetch.spec.whatwg.org/#concept-header-extract-mime-type pub fn extract_mime_type(&self) -> Vec { self.header_list.borrow().get_raw("content-type").map_or(vec![], |v| v[0].clone()) diff --git a/components/script/dom/request.rs b/components/script/dom/request.rs index f0b28f59b69..fcd396c1176 100644 --- a/components/script/dom/request.rs +++ b/components/script/dom/request.rs @@ -339,6 +339,9 @@ impl Request { _ => {}, } + // Copy the headers list onto the headers of net_traits::Request + r.request.borrow_mut().headers = r.Headers().get_headers_list(); + // Step 32 let mut input_body = if let RequestInfo::Request(ref input_request) = input { let input_request_request = input_request.request.borrow(); diff --git a/tests/wpt/metadata/fetch/api/basic/accept-header.any.js.ini b/tests/wpt/metadata/fetch/api/basic/accept-header.any.js.ini deleted file mode 100644 index fba2dac22d3..00000000000 --- a/tests/wpt/metadata/fetch/api/basic/accept-header.any.js.ini +++ /dev/null @@ -1,17 +0,0 @@ -[accept-header.any.html] - type: testharness - [Request through fetch should have 'accept' header with value 'custom/*'] - expected: FAIL - - [Request through fetch should have 'accept-language' header with value 'bzh'] - expected: FAIL - - -[accept-header.any.worker.html] - type: testharness - [Request through fetch should have 'accept' header with value 'custom/*'] - expected: FAIL - - [Request through fetch should have 'accept-language' header with value 'bzh'] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini b/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini index 385bc215a33..b99585d72cb 100644 --- a/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini +++ b/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini @@ -6,12 +6,6 @@ [CORS that succeeds with credentials: false; method: OK (allowed: *); header: X-Test,1 (allowed: *)] expected: FAIL - [CORS that fails with credentials: true; method: GET (allowed: get); header: X-Test,1 (allowed: *)] - expected: FAIL - - [CORS that fails with credentials: true; method: GET (allowed: *); header: X-Test,1 (allowed: *)] - expected: FAIL - [CORS that succeeds with credentials: true; method: PUT (allowed: PUT); header: (allowed: *)] expected: FAIL diff --git a/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini b/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini index 6729fb6e75e..5f6b56e0621 100644 --- a/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini +++ b/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini @@ -6,18 +6,9 @@ [CORS [NEW\], server allows] expected: FAIL - [CORS [GET\] [x-test-header: allowed\], server allows] - expected: FAIL - - [CORS [GET\] [x-test-header: refused\], server refuses] - expected: FAIL - [CORS [GET\] [several headers\], server allows] expected: FAIL - [CORS [GET\] [several headers\], server refuses] - expected: FAIL - [CORS [PUT\] [several headers\], server allows] expected: FAIL @@ -30,18 +21,9 @@ [CORS [NEW\], server allows] expected: FAIL - [CORS [GET\] [x-test-header: allowed\], server allows] - expected: FAIL - - [CORS [GET\] [x-test-header: refused\], server refuses] - expected: FAIL - [CORS [GET\] [several headers\], server allows] expected: FAIL - [CORS [GET\] [several headers\], server refuses] - expected: FAIL - [CORS [PUT\] [several headers\], server allows] expected: FAIL diff --git a/tests/wpt/metadata/fetch/api/cors/cors-redirect-preflight.any.js.ini b/tests/wpt/metadata/fetch/api/cors/cors-redirect-preflight.any.js.ini deleted file mode 100644 index 7b17b5619f6..00000000000 --- a/tests/wpt/metadata/fetch/api/cors/cors-redirect-preflight.any.js.ini +++ /dev/null @@ -1,185 +0,0 @@ -[cors-redirect-preflight.any.html] - type: testharness - [Redirect 301: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 301: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 301: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 301: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 301: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 301: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 302: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 302: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 302: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 302: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 302: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 302: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 303: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 303: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 303: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 303: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 303: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 303: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 307: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 307: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 307: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 307: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 307: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 307: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 308: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 308: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 308: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 308: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 308: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 308: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - -[cors-redirect-preflight.any.worker.html] - type: testharness - [Redirect 301: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 301: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 301: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 301: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 301: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 301: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 302: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 302: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 302: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 302: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 302: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 302: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 303: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 303: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 303: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 303: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 303: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 303: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 307: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 307: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 307: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 307: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 307: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 307: cors to another cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 308: same origin to cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 308: same origin to cors (preflight after redirection failure case)] - expected: FAIL - - [Redirect 308: cors to same origin (preflight after redirection success case)] - expected: FAIL - - [Redirect 308: cors to same origin (preflight after redirection failure case)] - expected: FAIL - - [Redirect 308: cors to another cors (preflight after redirection success case)] - expected: FAIL - - [Redirect 308: cors to another cors (preflight after redirection failure case)] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/credentials/authentication-basic.any.js.ini b/tests/wpt/metadata/fetch/api/credentials/authentication-basic.any.js.ini deleted file mode 100644 index 55038e25b7a..00000000000 --- a/tests/wpt/metadata/fetch/api/credentials/authentication-basic.any.js.ini +++ /dev/null @@ -1,23 +0,0 @@ -[authentication-basic.any.html] - type: testharness - [User-added Authorization header with include mode] - expected: FAIL - - [User-added Authorization header with same-origin mode] - expected: FAIL - - [User-added Authorization header with omit mode] - expected: FAIL - - -[authentication-basic.any.worker.html] - type: testharness - [User-added Authorization header with include mode] - expected: FAIL - - [User-added Authorization header with same-origin mode] - expected: FAIL - - [User-added Authorization header with omit mode] - expected: FAIL - From f36a4fb6d72e4b1637660a695cd0e3db028dc544 Mon Sep 17 00:00:00 2001 From: Gregory Terzian Date: Tue, 14 Nov 2017 23:59:00 +0800 Subject: [PATCH 3/7] pass cache_mode from request to request init --- components/script/fetch.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/components/script/fetch.rs b/components/script/fetch.rs index fe201f0105c..1085a215b03 100644 --- a/components/script/fetch.rs +++ b/components/script/fetch.rs @@ -60,6 +60,7 @@ fn request_init_from_request(request: NetTraitsRequest) -> NetTraitsRequestInit referrer_policy: request.referrer_policy, pipeline_id: request.pipeline_id, redirect_mode: request.redirect_mode, + cache_mode: request.cache_mode, ..NetTraitsRequestInit::default() } } From 2799b4eac9d903929246bda9d83cdaf9543f31d9 Mon Sep 17 00:00:00 2001 From: Gregory Terzian Date: Thu, 3 Aug 2017 12:17:29 +0200 Subject: [PATCH 4/7] more http cache work --- components/net/http_cache.rs | 1019 ++++++++++------- components/net/http_loader.rs | 89 +- components/net/resource_task.rs | 291 ----- components/net/resource_thread.rs | 3 + components/script/parse/html.rs | 268 ----- components/util/time.rs | 297 ----- tests/content/harness.js | 106 -- tests/content/netharness.js | 25 - tests/content/resources/helper.html | 2 - .../resources/helper_must_revalidate.html | 2 - .../helper_must_revalidate.html^headers | 2 - tests/content/resources/helper_nocache.html | 2 - .../resources/helper_nocache.html^headers | 2 - tests/content/test_cached_headers_differ.html | 14 - tests/content/test_cached_request.html | 14 - tests/content/test_document_url.html | 30 - tests/content/test_nocache.html | 14 - tests/content/test_revalidate.html | 14 - tests/contenttest.rs | 194 ---- tests/httpserver.py | 115 -- tests/wpt/metadata/MANIFEST.json | 8 +- tests/wpt/metadata/cors/304.htm.ini | 14 - ...request-cache-default-conditional.html.ini | 3 - .../request/request-cache-default.html.ini | 11 - .../request-cache-force-cache.html.ini | 29 - .../request-cache-only-if-cached.html.ini | 65 -- .../api/request/request-cache-reload.html.ini | 20 - .../fetch/http-cache/304-update.html.ini | 14 - .../fetch/http-cache/cc-request.html.ini | 6 - .../fetch/http-cache/freshness.html.ini | 20 - .../fetch/http-cache/heuristic.html.ini | 29 - .../fetch/http-cache/invalidate.html.ini | 9 - .../fetch/http-cache/partial.html.ini | 8 - .../metadata/fetch/http-cache/status.html.ini | 41 - .../metadata/fetch/http-cache/vary.html.ini | 17 - .../fetch/http-cache/cc-request.html | 3 +- .../fetch/http-cache/heuristic.html | 4 +- .../fetch/http-cache/partial.html | 103 +- .../fetch/http-cache/vary.html | 55 +- 39 files changed, 798 insertions(+), 2164 deletions(-) delete mode 100644 components/net/resource_task.rs delete mode 100644 components/script/parse/html.rs delete mode 100644 components/util/time.rs delete mode 100644 tests/content/harness.js delete mode 100644 tests/content/netharness.js delete mode 100644 tests/content/resources/helper.html delete mode 100644 tests/content/resources/helper_must_revalidate.html delete mode 100644 tests/content/resources/helper_must_revalidate.html^headers delete mode 100644 tests/content/resources/helper_nocache.html delete mode 100644 tests/content/resources/helper_nocache.html^headers delete mode 100644 tests/content/test_cached_headers_differ.html delete mode 100644 tests/content/test_cached_request.html delete mode 100644 tests/content/test_document_url.html delete mode 100644 tests/content/test_nocache.html delete mode 100644 tests/content/test_revalidate.html delete mode 100644 tests/contenttest.rs delete mode 100644 tests/httpserver.py delete mode 100644 tests/wpt/metadata/cors/304.htm.ini delete mode 100644 tests/wpt/metadata/fetch/api/request/request-cache-default-conditional.html.ini delete mode 100644 tests/wpt/metadata/fetch/api/request/request-cache-default.html.ini delete mode 100644 tests/wpt/metadata/fetch/api/request/request-cache-force-cache.html.ini delete mode 100644 tests/wpt/metadata/fetch/api/request/request-cache-only-if-cached.html.ini delete mode 100644 tests/wpt/metadata/fetch/api/request/request-cache-reload.html.ini delete mode 100644 tests/wpt/metadata/fetch/http-cache/304-update.html.ini delete mode 100644 tests/wpt/metadata/fetch/http-cache/freshness.html.ini delete mode 100644 tests/wpt/metadata/fetch/http-cache/heuristic.html.ini delete mode 100644 tests/wpt/metadata/fetch/http-cache/status.html.ini delete mode 100644 tests/wpt/metadata/fetch/http-cache/vary.html.ini diff --git a/components/net/http_cache.rs b/components/net/http_cache.rs index cfb2c8acc01..764fa3651df 100644 --- a/components/net/http_cache.rs +++ b/components/net/http_cache.rs @@ -4,497 +4,650 @@ #![deny(missing_docs)] -//! A non-validating memory cache that only evicts expired entries and grows -//! without bound. Implements the logic specified in http://tools.ietf.org/html/rfc7234 -//! and http://tools.ietf.org/html/rfc7232. - -use http_loader::send_error_direct; -use resource_task::{Metadata, ProgressMsg, LoadResponse, LoadData, Payload, Done, start_sending_opt}; - -use servo_util::time::parse_http_timestamp; - -use http::headers::etag::EntityTag; -use http::headers::HeaderEnum; -use http::headers::response::HeaderCollection as ResponseHeaderCollection; -use http::method::Get; -use http::status::Ok as StatusOk; +//! A memory cache implementing the logic specified in http://tools.ietf.org/html/rfc7234 +//! and . +use fetch::methods::DoneChannel; +use http_loader::is_redirect_status; +use hyper::header; +use hyper::header::ContentType; +use hyper::header::Headers; +use hyper::method::Method; +use hyper::status::StatusCode; +use hyper_serde::Serde; +use net_traits::{Metadata, FetchMetadata}; +use net_traits::request::Request; +use net_traits::response::{HttpsState, Response, ResponseBody}; +use servo_url::ServoUrl; use std::collections::HashMap; -use std::comm::Sender; -use std::iter::Map; -use std::mem; -use std::num::{Bounded, FromStrRadix}; -use std::str::CharSplits; +use std::str; use std::sync::{Arc, Mutex}; -use std::time::duration::{MAX, Duration}; use time; -use time::{Tm, Timespec}; -use url::Url; +use time::{Duration, Tm}; -//TODO: Store an Arc> instead? -//TODO: Cache HEAD requests -//TODO: Doom responses with network errors -//TODO: Send Err responses for doomed entries -//TODO: Enable forced eviction of a request instead of retrieving the cached response -//TODO: Doom incomplete entries -//TODO: Cache-Control: must-revalidate -//TODO: Last-Modified -//TODO: Range requests -//TODO: Revalidation rules for query strings -//TODO: Vary header /// The key used to differentiate requests in the cache. -#[deriving(Clone, Hash, PartialEq, Eq)] +#[derive(Clone, Eq, Hash, PartialEq)] pub struct CacheKey { - url: Url, - request_headers: Vec<(String, String)>, + url: ServoUrl } impl CacheKey { - fn new(load_data: LoadData) -> CacheKey { + fn new(request: Request) -> CacheKey { CacheKey { - url: load_data.url.clone(), - request_headers: load_data.headers - .iter() - .map(|header| (header.header_name(), header.header_value())) - .collect(), + url: request.url().clone() + } + } + + fn from_servo_url(servo_url: &ServoUrl) -> CacheKey { + CacheKey { + url: servo_url.clone() } } /// Retrieve the URL associated with this key - pub fn url(&self) -> Url { + pub fn url(&self) -> ServoUrl { self.url.clone() } } -/// The list of consumers waiting on this requests's response. -enum PendingConsumers { - /// Consumers awaiting the initial response metadata - AwaitingHeaders(Vec>), - /// Consumers awaiting the remaining response body. Incomplete body stored as Vec. - AwaitingBody(Metadata, Vec, Vec>), -} - -/// An unfulfilled request representing both the consumers waiting for the initial -/// metadata and the subsequent response body. If doomed, the entry will be removed -/// after the final payload. -struct PendingResource { - consumers: PendingConsumers, - expires: Duration, - last_validated: Tm, - doomed: bool, -} - /// A complete cached resource. +#[derive(Clone)] struct CachedResource { - metadata: Metadata, - body: Vec, + metadata: CachedMetadata, + request_headers: Arc>, + body: Arc>, + https_state: HttpsState, + status: Option, + raw_status: Option<(u16, Vec)>, + url_list: Vec, expires: Duration, - last_validated: Tm, - revalidating_consumers: Vec>, + last_validated: Tm } -/// A memory cache that tracks incomplete and complete responses, differentiated by -/// the initial request. -pub struct MemoryCache { - /// Complete cached responses. - complete_entries: HashMap, - /// Incomplete cached responses. - pending_entries: HashMap, - /// The time at which this cache was created for use by expiry checks. - base_time: Timespec, +/// Metadata about a loaded resource, such as is obtained from HTTP headers. +#[derive(Clone)] +struct CachedMetadata { + /// Final URL after redirects. + pub final_url: ServoUrl, + /// MIME type / subtype. + pub content_type: Option>, + /// Character set. + pub charset: Option, + /// Headers + pub headers: Arc>, + /// HTTP Status + pub status: Option<(u16, Vec)> } -/// Abstraction over the concept of a single target for HTTP response messages. -pub enum ResourceResponseTarget { - /// A response is being streamed into the cache. - CachedPendingResource(CacheKey, Arc>), - /// A response is being streamed directly to a consumer and skipping the cache. - UncachedPendingResource(Sender), +/// Wrapper around a cached response, including information on re-validation needs +pub struct CachedResponse { + /// The response constructed from the cached resource + pub response: Response, + /// The revalidation flag for the stored response + pub needs_validation: bool } -/// Abstraction over the concept of a single target for HTTP response payload messages. -pub enum ResourceProgressTarget { - /// A response is being streamed into the cache. - CachedInProgressResource(CacheKey, Arc>), - /// A response is being streamed directly to a consumer and skipping the cache. - UncachedInProgressResource(Sender), +/// A memory cache. +pub struct HttpCache { + /// cached responses. + entries: HashMap>, } -/// The result of matching a request against an HTTP cache. -pub enum CacheOperationResult { - /// The request cannot be cached for a given reason. - Uncacheable(&'static str), - /// The request is in the cache and the response data is forthcoming. - CachedContentPending, - /// The request is not present in the cache but will be cached with the given key. - NewCacheEntry(CacheKey), - /// The request is in the cache but requires revalidation. - Revalidate(CacheKey, RevalidationMethod), -} - -/// The means by which to revalidate stale cached content -pub enum RevalidationMethod { - /// The result of a stored Last-Modified or Expires header - ExpiryDate(Tm), - /// The result of a stored Etag header - Etag(EntityTag), -} - -/// Tokenize a header value. -fn split_header(header: &str) -> Map<&str, &str, CharSplits> { - header.split(',') - .map(|v| v.trim()) -} - -/// Match any header value token. -fn any_token_matches(header: &str, tokens: &[&str]) -> bool { - split_header(header).any(|token| tokens.iter().any(|&s| s == token)) -} /// Determine if a given response is cacheable based on the initial metadata received. -/// Based on http://tools.ietf.org/html/rfc7234#section-5 +/// Based on fn response_is_cacheable(metadata: &Metadata) -> bool { - if metadata.status != StatusOk { + // TODO: if we determine that this cache should be considered shared: + // 1. check for absence of private response directive + // 2. check for absence of the Authorization header field. + let mut is_cacheable = false; + let headers = metadata.headers.as_ref().unwrap(); + if headers.has::() || + headers.has::() || + headers.has::() { + is_cacheable = true; + } + if let Some(&header::CacheControl(ref directive)) = headers.get::() { + for directive in directive.iter() { + match *directive { + header::CacheDirective::NoStore => return false, + header::CacheDirective::Public | header::CacheDirective::SMaxAge(_) + | header::CacheDirective::MaxAge(_) | header::CacheDirective::NoCache => is_cacheable = true, + _ => {}, + } + } + } + if let Some(&header::Pragma::NoCache) = headers.get::() { return false; } - - if metadata.headers.is_none() { - return true; - } - - let headers = metadata.headers.as_ref().unwrap(); - match headers.cache_control { - Some(ref cache_control) => { - if any_token_matches(cache_control[], &["no-cache", "no-store", "max-age=0"]) { - return false; - } - } - None => () - } - - match headers.pragma { - Some(ref pragma) => { - if any_token_matches(pragma[], &["no-cache"]) { - return false; - } - } - None => () - } - - return true; + is_cacheable } -/// Determine the expiry date of the given response headers. -/// Returns a far-future date if the response does not expire. -fn get_response_expiry_from_headers(headers: &ResponseHeaderCollection) -> Duration { - headers.cache_control.as_ref().and_then(|cache_control| { - for token in split_header(cache_control[]) { - let mut parts = token.split('='); - if parts.next() == Some("max-age") { - return parts.next() - .and_then(|val| FromStrRadix::from_str_radix(val, 10)) - .map(|secs| Duration::seconds(secs)); +/// Calculating Age +/// +fn calculate_response_age(response: &Response) -> Duration { + // TODO: follow the spec more closely (Date headers, request/response lag, ...) + if let Some(secs) = response.headers.get_raw("Age") { + let seconds_string = String::from_utf8_lossy(&secs[0]); + if let Ok(secs) = seconds_string.parse::() { + return Duration::seconds(secs); + } + } + Duration::seconds(0i64) +} + +/// Determine the expiry date from relevant headers, +/// or uses a heuristic if none are present. +fn get_response_expiry(response: &Response) -> Duration { + // Calculating Freshness Lifetime + let age = calculate_response_age(&response); + if let Some(&header::CacheControl(ref directives)) = response.headers.get::() { + let has_no_cache_directive = directives.iter().any(|directive| { + header::CacheDirective::NoCache == *directive + }); + if has_no_cache_directive { + // Requires validation on first use. + return Duration::seconds(0i64); + } else { + for directive in directives { + match *directive { + header::CacheDirective::SMaxAge(secs) | header::CacheDirective::MaxAge(secs) => { + let max_age = Duration::seconds(secs as i64); + if max_age < age { + return Duration::seconds(0i64); + } + return max_age - age; + }, + _ => (), + } + } + } + } + if let Some(&header::Expires(header::HttpDate(t))) = response.headers.get::() { + // store the period of time from now until expiry + let desired = t.to_timespec(); + let current = time::now().to_timespec(); + if desired > current { + return desired - current; + } else { + return Duration::seconds(0i64); + } + } else { + if let Some(_) = response.headers.get_raw("Expires") { + // Malformed Expires header, shouldn't be used to construct a valid response. + return Duration::seconds(0i64); + } + } + // Calculating Heuristic Freshness + // + if let Some((ref code, _)) = response.raw_status { + // + // Since presently we do not generate a Warning header field with a 113 warn-code, + // 24 hours minus response age is the max for heuristic calculation. + let max_heuristic = Duration::hours(24) - age; + let heuristic_freshness = if let Some(&header::LastModified(header::HttpDate(t))) = + // If the response has a Last-Modified header field, + // caches are encouraged to use a heuristic expiration value + // that is no more than some fraction of the interval since that time. + response.headers.get::() { + let last_modified = t.to_timespec(); + let current = time::now().to_timespec(); + // A typical setting of this fraction might be 10%. + let raw_heuristic_calc = (current - last_modified) / 10; + let result = if raw_heuristic_calc < max_heuristic { + raw_heuristic_calc + } else { + max_heuristic + }; + result + } else { + max_heuristic + }; + match *code { + 200 | 203 | 204 | 206 | 300 | 301 | 404 | 405 | 410 | 414 | 501 => { + // Status codes that are cacheable by default + return heuristic_freshness + }, + _ => { + // Other status codes can only use heuristic freshness if the public cache directive is present. + if let Some(&header::CacheControl(ref directives)) = response.headers.get::() { + let has_public_directive = directives.iter().any(|directive| { + header::CacheDirective::Public == *directive + }); + if has_public_directive { + return heuristic_freshness; + } + } + }, + } + } + // Requires validation upon first use as default. + Duration::seconds(0i64) +} + +/// Request Cache-Control Directives +/// +fn get_expiry_adjustment_from_request_headers(request: &Request, expires: Duration) -> Duration { + let directive_data = match request.headers.get_raw("cache-control") { + Some(data) => data, + None => return expires, + }; + let directives_string = String::from_utf8_lossy(&directive_data[0]); + for directive in directives_string.split(",") { + let mut directive_info = directive.split("="); + match (directive_info.next(), directive_info.next()) { + (Some("max-stale"), Some(sec_str)) => { + if let Ok(secs) = sec_str.parse::() { + return expires + Duration::seconds(secs); + } + }, + (Some("max-age"), Some(sec_str)) => { + if let Ok(secs) = sec_str.parse::() { + let max_age = Duration::seconds(secs); + if expires > max_age { + return Duration::min_value(); + } + return expires - max_age; + } + }, + (Some("min-fresh"), Some(sec_str)) => { + if let Ok(secs) = sec_str.parse::() { + let min_fresh = Duration::seconds(secs); + if expires < min_fresh { + return Duration::min_value(); + } + return expires - min_fresh; + } + }, + (Some("no-cache"), _) | (Some("no-store"), _) => return Duration::min_value(), + _ => {} + } + } + expires +} + +/// Create a CachedResponse from a request and a CachedResource. +fn create_cached_response(request: &Request, cached_resource: &CachedResource, cached_headers: &Headers) + -> CachedResponse { + let mut response = Response::new(cached_resource.metadata.final_url.clone()); + response.headers = cached_headers.clone(); + response.body = cached_resource.body.clone(); + response.status = cached_resource.status.clone(); + response.raw_status = cached_resource.raw_status.clone(); + response.url_list = cached_resource.url_list.clone(); + response.https_state = cached_resource.https_state.clone(); + response.referrer = request.referrer.to_url().cloned(); + response.referrer_policy = request.referrer_policy.clone(); + let expires = cached_resource.expires; + let adjusted_expires = get_expiry_adjustment_from_request_headers(request, expires); + let now = Duration::seconds(time::now().to_timespec().sec); + let last_validated = Duration::seconds(cached_resource.last_validated.to_timespec().sec); + let time_since_validated = now - last_validated; + // TODO: take must-revalidate into account + // TODO: if this cache is to be considered shared, take proxy-revalidate into account + // + let has_expired = (adjusted_expires < time_since_validated) || + (adjusted_expires == time_since_validated); + CachedResponse { response: response, needs_validation: has_expired } +} + +/// Create a new resource, based on the bytes requested, and an existing resource, +/// with a status-code of 206. +fn create_resource_with_bytes_from_resource(bytes: &[u8], resource: &CachedResource) + -> CachedResource { + CachedResource { + metadata: resource.metadata.clone(), + request_headers: resource.request_headers.clone(), + body: Arc::new(Mutex::new(ResponseBody::Done(bytes.to_owned()))), + https_state: resource.https_state.clone(), + status: Some(StatusCode::PartialContent), + raw_status: Some((206, b"Partial Content".to_vec())), + url_list: resource.url_list.clone(), + expires: resource.expires.clone(), + last_validated: resource.last_validated.clone() + } +} + +/// Support for range requests . +fn handle_range_request(request: &Request, candidates: Vec<&CachedResource>, range_spec: &[header::ByteRangeSpec]) + -> Option { + let mut complete_cached_resources = candidates.iter().filter(|resource| { + match resource.raw_status { + Some((ref code, _)) => *code == 200, + None => false + } + }); + let partial_cached_resources = candidates.iter().filter(|resource| { + match resource.raw_status { + Some((ref code, _)) => *code == 206, + None => false + } + }); + match (range_spec.first().unwrap(), complete_cached_resources.next()) { + // TODO: take the full range spec into account. + // If we have a complete resource, take the request range from the body. + // When there isn't a complete resource available, we loop over cached partials, + // and see if any individual partial response can fulfill the current request for a bytes range. + // TODO: combine partials that in combination could satisfy the requested range? + // see . + // TODO: add support for complete and partial resources, + // whose body is in the ResponseBody::Receiving state. + (&header::ByteRangeSpec::FromTo(beginning, end), Some(ref complete_resource)) => { + if let ResponseBody::Done(ref body) = *complete_resource.body.lock().unwrap() { + let b = beginning as usize; + let e = end as usize + 1; + let requested = body.get(b..e); + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(bytes, complete_resource); + let cached_headers = new_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, &new_resource, &*cached_headers); + return Some(cached_response); + } + } + }, + (&header::ByteRangeSpec::FromTo(beginning, end), None) => { + for partial_resource in partial_cached_resources { + let headers = partial_resource.metadata.headers.lock().unwrap(); + let content_range = headers.get::(); + let (res_beginning, res_end) = match content_range { + Some(&header::ContentRange( + header::ContentRangeSpec::Bytes { + range: Some((res_beginning, res_end)), .. })) => (res_beginning, res_end), + _ => continue, + }; + if res_beginning - 1 < beginning && res_end + 1 > end { + let resource_body = &*partial_resource.body.lock().unwrap(); + let requested = match resource_body { + &ResponseBody::Done(ref body) => { + let b = beginning as usize - res_beginning as usize; + let e = end as usize - res_beginning as usize + 1; + body.get(b..e) + }, + _ => continue, + }; + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(&bytes, partial_resource); + let cached_response = create_cached_response(request, &new_resource, &*headers); + return Some(cached_response); + } + } + } + }, + (&header::ByteRangeSpec::AllFrom(beginning), Some(ref complete_resource)) => { + if let ResponseBody::Done(ref body) = *complete_resource.body.lock().unwrap() { + let b = beginning as usize; + let requested = body.get(b..); + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(bytes, complete_resource); + let cached_headers = new_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, &new_resource, &*cached_headers); + return Some(cached_response); + } + } + }, + (&header::ByteRangeSpec::AllFrom(beginning), None) => { + for partial_resource in partial_cached_resources { + let headers = partial_resource.metadata.headers.lock().unwrap(); + let content_range = headers.get::(); + let (res_beginning, res_end, total) = match content_range { + Some(&header::ContentRange( + header::ContentRangeSpec::Bytes { + range: Some((res_beginning, res_end)), + instance_length: Some(total) })) => (res_beginning, res_end, total), + _ => continue, + }; + if res_beginning < beginning && res_end == total - 1 { + let resource_body = &*partial_resource.body.lock().unwrap(); + let requested = match resource_body { + &ResponseBody::Done(ref body) => { + let from_byte = beginning as usize - res_beginning as usize; + body.get(from_byte..) + }, + _ => continue, + }; + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(&bytes, partial_resource); + let cached_response = create_cached_response(request, &new_resource, &*headers); + return Some(cached_response); + } + } + } + }, + (&header::ByteRangeSpec::Last(offset), Some(ref complete_resource)) => { + if let ResponseBody::Done(ref body) = *complete_resource.body.lock().unwrap() { + let from_byte = body.len() - offset as usize; + let requested = body.get(from_byte..); + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(bytes, complete_resource); + let cached_headers = new_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, &new_resource, &*cached_headers); + return Some(cached_response); + } + } + }, + (&header::ByteRangeSpec::Last(offset), None) => { + for partial_resource in partial_cached_resources { + let headers = partial_resource.metadata.headers.lock().unwrap(); + let content_range = headers.get::(); + let (res_beginning, res_end, total) = match content_range { + Some(&header::ContentRange( + header::ContentRangeSpec::Bytes { + range: Some((res_beginning, res_end)), + instance_length: Some(total) })) => (res_beginning, res_end, total), + _ => continue, + }; + if (total - res_beginning) > (offset - 1 ) && (total - res_end) < offset + 1 { + let resource_body = &*partial_resource.body.lock().unwrap(); + let requested = match resource_body { + &ResponseBody::Done(ref body) => { + let from_byte = body.len() - offset as usize; + body.get(from_byte..) + }, + _ => continue, + }; + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(&bytes, partial_resource); + let cached_response = create_cached_response(request, &new_resource, &*headers); + return Some(cached_response); + } + } + } + } + } + None +} + + +impl HttpCache { + /// Create a new memory cache instance. + pub fn new() -> HttpCache { + HttpCache { + entries: HashMap::new() + } + } + + /// Constructing Responses from Caches. + /// + pub fn construct_response(&self, request: &Request) -> Option { + // TODO: generate warning headers as appropriate + if request.method != Method::Get { + // Only Get requests are cached, avoid a url based match for others. + return None; + } + let entry_key = CacheKey::new(request.clone()); + let resources = match self.entries.get(&entry_key) { + Some(ref resources) => resources.clone(), + None => return None, + }; + let mut candidates = vec![]; + for cached_resource in resources.iter() { + let mut can_be_constructed = true; + let cached_headers = cached_resource.metadata.headers.lock().unwrap(); + let original_request_headers = cached_resource.request_headers.lock().unwrap(); + if let Some(vary_data) = cached_headers.get_raw("Vary") { + // Calculating Secondary Keys with Vary + let vary_data_string = String::from_utf8_lossy(&vary_data[0]); + let vary_values = vary_data_string.split(",").map(|val| val.trim()); + for vary_val in vary_values { + // For every header name found in the Vary header of the stored response. + if vary_val == "*" { + // A Vary header field-value of "*" always fails to match. + can_be_constructed = false; + break; + } + match request.headers.get_raw(vary_val) { + Some(header_data) => { + // If the header is present in the request. + let request_header_data_string = String::from_utf8_lossy(&header_data[0]); + if let Some(original_header_data) = original_request_headers.get_raw(vary_val) { + // Check that the value of the nominated header field, + // in the original request, matches the value in the current request. + let original_request_header_data_string = + String::from_utf8_lossy(&original_header_data[0]); + if original_request_header_data_string != request_header_data_string { + can_be_constructed = false; + break; + } + } + }, + None => { + // If a header field is absent from a request, + // it can only match a stored response if those headers, + // were also absent in the original request. + can_be_constructed = original_request_headers.get_raw(vary_val).is_none(); + }, + } + if !can_be_constructed { + break; + } + } + } + if can_be_constructed { + candidates.push(cached_resource); + } + } + // Support for range requests + if let Some(&header::Range::Bytes(ref range_spec)) = request.headers.get::() { + return handle_range_request(request, candidates, &range_spec); + } else { + // Not a Range request. + if let Some(ref cached_resource) = candidates.first() { + // Returning the first response that can be constructed + // TODO: select the most appropriate one, using a known mechanism from a selecting header field, + // or using the Date header to return the most recent one. + let cached_headers = cached_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, cached_resource, &*cached_headers); + return Some(cached_response); } } None - }).or_else(|| { - headers.expires.as_ref().and_then(|expires| { - parse_http_timestamp(expires[]).map(|t| { - // store the period of time from now until expiry - let desired = t.to_timespec(); - let current = time::now().to_timespec(); - if desired > current { - desired - current - } else { - Bounded::min_value() - } - }) - }) - }).unwrap_or(Bounded::max_value()) -} - -/// Determine the expiry date of the given response. -/// Returns a far-future date if this response does not expire. -fn get_response_expiry(metadata: &Metadata) -> Duration { - metadata.headers.as_ref().map(|headers| { - get_response_expiry_from_headers(headers) - }).unwrap_or(Bounded::max_value()) -} - -impl MemoryCache { - /// Create a new memory cache instance. - pub fn new() -> MemoryCache { - MemoryCache { - complete_entries: HashMap::new(), - pending_entries: HashMap::new(), - base_time: time::now().to_timespec(), - } } - /// Process a revalidation that returned new content for an expired entry. - pub fn process_revalidation_failed(&mut self, key: &CacheKey) { - debug!("recreating entry for {} (cache entry expired)", key.url); - let resource = self.complete_entries.remove(key).unwrap(); - self.add_pending_cache_entry(key.clone(), resource.revalidating_consumers); - } - - /// Mark an incomplete cached request as doomed. Any waiting consumers will immediately - /// receive an error message or a final body payload. The cache entry is immediately - /// removed. - pub fn doom_request(&mut self, key: &CacheKey, err: String) { - debug!("dooming entry for {} ({})", key.url, err); - - assert!(!self.complete_entries.contains_key(key)); - - let resource = self.pending_entries.remove(key).unwrap(); - match resource.consumers { - AwaitingHeaders(ref consumers) => { - for consumer in consumers.iter() { - send_error_direct(key.url.clone(), err.clone(), consumer.clone()); - } + /// Freshening Stored Responses upon Validation. + /// + pub fn refresh(&mut self, request: &Request, response: Response, done_chan: &mut DoneChannel) -> Option { + assert!(response.status == Some(StatusCode::NotModified)); + let entry_key = CacheKey::new(request.clone()); + if let Some(cached_resources) = self.entries.get_mut(&entry_key) { + for cached_resource in cached_resources.iter_mut() { + let mut stored_headers = cached_resource.metadata.headers.lock().unwrap(); + // Received a response with 304 status code, in response to a request that matches a cached resource. + // 1. update the headers of the cached resource. + // 2. return a response, constructed from the cached resource. + stored_headers.extend(response.headers.iter()); + let mut constructed_response = Response::new(cached_resource.metadata.final_url.clone()); + constructed_response.headers = stored_headers.clone(); + constructed_response.body = cached_resource.body.clone(); + constructed_response.status = cached_resource.status.clone(); + constructed_response.https_state = cached_resource.https_state.clone(); + constructed_response.referrer = request.referrer.to_url().cloned(); + constructed_response.referrer_policy = request.referrer_policy.clone(); + constructed_response.raw_status = cached_resource.raw_status.clone(); + constructed_response.url_list = cached_resource.url_list.clone(); + // done_chan will have been set to Some by http_network_fetch, + // set it back to None since the response returned here replaces the 304 one from the network. + *done_chan = None; + cached_resource.expires = get_response_expiry(&constructed_response); + return Some(constructed_response); } - AwaitingBody(_, _, ref consumers) => { - for consumer in consumers.iter() { - let _ = consumer.send_opt(Done(Ok(()))); - } + } + None + } + + fn invalidate_for_url(&mut self, url: &ServoUrl) { + let entry_key = CacheKey::from_servo_url(url); + if let Some(cached_resources) = self.entries.get_mut(&entry_key) { + for cached_resource in cached_resources.iter_mut() { + cached_resource.expires = Duration::seconds(0i64); } } } - /// Handle a 304 response to a revalidation request. Updates the cached response - /// metadata with any new expiration data. - pub fn process_not_modified(&mut self, key: &CacheKey, headers: &ResponseHeaderCollection) { - debug!("updating metadata for {}", key.url); - let resource = self.complete_entries.get_mut(key).unwrap(); - resource.expires = get_response_expiry_from_headers(headers); - - for consumer in mem::replace(&mut resource.revalidating_consumers, vec!()).into_iter() { - MemoryCache::send_complete_resource(resource, consumer); + /// Invalidation. + /// + pub fn invalidate(&mut self, request: &Request, response: &Response) { + if let Some(&header::Location(ref location)) = response.headers.get::() { + if let Ok(url) = request.current_url().join(location) { + self.invalidate_for_url(&url); + } } + // TODO: update hyper to use typed getter. + if let Some(url_data) = response.headers.get_raw("Content-Location") { + if let Ok(content_location) = str::from_utf8(&url_data[0]) { + if let Ok(url) = request.current_url().join(content_location) { + self.invalidate_for_url(&url); + } + } + } + self.invalidate_for_url(&request.url()); } - /// Handle the initial response metadata for an incomplete cached request. - /// If the response should not be cached, the entry will be doomed and any - /// subsequent requests will not see the cached request. All waiting consumers - /// will see the new metadata. - pub fn process_metadata(&mut self, key: &CacheKey, metadata: Metadata) { - debug!("storing metadata for {}", key.url); - let resource = self.pending_entries.get_mut(key).unwrap(); - let chans: Vec>; - match resource.consumers { - AwaitingHeaders(ref consumers) => { - chans = consumers.iter() - .map(|chan| start_sending_opt(chan.clone(), metadata.clone())) - .take_while(|chan| chan.is_ok()) - .map(|chan| chan.unwrap()) - .collect(); + /// Storing Responses in Caches. + /// + pub fn store(&mut self, request: &Request, response: &Response) { + if let Some(status) = response.status { + // Not caching redirects, for simplicity, not per the spec. + if is_redirect_status(status) { + return } - AwaitingBody(..) => panic!("obtained headers for {} but awaiting body?", key.url) } - + if request.method != Method::Get { + // Only Get requests are cached. + return + } + let entry_key = CacheKey::new(request.clone()); + let metadata = match response.metadata() { + Ok(FetchMetadata::Filtered { + filtered: _, + unsafe_: metadata }) | + Ok(FetchMetadata::Unfiltered(metadata)) => metadata, + _ => return, + }; if !response_is_cacheable(&metadata) { - resource.doomed = true; - } - - resource.expires = get_response_expiry(&metadata); - resource.last_validated = time::now(); - resource.consumers = AwaitingBody(metadata, vec!(), chans); - } - - /// Handle a repsonse body payload for an incomplete cached response. - /// All waiting consumers will see the new payload addition. - pub fn process_payload(&mut self, key: &CacheKey, payload: Vec) { - debug!("storing partial response for {}", key.url); - let resource = self.pending_entries.get_mut(key).unwrap(); - match resource.consumers { - AwaitingBody(_, ref mut body, ref consumers) => { - body.push_all(payload.as_slice()); - for consumer in consumers.iter() { - //FIXME: maybe remove consumer on failure to avoid extra clones? - let _ = consumer.send_opt(Payload(payload.clone())); - } - } - AwaitingHeaders(_) => panic!("obtained body for {} but awaiting headers?", key.url) - } - } - - /// Handle a response body final payload for an incomplete cached response. - /// All waiting consumers will see the new message. If the cache entry is - /// doomed, it will not be transferred to the set of complete cache entries. - pub fn process_done(&mut self, key: &CacheKey) { - debug!("finished fetching {}", key.url); - let resource = self.pending_entries.remove(key).unwrap(); - match resource.consumers { - AwaitingHeaders(_) => panic!("saw Done for {} but awaiting headers?", key.url), - AwaitingBody(_, _, ref consumers) => { - for consumer in consumers.iter() { - let _ = consumer.send_opt(Done(Ok(()))); - } - } - } - - if resource.doomed { - debug!("completing dooming of {}", key.url); return; } - - let (metadata, body) = match resource.consumers { - AwaitingBody(metadata, body, _) => (metadata, body), - _ => panic!("expected consumer list awaiting bodies"), + let expiry = get_response_expiry(&response); + let cacheable_metadata = CachedMetadata { + final_url: metadata.final_url, + content_type: metadata.content_type, + charset: metadata.charset, + status: metadata.status, + headers: Arc::new(Mutex::new(response.headers.clone())) }; - - let complete = CachedResource { - metadata: metadata, - body: body, - expires: resource.expires, - last_validated: resource.last_validated, - revalidating_consumers: vec!(), + let entry_resource = CachedResource { + metadata: cacheable_metadata, + request_headers: Arc::new(Mutex::new(request.headers.clone())), + body: response.body.clone(), + https_state: response.https_state.clone(), + status: response.status.clone(), + raw_status: response.raw_status.clone(), + url_list: response.url_list.clone(), + expires: expiry, + last_validated: time::now() }; - self.complete_entries.insert(key.clone(), complete); + let entry = self.entries.entry(entry_key).or_insert(vec![]); + entry.push(entry_resource); } - /// Match a new request against the set of incomplete and complete cached requests. - /// If the request matches an existing, non-doomed entry, any existing response data will - /// be synchronously streamed to the consumer. If the request does not match but can be - /// cached, a new cache entry will be created and the request will be responsible for - /// notifying the cache of the subsequent HTTP response. If the request does not match - /// and cannot be cached, the request is responsible for handling its own response and - /// consumer. - pub fn process_pending_request(&mut self, load_data: &LoadData, start_chan: Sender) - -> CacheOperationResult { - fn revalidate(resource: &mut CachedResource, - key: &CacheKey, - start_chan: Sender, - method: RevalidationMethod) -> CacheOperationResult { - // Ensure that at most one revalidation is taking place at a time for a - // cached resource. - resource.revalidating_consumers.push(start_chan); - if resource.revalidating_consumers.len() > 1 { - CachedContentPending - } else { - Revalidate(key.clone(), method) - } - } - - if load_data.method != Get { - return Uncacheable("Only GET requests can be cached."); - } - - let key = CacheKey::new(load_data.clone()); - match self.complete_entries.get_mut(&key) { - Some(resource) => { - if self.base_time + resource.expires < time::now().to_timespec() { - debug!("entry for {} has expired", key.url()); - let expiry = time::at(self.base_time + resource.expires); - return revalidate(resource, &key, start_chan, ExpiryDate(expiry)); - } - - let must_revalidate = resource.metadata.headers.as_ref().and_then(|headers| { - headers.cache_control.as_ref().map(|header| { - any_token_matches(header[], &["must-revalidate"]) - }) - }).unwrap_or(false); - - if must_revalidate { - debug!("entry for {} must be revalidated", key.url()); - let last_validated = resource.last_validated; - return revalidate(resource, &key, start_chan, ExpiryDate(last_validated)); - } - - let etag = resource.metadata.headers.as_ref().and_then(|headers| headers.etag.clone()); - match etag { - Some(etag) => { - debug!("entry for {} has an Etag", key.url()); - return revalidate(resource, &key, start_chan, Etag(etag.clone())); - } - None => () - } - - //TODO: Revalidate once per session for response with no explicit expiry - } - - None => () - } - - if self.complete_entries.contains_key(&key) { - self.send_complete_entry(key, start_chan); - return CachedContentPending; - } - - let new_entry = match self.pending_entries.get(&key) { - Some(resource) if resource.doomed => return Uncacheable("Cache entry already doomed"), - Some(_) => false, - None => true, - }; - - if new_entry { - self.add_pending_cache_entry(key.clone(), vec!(start_chan)); - NewCacheEntry(key) - } else { - self.send_partial_entry(key, start_chan); - CachedContentPending - } - } - - /// Add a new pending request to the set of incomplete cache entries. - fn add_pending_cache_entry(&mut self, key: CacheKey, consumers: Vec>) { - let resource = PendingResource { - consumers: AwaitingHeaders(consumers), - expires: MAX, - last_validated: time::now(), - doomed: false, - }; - debug!("creating cache entry for {}", key.url); - self.pending_entries.insert(key, resource); - } - - /// Synchronously send the entire cached response body to the given consumer. - fn send_complete_resource(resource: &CachedResource, start_chan: Sender) { - let progress_chan = start_sending_opt(start_chan, resource.metadata.clone()); - match progress_chan { - Ok(chan) => { - let _ = chan.send_opt(Payload(resource.body.clone())); - let _ = chan.send_opt(Done(Ok(()))); - } - Err(_) => () - } - } - - /// Synchronously send the entire cached response body to the given consumer. - fn send_complete_entry(&self, key: CacheKey, start_chan: Sender) { - debug!("returning full cache body for {}", key.url); - let resource = self.complete_entries.get(&key).unwrap(); - MemoryCache::send_complete_resource(resource, start_chan) - } - - /// Synchronously send all partial stored response data for a cached request to the - /// given consumer. - fn send_partial_entry(&mut self, key: CacheKey, start_chan: Sender) { - debug!("returning partial cache data for {}", key.url); - - let resource = self.pending_entries.get_mut(&key).unwrap(); - - match resource.consumers { - AwaitingHeaders(ref mut consumers) => { - consumers.push(start_chan); - } - AwaitingBody(ref metadata, ref body, ref mut consumers) => { - debug!("headers available for {}", key.url); - let progress_chan = start_sending_opt(start_chan, metadata.clone()); - match progress_chan { - Ok(chan) => { - consumers.push(chan.clone()); - - if !body.is_empty() { - debug!("partial body available for {}", key.url); - let _ = chan.send_opt(Payload(body.clone())); - } - } - - Err(_) => () - } - } - } - } } diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index 05fb6dbf1ee..0b7187bb8a6 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -13,6 +13,7 @@ use fetch::methods::{Data, DoneChannel, FetchContext, Target}; use fetch::methods::{is_cors_safelisted_request_header, is_cors_safelisted_method, main_fetch}; use flate2::read::{DeflateDecoder, GzDecoder}; use hsts::HstsList; +use http_cache::HttpCache; use hyper::Error as HttpError; use hyper::LanguageTag; use hyper::client::{Pool, Request as HyperRequest, Response as HyperResponse}; @@ -22,7 +23,7 @@ use hyper::header::{AccessControlMaxAge, AccessControlRequestHeaders}; use hyper::header::{AccessControlRequestMethod, AcceptEncoding, AcceptLanguage}; use hyper::header::{Authorization, Basic, CacheControl, CacheDirective}; use hyper::header::{ContentEncoding, ContentLength, Encoding, Header, Headers}; -use hyper::header::{Host, Origin as HyperOrigin, IfMatch, IfRange}; +use hyper::header::{Host, HttpDate, Origin as HyperOrigin, IfMatch, IfRange}; use hyper::header::{IfUnmodifiedSince, IfModifiedSince, IfNoneMatch, Location}; use hyper::header::{Pragma, Quality, QualityItem, Referer, SetCookie}; use hyper::header::{UserAgent, q, qitem}; @@ -45,6 +46,7 @@ use std::io::{self, Read, Write}; use std::iter::FromIterator; use std::mem; use std::ops::Deref; +use std::str::FromStr; use std::sync::RwLock; use std::sync::mpsc::{channel, Sender}; use std::thread; @@ -69,6 +71,7 @@ fn read_block(reader: &mut R) -> Result { pub struct HttpState { pub hsts_list: RwLock, pub cookie_jar: RwLock, + pub http_cache: RwLock, pub auth_cache: RwLock, pub ssl_client: OpensslClient, pub connector: Pool, @@ -80,6 +83,7 @@ impl HttpState { hsts_list: RwLock::new(HstsList::new()), cookie_jar: RwLock::new(CookieStorage::new(150)), auth_cache: RwLock::new(AuthCache::new()), + http_cache: RwLock::new(HttpCache::new()), ssl_client: ssl_client.clone(), connector: create_http_connector(ssl_client), } @@ -893,34 +897,35 @@ fn http_network_or_cache_fetch(request: &mut Request, let mut revalidating_flag = false; // Step 21 - // TODO have a HTTP cache to check for a completed response - let complete_http_response_from_cache: Option = None; - if http_request.cache_mode != CacheMode::NoStore && - http_request.cache_mode != CacheMode::Reload && - complete_http_response_from_cache.is_some() { - // TODO Substep 1 and 2. Select a response from HTTP cache. + if let Ok(http_cache) = context.state.http_cache.read() { + if let Some(response_from_cache) = http_cache.construct_response(&http_request) { + let response_headers = response_from_cache.response.headers.clone(); + // Substep 1, 2, 3, 4 + let (cached_response, needs_revalidation) = match (http_request.cache_mode, &http_request.mode) { + (CacheMode::ForceCache, _) => (Some(response_from_cache.response), false), + (CacheMode::OnlyIfCached, &RequestMode::SameOrigin) => (Some(response_from_cache.response), false), + (CacheMode::OnlyIfCached, _) | (CacheMode::NoStore, _) | (CacheMode::Reload, _) => (None, false), + (_, _) => (Some(response_from_cache.response), response_from_cache.needs_validation) + }; + if needs_revalidation { + revalidating_flag = true; + // Substep 5 + // TODO: find out why the typed header getter return None from the headers of cached responses. + if let Some(date_slice) = response_headers.get_raw("Last-Modified") { + let date_string = String::from_utf8_lossy(&date_slice[0]); + if let Ok(http_date) = HttpDate::from_str(&date_string) { + http_request.headers.set(IfModifiedSince(http_date)); + } + } + if let Some(entity_tag) = + response_headers.get_raw("ETag") { + http_request.headers.set_raw("If-None-Match", entity_tag.to_vec()); - // Substep 3 - if let Some(ref response) = response { - revalidating_flag = response_needs_revalidation(&response); - }; - - // Substep 4 - if http_request.cache_mode == CacheMode::ForceCache || - http_request.cache_mode == CacheMode::OnlyIfCached { - // TODO pull response from HTTP cache - // response = http_request - } - - if revalidating_flag { - // Substep 5 - // TODO set If-None-Match and If-Modified-Since according to cached - // response headers. - } else { - // Substep 6 - // TODO pull response from HTTP cache - // response = http_request - // response.cache_state = CacheState::Local; + } + } else { + // Substep 6 + response = cached_response; + } } } @@ -931,26 +936,37 @@ fn http_network_or_cache_fetch(request: &mut Request, return Response::network_error( NetworkError::Internal("Couldn't find response in cache".into())) } + } + // More Step 22 + if response.is_none() { // Substep 2 let forward_response = http_network_fetch(http_request, credentials_flag, done_chan, context); // Substep 3 if let Some((200...399, _)) = forward_response.raw_status { if !http_request.method.safe() { - // TODO Invalidate HTTP cache response + if let Ok(mut http_cache) = context.state.http_cache.write() { + http_cache.invalidate(&http_request, &forward_response); + } } } // Substep 4 if revalidating_flag && forward_response.status.map_or(false, |s| s == StatusCode::NotModified) { - // TODO update forward_response headers with cached response headers + if let Ok(mut http_cache) = context.state.http_cache.write() { + response = http_cache.refresh(&http_request, forward_response.clone(), done_chan); + } } // Substep 5 if response.is_none() { + if http_request.cache_mode != CacheMode::NoStore { + // Subsubstep 2, doing it first to avoid a clone of forward_response. + if let Ok(mut http_cache) = context.state.http_cache.write() { + http_cache.store(&http_request, &forward_response); + } + } // Subsubstep 1 response = Some(forward_response); - // Subsubstep 2 - // TODO: store http_request and forward_response in cache } } @@ -1168,7 +1184,9 @@ fn http_network_fetch(request: &Request, // Step 14 if !response.is_network_error() && request.cache_mode != CacheMode::NoStore { - // TODO update response in the HTTP cache for request + if let Ok(mut http_cache) = context.state.http_cache.write() { + http_cache.store(&request, &response); + } } // TODO this step isn't possible yet @@ -1366,11 +1384,6 @@ fn is_no_store_cache(headers: &Headers) -> bool { headers.has::() } -fn response_needs_revalidation(_response: &Response) -> bool { - // TODO this function - false -} - /// pub fn is_redirect_status(status: StatusCode) -> bool { match status { diff --git a/components/net/resource_task.rs b/components/net/resource_task.rs deleted file mode 100644 index ec6d49f065a..00000000000 --- a/components/net/resource_task.rs +++ /dev/null @@ -1,291 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! A task that takes a URL and streams back the binary data. - -use about_loader; -use data_loader; -use file_loader; -use http_cache::MemoryCache; -use http_loader; -use sniffer_task; - -use std::comm::{channel, Receiver, Sender}; -use std::sync::{Arc, Mutex}; -use http::headers::content_type::MediaType; -use http::headers::response::HeaderCollection as ResponseHeaderCollection; -use http::headers::request::HeaderCollection as RequestHeaderCollection; -use http::method::{Method, Get}; -use url::Url; - -use http::status::Ok as StatusOk; -use http::status::Status; - -use servo_util::task::spawn_named; - -pub enum ControlMsg { - /// Request the data associated with a particular URL - Load(LoadData, Sender), - Exit -} - -#[deriving(Clone)] -pub struct LoadData { - pub url: Url, - pub method: Method, - pub headers: RequestHeaderCollection, - pub data: Option>, - pub cors: Option -} - -impl LoadData { - pub fn new(url: Url) -> LoadData { - LoadData { - url: url, - method: Get, - headers: RequestHeaderCollection::new(), - data: None, - cors: None - } - } -} - -#[deriving(Clone)] -pub struct ResourceCORSData { - /// CORS Preflight flag - pub preflight: bool, - /// Origin of CORS Request - pub origin: Url -} - -/// Metadata about a loaded resource, such as is obtained from HTTP headers. -#[deriving(Clone)] -pub struct Metadata { - /// Final URL after redirects. - pub final_url: Url, - - /// MIME type / subtype. - pub content_type: Option<(String, String)>, - - /// Character set. - pub charset: Option, - - /// Headers - pub headers: Option, - - /// HTTP Status - pub status: Status -} - -impl Metadata { - /// Metadata with defaults for everything optional. - pub fn default(url: Url) -> Metadata { - Metadata { - final_url: url, - content_type: None, - charset: None, - headers: None, - status: StatusOk // http://fetch.spec.whatwg.org/#concept-response-status-message - } - } - - /// Extract the parts of a MediaType that we care about. - pub fn set_content_type(&mut self, content_type: &Option) { - match *content_type { - None => (), - Some(MediaType { ref type_, - ref subtype, - ref parameters }) => { - self.content_type = Some((type_.clone(), subtype.clone())); - for &(ref k, ref v) in parameters.iter() { - if "charset" == k.as_slice() { - self.charset = Some(v.clone()); - } - } - } - } - } -} - -/// Message sent in response to `Load`. Contains metadata, and a port -/// for receiving the data. -/// -/// Even if loading fails immediately, we send one of these and the -/// progress_port will provide the error. -pub struct LoadResponse { - /// Metadata, such as from HTTP headers. - pub metadata: Metadata, - /// Port for reading data. - pub progress_port: Receiver, -} - -/// Messages sent in response to a `Load` message -#[deriving(PartialEq,Show)] -pub enum ProgressMsg { - /// Binary data - there may be multiple of these - Payload(Vec), - /// Indicates loading is complete, either successfully or not - Done(Result<(), String>) -} - -/// For use by loaders in responding to a Load message. -pub fn start_sending(start_chan: Sender, metadata: Metadata) -> Sender { - start_sending_opt(start_chan, metadata).ok().unwrap() -} - -/// For use by loaders in responding to a Load message. -pub fn start_sending_opt(start_chan: Sender, metadata: Metadata) -> Result, ()> { - let (progress_chan, progress_port) = channel(); - let result = start_chan.send_opt(LoadResponse { - metadata: metadata, - progress_port: progress_port, - }); - match result { - Ok(_) => Ok(progress_chan), - Err(_) => Err(()) - } -} - -/// Convenience function for synchronously loading a whole resource. -pub fn load_whole_resource(resource_task: &ResourceTask, url: Url) - -> Result<(Metadata, Vec), String> { - let (start_chan, start_port) = channel(); - resource_task.send(Load(LoadData::new(url), start_chan)); - let response = start_port.recv(); - - let mut buf = vec!(); - loop { - match response.progress_port.recv() { - Payload(data) => buf.push_all(data.as_slice()), - Done(Ok(())) => return Ok((response.metadata, buf)), - Done(Err(e)) => return Err(e) - } - } -} - -/// Handle to a resource task -pub type ResourceTask = Sender; - -/// Create a ResourceTask -pub fn new_resource_task(user_agent: Option) -> ResourceTask { - let (setup_chan, setup_port) = channel(); - spawn_named("ResourceManager", proc() { - ResourceManager::new(setup_port, user_agent).start(); - }); - setup_chan -} - -struct ResourceManager { - from_client: Receiver, - user_agent: Option, - memory_cache: Arc>, -} - -impl ResourceManager { - fn new(from_client: Receiver, user_agent: Option) -> ResourceManager { - ResourceManager { - from_client: from_client, - user_agent: user_agent, - memory_cache: Arc::new(Mutex::new(MemoryCache::new())), - } - } -} - - -impl ResourceManager { - fn start(&self) { - loop { - match self.from_client.recv() { - Load(load_data, start_chan) => { - self.load(load_data, start_chan) - } - Exit => { - break - } - } - } - } - - fn load(&self, load_data: LoadData, start_chan: Sender) { - let mut load_data = load_data; - load_data.headers.user_agent = self.user_agent.clone(); - - // Create new communication channel, create new sniffer task, - // send all the data to the new sniffer task with the send - // end of the pipe, receive all the data. - - let sniffer_task = sniffer_task::new_sniffer_task(start_chan.clone()); - - fn from_factory<'a>(factory: fn(LoadData, Sender)) - -> proc(LoadData, Sender): 'a { - proc(load_data: LoadData, start_chan: Sender) { - factory(load_data, start_chan) - } - } - - let loader = match load_data.url.scheme.as_slice() { - "file" => from_factory(file_loader::factory), - "http" | "https" => http_loader::factory(self.memory_cache.clone()), - "data" => from_factory(data_loader::factory), - "about" => from_factory(about_loader::factory), - _ => { - debug!("resource_task: no loader for scheme {:s}", load_data.url.scheme); - start_sending(start_chan, Metadata::default(load_data.url)) - .send(Done(Err("no loader for scheme".to_string()))); - return - } - }; - debug!("resource_task: loading url: {:s}", load_data.url.serialize()); - - loader(load_data, sniffer_task); - } -} - -/// Load a URL asynchronously and iterate over chunks of bytes from the response. -pub fn load_bytes_iter(resource_task: &ResourceTask, url: Url) -> (Metadata, ProgressMsgPortIterator) { - let (input_chan, input_port) = channel(); - resource_task.send(Load(LoadData::new(url), input_chan)); - - let response = input_port.recv(); - let iter = ProgressMsgPortIterator { progress_port: response.progress_port }; - (response.metadata, iter) -} - -/// Iterator that reads chunks of bytes from a ProgressMsg port -pub struct ProgressMsgPortIterator { - progress_port: Receiver -} - -impl Iterator> for ProgressMsgPortIterator { - fn next(&mut self) -> Option> { - match self.progress_port.recv() { - Payload(data) => Some(data), - Done(Ok(())) => None, - Done(Err(e)) => { - error!("error receiving bytes: {}", e); - None - } - } - } -} - -#[test] -fn test_exit() { - let resource_task = new_resource_task(None); - resource_task.send(Exit); -} - -#[test] -fn test_bad_scheme() { - let resource_task = new_resource_task(None); - let (start_chan, start) = channel(); - let url = Url::parse("bogus://whatever").unwrap(); - resource_task.send(Load(LoadData::new(url), start_chan)); - let response = start.recv(); - match response.progress_port.recv() { - Done(result) => { assert!(result.is_err()) } - _ => panic!("bleh") - } - resource_task.send(Exit); -} diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index 9e1a3c9eee7..22366c562cd 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -12,6 +12,7 @@ use fetch::cors_cache::CorsCache; use fetch::methods::{FetchContext, fetch}; use filemanager_thread::{FileManager, TFDProvider}; use hsts::HstsList; +use http_cache::HttpCache; use http_loader::{HttpState, http_redirect_fetch}; use hyper_serde::Serde; use ipc_channel::ipc::{self, IpcReceiver, IpcReceiverSet, IpcSender}; @@ -91,6 +92,7 @@ struct ResourceChannelManager { fn create_http_states(config_dir: Option<&Path>) -> (Arc, Arc) { let mut hsts_list = HstsList::from_servo_preload(); let mut auth_cache = AuthCache::new(); + let http_cache = HttpCache::new(); let mut cookie_jar = CookieStorage::new(150); if let Some(config_dir) = config_dir { read_json_from_file(&mut auth_cache, config_dir, "auth_cache.json"); @@ -109,6 +111,7 @@ fn create_http_states(config_dir: Option<&Path>) -> (Arc, Arc String { - parse_http_timestamp(timestamp).map(|t| { - t.to_local().strftime("%m/%d/%Y %H:%M:%S").unwrap() - }).unwrap_or(String::new()) -} - -trait SinkHelpers { - fn get_or_create(&self, child: NodeOrText) -> Temporary; -} - -impl SinkHelpers for servohtmlparser::Sink { - fn get_or_create(&self, child: NodeOrText) -> Temporary { - match child { - AppendNode(n) => Temporary::new(unsafe { JS::from_trusted_node_address(n) }), - AppendText(t) => { - let doc = self.document.root(); - let text = Text::new(t, *doc); - NodeCast::from_temporary(text) - } - } - } -} - -impl<'a> TreeSink for servohtmlparser::Sink { - fn get_document(&mut self) -> TrustedNodeAddress { - let doc = self.document.root(); - let node: JSRef = NodeCast::from_ref(*doc); - node.to_trusted_node_address() - } - - fn same_node(&self, x: TrustedNodeAddress, y: TrustedNodeAddress) -> bool { - x == y - } - - fn elem_name(&self, target: TrustedNodeAddress) -> QualName { - let node: Root = unsafe { JS::from_trusted_node_address(target).root() }; - let elem: JSRef = ElementCast::to_ref(*node) - .expect("tried to get name of non-Element in HTML parsing"); - QualName { - ns: elem.get_namespace().clone(), - local: elem.get_local_name().clone(), - } - } - - fn create_element(&mut self, name: QualName, attrs: Vec) - -> TrustedNodeAddress { - let doc = self.document.root(); - let elem = Element::create(name, None, *doc, ParserCreated).root(); - - for attr in attrs.into_iter() { - elem.set_attribute_from_parser(attr.name, attr.value, None); - } - - let node: JSRef = NodeCast::from_ref(*elem); - node.to_trusted_node_address() - } - - fn create_comment(&mut self, text: String) -> TrustedNodeAddress { - let doc = self.document.root(); - let comment = Comment::new(text, *doc); - let node: Root = NodeCast::from_temporary(comment).root(); - node.to_trusted_node_address() - } - - fn append_before_sibling(&mut self, - sibling: TrustedNodeAddress, - new_node: NodeOrText) -> Result<(), NodeOrText> { - // If there is no parent, return the node to the parser. - let sibling: Root = unsafe { JS::from_trusted_node_address(sibling).root() }; - let parent = match sibling.parent_node() { - Some(p) => p.root(), - None => return Err(new_node), - }; - - let child = self.get_or_create(new_node).root(); - assert!(parent.InsertBefore(*child, Some(*sibling)).is_ok()); - Ok(()) - } - - fn parse_error(&mut self, msg: MaybeOwned<'static>) { - debug!("Parse error: {:s}", msg); - } - - fn set_quirks_mode(&mut self, mode: QuirksMode) { - let doc = self.document.root(); - doc.set_quirks_mode(mode); - } - - fn append(&mut self, parent: TrustedNodeAddress, child: NodeOrText) { - let parent: Root = unsafe { JS::from_trusted_node_address(parent).root() }; - let child = self.get_or_create(child).root(); - - // FIXME(#3701): Use a simpler algorithm and merge adjacent text nodes - assert!(parent.AppendChild(*child).is_ok()); - } - - fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) { - let doc = self.document.root(); - let doc_node: JSRef = NodeCast::from_ref(*doc); - let doctype = DocumentType::new(name, Some(public_id), Some(system_id), *doc); - let node: Root = NodeCast::from_temporary(doctype).root(); - - assert!(doc_node.AppendChild(*node).is_ok()); - } - - fn add_attrs_if_missing(&mut self, target: TrustedNodeAddress, attrs: Vec) { - let node: Root = unsafe { JS::from_trusted_node_address(target).root() }; - let elem: JSRef = ElementCast::to_ref(*node) - .expect("tried to set attrs on non-Element in HTML parsing"); - for attr in attrs.into_iter() { - elem.set_attribute_from_parser(attr.name, attr.value, None); - } - } - - fn remove_from_parent(&mut self, _target: TrustedNodeAddress) { - error!("remove_from_parent not implemented!"); - } - - fn mark_script_already_started(&mut self, node: TrustedNodeAddress) { - let node: Root = unsafe { JS::from_trusted_node_address(node).root() }; - let script: Option> = HTMLScriptElementCast::to_ref(*node); - script.map(|script| script.mark_already_started()); - } - - fn complete_script(&mut self, node: TrustedNodeAddress) { - let node: Root = unsafe { JS::from_trusted_node_address(node).root() }; - let script: Option> = HTMLScriptElementCast::to_ref(*node); - script.map(|script| script.prepare()); - } -} - -// The url from msg_load_data is ignored here -pub fn parse_html(page: &Page, - document: JSRef, - input: HTMLInput, - resource_task: ResourceTask, - msg_load_data: Option) { - let (base_url, load_response) = match input { - InputUrl(ref url) => { - // Wait for the LoadResponse so that the parser knows the final URL. - let (input_chan, input_port) = channel(); - let mut load_data = LoadData::new(url.clone()); - msg_load_data.map(|m| { - load_data.headers = m.headers; - load_data.method = m.method; - load_data.data = m.data; - }); - resource_task.send(Load(load_data, input_chan)); - - let load_response = input_port.recv(); - - load_response.metadata.headers.as_ref().map(|headers| { - let header = headers.iter().find(|h| - h.header_name().as_slice().to_ascii_lower() == "last-modified".to_string() - ); - - match header { - Some(h) => document.set_last_modified( - parse_last_modified(h.header_value().as_slice())), - None => {}, - }; - }); - - let base_url = load_response.metadata.final_url.clone(); - - { - // Store the final URL before we start parsing, so that DOM routines - // (e.g. HTMLImageElement::update_image) can resolve relative URLs - // correctly. - *page.mut_url() = Some((base_url.clone(), true)); - } - - (Some(base_url), Some(load_response)) - }, - InputString(_) => { - match *page.url() { - Some((ref page_url, _)) => (Some(page_url.clone()), None), - None => (None, None), - } - }, - }; - - let parser = ServoHTMLParser::new(base_url.clone(), document).root(); - let parser: JSRef = *parser; - - task_state::enter(IN_HTML_PARSER); - - match input { - InputString(s) => { - parser.parse_chunk(s); - } - InputUrl(url) => { - let load_response = load_response.unwrap(); - match load_response.metadata.content_type { - Some((ref t, _)) if t.as_slice().eq_ignore_ascii_case("image") => { - let page = format!("", base_url.as_ref().unwrap().serialize()); - parser.parse_chunk(page); - }, - _ => { - for msg in load_response.progress_port.iter() { - match msg { - Payload(data) => { - // FIXME: use Vec (html5ever #34) - let data = UTF_8.decode(data.as_slice(), DecodeReplace).unwrap(); - parser.parse_chunk(data); - } - Done(Err(err)) => { - panic!("Failed to load page URL {:s}, error: {:s}", url.serialize(), err); - } - Done(Ok(())) => break, - } - } - } - } - } - } - - parser.finish(); - - task_state::exit(IN_HTML_PARSER); - - debug!("finished parsing"); -} diff --git a/components/util/time.rs b/components/util/time.rs deleted file mode 100644 index ffc537b4db2..00000000000 --- a/components/util/time.rs +++ /dev/null @@ -1,297 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! Timing functions. - -use collections::TreeMap; -use std::comm::{Sender, channel, Receiver}; -use std::f64; -use std::io::timer::sleep; -use std::iter::AdditiveIterator; -use std::time::duration::Duration; -use std_time::{Tm, precise_time_ns, strptime}; -use task::{spawn_named}; -use url::Url; - -// front-end representation of the profiler used to communicate with the profiler -#[deriving(Clone)] -pub struct TimeProfilerChan(pub Sender); - -impl TimeProfilerChan { - pub fn send(&self, msg: TimeProfilerMsg) { - let TimeProfilerChan(ref c) = *self; - c.send(msg); - } -} - -#[deriving(PartialEq, Clone, PartialOrd, Eq, Ord)] -pub struct TimerMetadata { - url: String, - iframe: bool, - incremental: bool, -} - -pub trait Formatable { - fn format(&self) -> String; -} - -impl Formatable for Option { - fn format(&self) -> String { - match self { - // TODO(cgaebel): Center-align in the format strings as soon as rustc supports it. - &Some(ref meta) => { - let url = meta.url.as_slice(); - let url = if url.len() > 30 { - url.slice_to(30) - } else { - url - }; - let incremental = if meta.incremental { " yes" } else { " no " }; - let iframe = if meta.iframe { " yes" } else { " no " }; - format!(" {:14} {:9} {:30}", incremental, iframe, url) - }, - &None => - format!(" {:14} {:9} {:30}", " N/A", " N/A", " N/A") - } - } -} - -#[deriving(Clone)] -pub enum TimeProfilerMsg { - /// Normal message used for reporting time - TimeMsg((TimeProfilerCategory, Option), f64), - /// Message used to force print the profiling metrics - PrintMsg, - /// Tells the profiler to shut down. - ExitMsg, -} - -#[repr(u32)] -#[deriving(PartialEq, Clone, PartialOrd, Eq, Ord)] -pub enum TimeProfilerCategory { - CompositingCategory, - LayoutPerformCategory, - LayoutStyleRecalcCategory, - LayoutRestyleDamagePropagation, - LayoutNonIncrementalReset, - LayoutSelectorMatchCategory, - LayoutTreeBuilderCategory, - LayoutDamagePropagateCategory, - LayoutMainCategory, - LayoutParallelWarmupCategory, - LayoutShapingCategory, - LayoutDispListBuildCategory, - PaintingPerTileCategory, - PaintingPrepBuffCategory, - PaintingCategory, -} - -impl Formatable for TimeProfilerCategory { - // some categories are subcategories of LayoutPerformCategory - // and should be printed to indicate this - fn format(&self) -> String { - let padding = match *self { - LayoutStyleRecalcCategory | - LayoutRestyleDamagePropagation | - LayoutNonIncrementalReset | - LayoutMainCategory | - LayoutDispListBuildCategory | - LayoutShapingCategory | - LayoutDamagePropagateCategory | - PaintingPerTileCategory | - PaintingPrepBuffCategory => "+ ", - LayoutParallelWarmupCategory | - LayoutSelectorMatchCategory | - LayoutTreeBuilderCategory => "| + ", - _ => "" - }; - let name = match *self { - CompositingCategory => "Compositing", - LayoutPerformCategory => "Layout", - LayoutStyleRecalcCategory => "Style Recalc", - LayoutRestyleDamagePropagation => "Restyle Damage Propagation", - LayoutNonIncrementalReset => "Non-incremental reset (temporary)", - LayoutSelectorMatchCategory => "Selector Matching", - LayoutTreeBuilderCategory => "Tree Building", - LayoutDamagePropagateCategory => "Damage Propagation", - LayoutMainCategory => "Primary Layout Pass", - LayoutParallelWarmupCategory => "Parallel Warmup", - LayoutShapingCategory => "Shaping", - LayoutDispListBuildCategory => "Display List Construction", - PaintingPerTileCategory => "Painting Per Tile", - PaintingPrepBuffCategory => "Buffer Prep", - PaintingCategory => "Painting", - }; - format!("{:s}{}", padding, name) - } -} - -type TimeProfilerBuckets = TreeMap<(TimeProfilerCategory, Option), Vec>; - -// back end of the profiler that handles data aggregation and performance metrics -pub struct TimeProfiler { - pub port: Receiver, - buckets: TimeProfilerBuckets, - pub last_msg: Option, -} - -impl TimeProfiler { - pub fn create(period: Option) -> TimeProfilerChan { - let (chan, port) = channel(); - match period { - Some(period) => { - let period = Duration::milliseconds((period * 1000f64) as i64); - let chan = chan.clone(); - spawn_named("Time profiler timer", proc() { - loop { - sleep(period); - if chan.send_opt(PrintMsg).is_err() { - break; - } - } - }); - // Spawn the time profiler. - spawn_named("Time profiler", proc() { - let mut profiler = TimeProfiler::new(port); - profiler.start(); - }); - } - None => { - // No-op to handle messages when the time profiler is inactive. - spawn_named("Time profiler", proc() { - loop { - match port.recv_opt() { - Err(_) | Ok(ExitMsg) => break, - _ => {} - } - } - }); - } - } - - TimeProfilerChan(chan) - } - - pub fn new(port: Receiver) -> TimeProfiler { - TimeProfiler { - port: port, - buckets: TreeMap::new(), - last_msg: None, - } - } - - pub fn start(&mut self) { - loop { - let msg = self.port.recv_opt(); - match msg { - Ok(msg) => { - if !self.handle_msg(msg) { - break - } - } - _ => break - } - } - } - - fn find_or_insert(&mut self, k: (TimeProfilerCategory, Option), t: f64) { - match self.buckets.get_mut(&k) { - None => {}, - Some(v) => { v.push(t); return; }, - } - - self.buckets.insert(k, vec!(t)); - } - - fn handle_msg(&mut self, msg: TimeProfilerMsg) -> bool { - match msg.clone() { - TimeMsg(k, t) => self.find_or_insert(k, t), - PrintMsg => match self.last_msg { - // only print if more data has arrived since the last printout - Some(TimeMsg(..)) => self.print_buckets(), - _ => () - }, - ExitMsg => return false, - }; - self.last_msg = Some(msg); - true - } - - fn print_buckets(&mut self) { - println!("{:35s} {:14} {:9} {:30} {:15s} {:15s} {:-15s} {:-15s} {:-15s}", - "_category_", "_incremental?_", "_iframe?_", - " _url_", " _mean (ms)_", " _median (ms)_", - " _min (ms)_", " _max (ms)_", " _events_"); - for (&(ref category, ref meta), ref mut data) in self.buckets.iter_mut() { - data.sort_by(|a, b| { - if a < b { - Less - } else { - Greater - } - }); - let data_len = data.len(); - if data_len > 0 { - let (mean, median, min, max) = - (data.iter().map(|&x|x).sum() / (data_len as f64), - data.as_slice()[data_len / 2], - data.iter().fold(f64::INFINITY, |a, &b| a.min(b)), - data.iter().fold(-f64::INFINITY, |a, &b| a.max(b))); - println!("{:-35s}{} {:15.4f} {:15.4f} {:15.4f} {:15.4f} {:15u}", - category.format(), meta.format(), mean, median, min, max, data_len); - } - } - println!(""); - } -} - - -pub fn profile(category: TimeProfilerCategory, - // url, iframe?, first reflow? - meta: Option<(&Url, bool, bool)>, - time_profiler_chan: TimeProfilerChan, - callback: || -> T) - -> T { - let start_time = precise_time_ns(); - let val = callback(); - let end_time = precise_time_ns(); - let ms = (end_time - start_time) as f64 / 1000000f64; - let meta = meta.map(|(url, iframe, first_reflow)| - TimerMetadata { - url: url.serialize(), - iframe: iframe, - incremental: !first_reflow, - }); - time_profiler_chan.send(TimeMsg((category, meta), ms)); - return val; -} - -pub fn time(msg: &str, callback: || -> T) -> T{ - let start_time = precise_time_ns(); - let val = callback(); - let end_time = precise_time_ns(); - let ms = (end_time - start_time) as f64 / 1000000f64; - if ms >= 5f64 { - debug!("{:s} took {} ms", msg, ms); - } - return val; -} - -// Parses an RFC 2616 compliant date/time string -pub fn parse_http_timestamp(timestamp: &str) -> Option { - // RFC 822, updated by RFC 1123 - match strptime(timestamp, "%a, %d %b %Y %T %Z") { - Ok(t) => return Some(t), - Err(_) => () - } - - // RFC 850, obsoleted by RFC 1036 - match strptime(timestamp, "%A, %d-%b-%y %T %Z") { - Ok(t) => return Some(t), - Err(_) => () - } - - // ANSI C's asctime() format - strptime(timestamp, "%c").ok() -} diff --git a/tests/content/harness.js b/tests/content/harness.js deleted file mode 100644 index 452c72fa67d..00000000000 --- a/tests/content/harness.js +++ /dev/null @@ -1,106 +0,0 @@ -function _oneline(x) { - var i = x.indexOf("\n"); - return (i == -1) ? x : (x.slice(0, i) + "..."); -} - -var _expectations = 0; -var _tests = 0; -function expect(num) { - _expectations = num; -} - -function _fail(s, m) { - _tests++; - // string split to avoid problems with tests that end up printing the value of window._fail. - window.alert(_oneline("TEST-UNEXPECTED" + "-FAIL | " + s + ": " + m)); -} - -function _pass(s, m) { - _tests++; - window.alert(_oneline("TEST-PASS | " + s + ": " + m)); -} - -function _printer(opstr, op) { - return function (a, b, msg) { - var f = op(a,b) ? _pass : _fail; - if (!msg) msg = ""; - f(a + " " + opstr + " " + b, msg); - }; -} - -var is = _printer("===", function (a,b) { return a === b; }); -var is_not = _printer("!==", function (a,b) { return a !== b; }); -var is_a = _printer("is a", function (a,b) { return a instanceof b; }); -var is_not_a = _printer("is not a", function (a,b) { return !(a instanceof b); }); -var is_in = _printer("is in", function (a,b) { return a in b; }); -var is_not_in = _printer("is not in", function (a,b) { return !(a in b); }); -var as_str_is = _printer("as string is", function (a,b) { return String(a) == b; }); -var lt = _printer("<", function (a,b) { return a < b; }); -var gt = _printer(">", function (a,b) { return a > b; }); -var leq = _printer("<=", function (a,b) { return a <= b; }); -var geq = _printer(">=", function (a,b) { return a >= b; }); -var starts_with = _printer("starts with", function (a,b) { return a.indexOf(b) == 0; }); - -function is_function(val, name) { - starts_with(String(val), "function " + name + "("); -} - -function should_throw(f) { - try { - f(); - _fail("operation should have thrown but did not"); - } catch (x) { - _pass("operation successfully threw an exception", x.toString()); - } -} - -function should_not_throw(f) { - try { - f(); - _pass("operation did not throw an exception"); - } catch (x) { - _fail("operation should have not thrown", x.toString()); - } -} - -function check_selector(elem, selector, matches) { - is(elem.matches(selector), matches); -} - -function check_disabled_selector(elem, disabled) { - check_selector(elem, ":disabled", disabled); - check_selector(elem, ":enabled", !disabled); -} - -var _test_complete = false; -var _test_timeout = 10000; //10 seconds -function finish() { - if (_test_complete) { - _fail('finish called multiple times'); - } - if (_expectations > _tests) { - _fail('expected ' + _expectations + ' tests, fullfilled ' + _tests); - } - _test_complete = true; - window.close(); -} - -function _test_timed_out() { - if (!_test_complete) { - _fail('test timed out (' + _test_timeout/1000 + 's)'); - finish(); - } -} - -setTimeout(_test_timed_out, _test_timeout); - -var _needs_finish = false; -function waitForExplicitFinish() { - _needs_finish = true; -} - -addEventListener('load', function() { - if (!_needs_finish) { - finish(); - } -}); diff --git a/tests/content/netharness.js b/tests/content/netharness.js deleted file mode 100644 index cc18fe1300d..00000000000 --- a/tests/content/netharness.js +++ /dev/null @@ -1,25 +0,0 @@ -function assert_requests_made(url, n) { - var x = new XMLHttpRequest(); - x.open('GET', 'stats?' + url, false); - x.send(); - is(parseInt(x.responseText), n, '# of requests for ' + url + ' should be ' + n); -} - -function reset_stats() { - var x = new XMLHttpRequest(); - x.open('POST', 'reset', false); - x.send(); - is(x.status, 200, 'resetting stats should succeed'); -} - -function fetch(url, headers) { - var x = new XMLHttpRequest(); - x.open('GET', url, false); - if (headers) { - for (var i = 0; i < headers.length; i++) { - x.setRequestHeader(headers[i][0], headers[i][1]); - } - } - x.send(); - is(x.status, 200, 'fetching ' + url + ' should succeed '); -} diff --git a/tests/content/resources/helper.html b/tests/content/resources/helper.html deleted file mode 100644 index 90531a4b3ed..00000000000 --- a/tests/content/resources/helper.html +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/content/resources/helper_must_revalidate.html b/tests/content/resources/helper_must_revalidate.html deleted file mode 100644 index 90531a4b3ed..00000000000 --- a/tests/content/resources/helper_must_revalidate.html +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/content/resources/helper_must_revalidate.html^headers b/tests/content/resources/helper_must_revalidate.html^headers deleted file mode 100644 index 5f4c23137e1..00000000000 --- a/tests/content/resources/helper_must_revalidate.html^headers +++ /dev/null @@ -1,2 +0,0 @@ -200 -Cache-Control: must-revalidate \ No newline at end of file diff --git a/tests/content/resources/helper_nocache.html b/tests/content/resources/helper_nocache.html deleted file mode 100644 index 90531a4b3ed..00000000000 --- a/tests/content/resources/helper_nocache.html +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/content/resources/helper_nocache.html^headers b/tests/content/resources/helper_nocache.html^headers deleted file mode 100644 index e510c1a6f9a..00000000000 --- a/tests/content/resources/helper_nocache.html^headers +++ /dev/null @@ -1,2 +0,0 @@ -200 -Cache-Control: no-cache \ No newline at end of file diff --git a/tests/content/test_cached_headers_differ.html b/tests/content/test_cached_headers_differ.html deleted file mode 100644 index ba0e005a8c9..00000000000 --- a/tests/content/test_cached_headers_differ.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/content/test_cached_request.html b/tests/content/test_cached_request.html deleted file mode 100644 index 978e783f220..00000000000 --- a/tests/content/test_cached_request.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/content/test_document_url.html b/tests/content/test_document_url.html deleted file mode 100644 index 99b2a602b36..00000000000 --- a/tests/content/test_document_url.html +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - diff --git a/tests/content/test_nocache.html b/tests/content/test_nocache.html deleted file mode 100644 index d360841b5e2..00000000000 --- a/tests/content/test_nocache.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/content/test_revalidate.html b/tests/content/test_revalidate.html deleted file mode 100644 index 1caa1562b4a..00000000000 --- a/tests/content/test_revalidate.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/contenttest.rs b/tests/contenttest.rs deleted file mode 100644 index 95e017ee768..00000000000 --- a/tests/contenttest.rs +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2013 The Servo Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![deny(unused_imports)] -#![deny(unused_variables)] - -extern crate getopts; -extern crate regex; -extern crate test; - -use test::{AutoColor, TestOpts, run_tests_console, TestDesc, TestDescAndFn, DynTestFn, DynTestName}; -use getopts::{getopts, reqopt}; -use std::comm::channel; -use std::from_str::FromStr; -use std::{os, str}; -use std::io::fs; -use std::io::Reader; -use std::io::process::{Command, Ignored, CreatePipe, InheritFd, ExitStatus}; -use std::task; -use regex::Regex; - -#[deriving(Clone)] -struct Config { - source_dir: String, - filter: Option -} - -fn main() { - let args = os::args(); - let config = parse_config(args.into_iter().collect()); - let opts = test_options(&config); - let tests = find_tests(&config); - match run_tests_console(&opts, tests) { - Ok(false) => os::set_exit_status(1), // tests failed - Err(_) => os::set_exit_status(2), // I/O-related failure - _ => (), - } -} - -enum ServerMsg { - IsAlive(Sender), - Exit, -} - -fn run_http_server(source_dir: String) -> (Sender, u16) { - let (tx, rx) = channel(); - let (port_sender, port_receiver) = channel(); - task::spawn(proc() { - let mut prc = Command::new("python") - .args(["../httpserver.py"]) - .stdin(Ignored) - .stdout(CreatePipe(false, true)) - .stderr(Ignored) - .cwd(&Path::new(source_dir)) - .spawn() - .ok() - .expect("Unable to spawn server."); - - let mut bytes = vec!(); - loop { - let byte = prc.stdout.as_mut().unwrap().read_byte().unwrap(); - if byte == '\n' as u8 { - break; - } else { - bytes.push(byte); - } - } - - let mut words = str::from_utf8(bytes.as_slice()).unwrap().split(' '); - let port = FromStr::from_str(words.last().unwrap()).unwrap(); - port_sender.send(port); - - loop { - match rx.recv() { - IsAlive(reply) => reply.send(prc.signal(0).is_ok()), - Exit => { - let _ = prc.signal_exit(); - break; - } - } - } - }); - (tx, port_receiver.recv()) -} - -fn parse_config(args: Vec) -> Config { - let args = args.tail(); - let opts = vec!(reqopt("s", "source-dir", "source-dir", "source-dir")); - let matches = match getopts(args, opts.as_slice()) { - Ok(m) => m, - Err(f) => panic!(format!("{}", f)) - }; - - Config { - source_dir: matches.opt_str("source-dir").unwrap(), - filter: matches.free.as_slice().head().map(|s| Regex::new(s.as_slice()).unwrap()) - } -} - -fn test_options(config: &Config) -> TestOpts { - TestOpts { - filter: config.filter.clone(), - run_ignored: false, - run_tests: true, - run_benchmarks: false, - ratchet_metrics: None, - ratchet_noise_percent: None, - save_metrics: None, - test_shard: None, - logfile: None, - nocapture: false, - color: AutoColor - } -} - -fn find_tests(config: &Config) -> Vec { - let files_res = fs::readdir(&Path::new(config.source_dir.clone())); - let mut files = match files_res { - Ok(files) => files, - _ => panic!("Error reading directory."), - }; - files.retain(|file| file.extension_str() == Some("html") ); - return files.iter().map(|file| make_test(format!("{}", file.display()), - config.source_dir.clone())).collect(); -} - -fn make_test(file: String, source_dir: String) -> TestDescAndFn { - TestDescAndFn { - desc: TestDesc { - name: DynTestName(file.clone()), - ignore: false, - should_fail: false - }, - testfn: DynTestFn(proc() { run_test(file, source_dir) }) - } -} - -fn run_test(file: String, source_dir: String) { - let (server, port) = run_http_server(source_dir); - - let path = os::make_absolute(&Path::new(file)); - // FIXME (#1094): not the right way to transform a path - let infile = format!("http://localhost:{}/{}", port, path.filename_display()); - let stdout = CreatePipe(false, true); - let stderr = InheritFd(2); - let args = ["-z", "-f", infile.as_slice()]; - - let (tx, rx) = channel(); - server.send(IsAlive(tx)); - assert!(rx.recv(), "HTTP server must be running."); - - let mut prc = match Command::new("target/servo") - .args(args) - .stdin(Ignored) - .stdout(stdout) - .stderr(stderr) - .spawn() - { - Ok(p) => p, - _ => panic!("Unable to spawn process."), - }; - let mut output = Vec::new(); - loop { - let byte = prc.stdout.as_mut().unwrap().read_byte(); - match byte { - Ok(byte) => { - print!("{}", byte as char); - output.push(byte); - } - _ => break - } - } - - server.send(Exit); - - let out = str::from_utf8(output.as_slice()); - let lines: Vec<&str> = out.unwrap().split('\n').collect(); - for &line in lines.iter() { - if line.contains("TEST-UNEXPECTED-FAIL") { - panic!(line.to_string()); - } - } - - let retval = prc.wait(); - if retval != Ok(ExitStatus(0)) { - panic!("Servo exited with non-zero status {}", retval); - } -} diff --git a/tests/httpserver.py b/tests/httpserver.py deleted file mode 100644 index 689d29bef91..00000000000 --- a/tests/httpserver.py +++ /dev/null @@ -1,115 +0,0 @@ -from SimpleHTTPServer import SimpleHTTPRequestHandler -import SocketServer -import os -import sys -from collections import defaultdict - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 0 - -requests = defaultdict(int) - -class CountingRequestHandler(SimpleHTTPRequestHandler): - def __init__(self, req, client_addr, server): - SimpleHTTPRequestHandler.__init__(self, req, client_addr, server) - - def do_POST(self): - global requests - parts = self.path.split('/') - - if parts[1] == 'reset': - requests = defaultdict(int) - self.send_response(200) - self.send_header('Content-Type', 'text/plain') - self.send_header('Content-Length', 0) - self.end_headers() - self.wfile.write('') - return - - def do_GET(self): - global requests - parts = self.path.split('?') - if parts[0] == '/stats': - self.send_response(200) - self.send_header('Content-Type', 'text/plain') - if len(parts) > 1: - body = str(requests['/' + parts[1]]) - else: - body = '' - for key, value in requests.iteritems(): - body += key + ': ' + str(value) + '\n' - self.send_header('Content-Length', len(body)) - self.end_headers() - self.wfile.write(body) - return - - header_list = [] - status = None - - path = self.translate_path(self.path) - headers = path + '^headers' - - if os.path.isfile(headers): - try: - h = open(headers, 'rb') - except IOError: - self.send_error(404, "Header file not found") - return - - header_lines = h.readlines() - status = int(header_lines[0]) - for header in header_lines[1:]: - parts = map(lambda x: x.strip(), header.split(':')) - header_list += [parts] - - if self.headers.get('If-Modified-Since'): - self.send_response(304) - self.end_headers() - return - - if not status or status == 200: - requests[self.path] += 1 - - if status or header_list: - ctype = self.guess_type(path) - try: - # Always read in binary mode. Opening files in text mode may cause - # newline translations, making the actual size of the content - # transmitted *less* than the content-length! - f = open(path, 'rb') - except IOError: - self.send_error(404, "File not found") - return - - try: - self.send_response(status or 200) - self.send_header("Content-type", ctype) - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) - - for header in header_list: - self.send_header(header[0], header[1]) - - self.end_headers() - - try: - self.copyfile(f, self.wfile) - finally: - f.close() - except: - f.close() - raise - else: - SimpleHTTPRequestHandler.do_GET(self) - -class MyTCPServer(SocketServer.TCPServer): - request_queue_size = 2000 - allow_reuse_address = True - -httpd = MyTCPServer(("", PORT), CountingRequestHandler) -if not PORT: - ip, PORT = httpd.server_address - -print "serving at port", PORT -sys.stdout.flush() -httpd.serve_forever() diff --git a/tests/wpt/metadata/MANIFEST.json b/tests/wpt/metadata/MANIFEST.json index 7a2c8735267..180cb4feebb 100644 --- a/tests/wpt/metadata/MANIFEST.json +++ b/tests/wpt/metadata/MANIFEST.json @@ -524761,7 +524761,7 @@ "support" ], "fetch/http-cache/cc-request.html": [ - "d4417b8fd444362a3f217d1c95d37811a608e1a7", + "2002d341679139428e164cfe916dd39b9b664a3e", "testharness" ], "fetch/http-cache/freshness.html": [ @@ -524769,7 +524769,7 @@ "testharness" ], "fetch/http-cache/heuristic.html": [ - "5b0d55f891cb2e235456cd65f4e9f63e07999410", + "63837026eb6085fc7d6220c3dcab200b4bcd1eca", "testharness" ], "fetch/http-cache/http-cache.js": [ @@ -524781,7 +524781,7 @@ "testharness" ], "fetch/http-cache/partial.html": [ - "243e57c39f9e45e3e2acf845b36f3a140e3763bc", + "685057fe8876321a5d42bcf1e7582e6f0b745f85", "testharness" ], "fetch/http-cache/resources/http-cache.py": [ @@ -524793,7 +524793,7 @@ "testharness" ], "fetch/http-cache/vary.html": [ - "fa9a2e0554671bf2de5826e66ac0ea73de28d530", + "45f337270cfa90932c7469802655e313367ac92f", "testharness" ], "fetch/nosniff/image.html": [ diff --git a/tests/wpt/metadata/cors/304.htm.ini b/tests/wpt/metadata/cors/304.htm.ini deleted file mode 100644 index 9b7b2c0cf46..00000000000 --- a/tests/wpt/metadata/cors/304.htm.ini +++ /dev/null @@ -1,14 +0,0 @@ -[304.htm] - type: testharness - [A 304 response with no CORS headers inherits from the stored response] - expected: FAIL - - [A 304 can expand Access-Control-Expose-Headers] - expected: FAIL - - [A 304 can contract Access-Control-Expose-Headers] - expected: FAIL - - [A 304 can change Access-Control-Allow-Origin] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-default-conditional.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-default-conditional.html.ini deleted file mode 100644 index 919c03caf2a..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-default-conditional.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[request-cache-default-conditional.html] - type: testharness - disabled: https://github.com/servo/servo/issues/13441 diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-default.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-default.html.ini deleted file mode 100644 index 1363bbf5134..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-default.html.ini +++ /dev/null @@ -1,11 +0,0 @@ -[request-cache-default.html] - type: testharness - [RequestCache "default" mode checks the cache for previously cached content and avoids going to the network if a fresh response exists with Etag and fresh response] - expected: FAIL - - [RequestCache "default" mode checks the cache for previously cached content and avoids going to the network if a fresh response exists with date and fresh response] - expected: FAIL - - [RequestCache "default" mode checks the cache for previously cached content and avoids going to the network if a fresh response exists with Last-Modified and fresh response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-force-cache.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-force-cache.html.ini deleted file mode 100644 index 24d84797f31..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-force-cache.html.ini +++ /dev/null @@ -1,29 +0,0 @@ -[request-cache-force-cache.html] - type: testharness - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for stale responses with Etag and stale response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for stale responses with date and stale response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for fresh responses with Etag and fresh response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for fresh responses with date and fresh response] - expected: FAIL - - [RequestCache "force-cache" stores the response in the cache if it goes to the network with Etag and fresh response] - expected: FAIL - - [RequestCache "force-cache" stores the response in the cache if it goes to the network with date and fresh response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for stale responses with Last-Modified and stale response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for fresh responses with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "force-cache" stores the response in the cache if it goes to the network with Last-Modified and fresh response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-only-if-cached.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-only-if-cached.html.ini deleted file mode 100644 index a3406a375cc..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-only-if-cached.html.ini +++ /dev/null @@ -1,65 +0,0 @@ -[request-cache-only-if-cached.html] - type: testharness - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for stale responses with Etag and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for stale responses with date and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for fresh responses with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for fresh responses with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and does not go to the network if a cached response is not found with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and does not go to the network if a cached response is not found with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Etag and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with date and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Etag and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with date and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for stale responses with Last-Modified and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for fresh responses with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and does not go to the network if a cached response is not found with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Last-Modified and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Last-Modified and stale response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-reload.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-reload.html.ini deleted file mode 100644 index 4538f3d560f..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-reload.html.ini +++ /dev/null @@ -1,20 +0,0 @@ -[request-cache-reload.html] - type: testharness - [RequestCache "reload" mode does store the response in the cache with Etag and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache with date and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache even if a previous response is already stored with Etag and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache even if a previous response is already stored with date and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache even if a previous response is already stored with Last-Modified and fresh response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/304-update.html.ini b/tests/wpt/metadata/fetch/http-cache/304-update.html.ini deleted file mode 100644 index d6a1386dd4c..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/304-update.html.ini +++ /dev/null @@ -1,14 +0,0 @@ -[304-update.html] - type: testharness - [HTTP cache updates returned headers from a Last-Modified 304.] - expected: FAIL - - [HTTP cache updates stored headers from a Last-Modified 304.] - expected: FAIL - - [HTTP cache updates returned headers from a ETag 304.] - expected: FAIL - - [HTTP cache updates stored headers from a ETag 304.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini b/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini index 253204fe700..cc3c925009b 100644 --- a/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini +++ b/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini @@ -1,11 +1,5 @@ [cc-request.html] type: testharness - [HTTP cache does use aged stale response when request contains Cache-Control: max-stale that permits its use.] - expected: FAIL - - [HTTP cache does reuse stale response with Age header when request contains Cache-Control: max-stale that permits its use.] - expected: FAIL - [HTTP cache generates 504 status code when nothing is in cache and request contains Cache-Control: only-if-cached.] expected: FAIL diff --git a/tests/wpt/metadata/fetch/http-cache/freshness.html.ini b/tests/wpt/metadata/fetch/http-cache/freshness.html.ini deleted file mode 100644 index 28ad1e13bd3..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/freshness.html.ini +++ /dev/null @@ -1,20 +0,0 @@ -[freshness.html] - type: testharness - [HTTP cache reuses a response with a future Expires.] - expected: FAIL - - [HTTP cache reuses a response with positive Cache-Control: max-age.] - expected: FAIL - - [HTTP cache reuses a response with positive Cache-Control: max-age and a past Expires.] - expected: FAIL - - [HTTP cache reuses a response with positive Cache-Control: max-age and an invalid Expires.] - expected: FAIL - - [HTTP cache stores a response with Cache-Control: no-cache, but revalidates upon use.] - expected: FAIL - - [HTTP cache stores a response with Cache-Control: no-cache, but revalidates upon use, even with max-age and Expires.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/heuristic.html.ini b/tests/wpt/metadata/fetch/http-cache/heuristic.html.ini deleted file mode 100644 index 9a1905eba23..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/heuristic.html.ini +++ /dev/null @@ -1,29 +0,0 @@ -[heuristic.html] - type: testharness - [HTTP cache reuses an unknown response with Last-Modified based upon heuristic freshness when Cache-Control: public is present.] - expected: FAIL - - [HTTP cache reuses a 200 OK response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 203 Non-Authoritative Information response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 204 No Content response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 404 Not Found response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 405 Method Not Allowed response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 410 Gone response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 414 URI Too Long response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 501 Not Implemented response with Last-Modified based upon heuristic freshness.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini b/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini index ccf74fbef31..2ef2eb49c2b 100644 --- a/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini +++ b/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini @@ -1,20 +1,11 @@ [invalidate.html] type: testharness - [HTTP cache does not invalidate after a failed response from an unsafe request] - expected: FAIL - [HTTP cache invalidates after a successful response from an unknown method] expected: FAIL - [HTTP cache does not invalidate Location URL after a failed response from an unsafe request] - expected: FAIL - [HTTP cache invalidates Location URL after a successful response from an unknown method] expected: FAIL - [HTTP cache does not invalidate Content-Location URL after a failed response from an unsafe request] - expected: FAIL - [HTTP cache invalidates Content-Location URL after a successful response from an unknown method] expected: FAIL diff --git a/tests/wpt/metadata/fetch/http-cache/partial.html.ini b/tests/wpt/metadata/fetch/http-cache/partial.html.ini index e4afd89f90b..65c7f3ebc0c 100644 --- a/tests/wpt/metadata/fetch/http-cache/partial.html.ini +++ b/tests/wpt/metadata/fetch/http-cache/partial.html.ini @@ -1,13 +1,5 @@ [partial.html] type: testharness - [HTTP cache stores partial content and reuses it.] - expected: FAIL - - [HTTP cache stores complete response and serves smaller ranges from it.] - expected: FAIL - - [HTTP cache stores partial response and serves smaller ranges from it.] - expected: FAIL [HTTP cache stores partial content and completes it.] expected: FAIL diff --git a/tests/wpt/metadata/fetch/http-cache/status.html.ini b/tests/wpt/metadata/fetch/http-cache/status.html.ini deleted file mode 100644 index c5990d83543..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/status.html.ini +++ /dev/null @@ -1,41 +0,0 @@ -[status.html] - type: testharness - [HTTP cache avoids going to the network if it has a fresh 200 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 203 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 204 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 299 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 400 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 404 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 410 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 499 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 500 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 502 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 503 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 504 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 599 response.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/vary.html.ini b/tests/wpt/metadata/fetch/http-cache/vary.html.ini deleted file mode 100644 index fe697287a21..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/vary.html.ini +++ /dev/null @@ -1,17 +0,0 @@ -[vary.html] - type: testharness - [HTTP cache reuses Vary response when request matches.] - expected: FAIL - - [HTTP cache doesn't invalidate existing Vary response.] - expected: FAIL - - [HTTP cache doesn't pay attention to headers not listed in Vary.] - expected: FAIL - - [HTTP cache reuses two-way Vary response when request matches.] - expected: FAIL - - [HTTP cache reuses three-way Vary response when request matches.] - expected: FAIL - diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html b/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html index 05d6f6b8c09..6ea8fbc92fd 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html @@ -201,7 +201,8 @@ request_headers: [ ["Cache-Control", "only-if-cached"] ], - expected_status: 504 + expected_status: 504, + expected_response_text: "" } ] } diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html b/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html index 429dddace6a..81deb1d0688 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html @@ -26,6 +26,7 @@ }, { expected_type: "cached", + response_status: [299, "Whatever"], } ] }, @@ -35,8 +36,7 @@ { response_status: [299, "Whatever"], response_headers: [ - ['Last-Modified', http_date(-3 * 100)], - ['Cache-Control', 'public'] + ['Last-Modified', http_date(-3 * 100)] ], }, { diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/partial.html b/tests/wpt/web-platform-tests/fetch/http-cache/partial.html index 3ad593375ac..8d5d61d46c4 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/partial.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/partial.html @@ -24,7 +24,7 @@ response_status: [206, "Partial Content"], response_headers: [ ['Cache-Control', 'max-age=3600'], - ['Content-Range', 'bytes 0-4/10'] + ['Content-Range', 'bytes 4-9/10'] ], response_body: "01234", expected_request_headers: [ @@ -36,12 +36,13 @@ ['Range', "bytes=-5"] ], expected_type: "cached", - expected_status: 206 + expected_status: 206, + expected_response_text: "01234" } ] }, { - name: 'HTTP cache stores complete response and serves smaller ranges from it.', + name: 'HTTP cache stores complete response and serves smaller ranges from it(byte-range-spec).', requests: [ { response_headers: [ @@ -51,15 +52,54 @@ }, { request_headers: [ - ['Range', "bytes=-1"] + ['Range', "bytes=0-1"] ], expected_type: "cached", + expected_status: 206, expected_response_text: "01" + }, + ] + }, + { + name: 'HTTP cache stores complete response and serves smaller ranges from it(absent last-byte-pos).', + requests: [ + { + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ], + response_body: "01234567890", + }, + { + request_headers: [ + ['Range', "bytes=1-"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "1234567890" } ] }, { - name: 'HTTP cache stores partial response and serves smaller ranges from it.', + name: 'HTTP cache stores complete response and serves smaller ranges from it(suffix-byte-range-spec).', + requests: [ + { + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ], + response_body: "0123456789A", + }, + { + request_headers: [ + ['Range', "bytes=-1"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "A" + } + ] + }, + { + name: 'HTTP cache stores partial response and serves smaller ranges from it(byte-range-spec).', requests: [ { request_headers: [ @@ -68,7 +108,55 @@ response_status: [206, "Partial Content"], response_headers: [ ['Cache-Control', 'max-age=3600'], - ['Content-Range', 'bytes 0-4/10'] + ['Content-Range', 'bytes 4-9/10'] + ], + response_body: "01234", + }, + { + request_headers: [ + ['Range', "bytes=6-8"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "234" + } + ] + }, + { + name: 'HTTP cache stores partial response and serves smaller ranges from it(absent last-byte-pos).', + requests: [ + { + request_headers: [ + ['Range', "bytes=-5"] + ], + response_status: [206, "Partial Content"], + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ['Content-Range', 'bytes 4-9/10'] + ], + response_body: "01234", + }, + { + request_headers: [ + ['Range', "bytes=6-"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "234" + } + ] + }, + { + name: 'HTTP cache stores partial response and serves smaller ranges from it(suffix-byte-range-spec).', + requests: [ + { + request_headers: [ + ['Range', "bytes=-5"] + ], + response_status: [206, "Partial Content"], + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ['Content-Range', 'bytes 4-9/10'] ], response_body: "01234", }, @@ -77,7 +165,8 @@ ['Range', "bytes=-1"] ], expected_type: "cached", - expected_response_text: "01" + expected_status: 206, + expected_response_text: "4" } ] }, diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/vary.html b/tests/wpt/web-platform-tests/fetch/http-cache/vary.html index 2f4b945b0af..dd42b14f27a 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/vary.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/vary.html @@ -103,6 +103,7 @@ request_headers: [ ["Foo", "1"] ], + response_body: http_content('foo_1'), expected_type: "cached" } ] @@ -245,7 +246,32 @@ ] }, { - name: "HTTP cache doesn't use three-way Vary response when request omits variant header.", + name: "HTTP cache doesn't use three-way Vary response when request doesn't match, regardless of header order.", + requests: [ + { + request_headers: [ + ["Foo", "1"], + ["Bar", "abc4"], + ["Baz", "789"] + ], + response_headers: [ + ["Expires", http_date(5000)], + ["Last-Modified", http_date(-3000)], + ["Vary", "Foo, Bar, Baz"] + ] + }, + { + request_headers: [ + ["Foo", "1"], + ["Bar", "abc"], + ["Baz", "789"] + ], + expected_type: "not_cached" + } + ] + }, + { + name: "HTTP cache uses three-way Vary response when both request and the original request omited a variant header.", requests: [ { request_headers: [ @@ -259,6 +285,33 @@ ] }, { + request_headers: [ + ["Foo", "1"], + ["Baz", "789"] + ], + expected_type: "cached" + } + ] + }, + { + name: "HTTP cache doesn't use Vary response with a field value of '*'.", + requests: [ + { + request_headers: [ + ["Foo", "1"], + ["Baz", "789"] + ], + response_headers: [ + ["Expires", http_date(5000)], + ["Last-Modified", http_date(-3000)], + ["Vary", "*"] + ] + }, + { + request_headers: [ + ["*", "1"], + ["Baz", "789"] + ], expected_type: "not_cached" } ] From b98635f2122b5c1c47252b351994a0b2c6bc2140 Mon Sep 17 00:00:00 2001 From: Gregory Terzian Date: Wed, 15 Nov 2017 10:16:16 +0800 Subject: [PATCH 5/7] allow unknown HTTP methods --- components/script/dom/request.rs | 12 +----------- .../fetch/api/basic/request-headers.any.js.ini | 6 ------ .../fetch/api/cors/cors-preflight-star.any.js.ini | 3 --- .../fetch/api/cors/cors-preflight.any.js.ini | 12 ------------ .../metadata/fetch/http-cache/invalidate.html.ini | 11 ----------- 5 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 tests/wpt/metadata/fetch/http-cache/invalidate.html.ini diff --git a/components/script/dom/request.rs b/components/script/dom/request.rs index fcd396c1176..85998533ae9 100644 --- a/components/script/dom/request.rs +++ b/components/script/dom/request.rs @@ -462,17 +462,7 @@ fn normalize_method(m: &str) -> HttpMethod { // https://fetch.spec.whatwg.org/#concept-method fn is_method(m: &ByteString) -> bool { - match m.to_lower().as_str() { - Some("get") => true, - Some("head") => true, - Some("post") => true, - Some("put") => true, - Some("delete") => true, - Some("connect") => true, - Some("options") => true, - Some("trace") => true, - _ => false, - } + m.as_str().is_some() } // https://fetch.spec.whatwg.org/#forbidden-method diff --git a/tests/wpt/metadata/fetch/api/basic/request-headers.any.js.ini b/tests/wpt/metadata/fetch/api/basic/request-headers.any.js.ini index fc724ad68bd..ea7c5b5edf7 100644 --- a/tests/wpt/metadata/fetch/api/basic/request-headers.any.js.ini +++ b/tests/wpt/metadata/fetch/api/basic/request-headers.any.js.ini @@ -39,12 +39,6 @@ [Fetch with Chicken with body] expected: FAIL - [Fetch with TacO and mode "same-origin" needs an Origin header] - expected: FAIL - - [Fetch with TacO and mode "cors" needs an Origin header] - expected: FAIL - [request-headers.any.worker.html] type: testharness diff --git a/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini b/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini index b99585d72cb..775345b3f73 100644 --- a/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini +++ b/tests/wpt/metadata/fetch/api/cors/cors-preflight-star.any.js.ini @@ -1,8 +1,5 @@ [cors-preflight-star.any.html] type: testharness - [CORS that succeeds with credentials: false; method: SUPER (allowed: *); header: X-Test,1 (allowed: x-test)] - expected: FAIL - [CORS that succeeds with credentials: false; method: OK (allowed: *); header: X-Test,1 (allowed: *)] expected: FAIL diff --git a/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini b/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini index 5f6b56e0621..338a4ba3c2b 100644 --- a/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini +++ b/tests/wpt/metadata/fetch/api/cors/cors-preflight.any.js.ini @@ -1,11 +1,5 @@ [cors-preflight.any.html] type: testharness - [CORS [PATCH\], server allows] - expected: FAIL - - [CORS [NEW\], server allows] - expected: FAIL - [CORS [GET\] [several headers\], server allows] expected: FAIL @@ -15,12 +9,6 @@ [cors-preflight.any.worker.html] type: testharness - [CORS [PATCH\], server allows] - expected: FAIL - - [CORS [NEW\], server allows] - expected: FAIL - [CORS [GET\] [several headers\], server allows] expected: FAIL diff --git a/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini b/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini deleted file mode 100644 index 2ef2eb49c2b..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini +++ /dev/null @@ -1,11 +0,0 @@ -[invalidate.html] - type: testharness - [HTTP cache invalidates after a successful response from an unknown method] - expected: FAIL - - [HTTP cache invalidates Location URL after a successful response from an unknown method] - expected: FAIL - - [HTTP cache invalidates Content-Location URL after a successful response from an unknown method] - expected: FAIL - From 5f8a310bd16b3ecbe131a516ce79afd221a95535 Mon Sep 17 00:00:00 2001 From: Gregory Terzian Date: Fri, 17 Nov 2017 21:36:30 +0800 Subject: [PATCH 6/7] also cache redirects --- components/net/http_cache.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/components/net/http_cache.rs b/components/net/http_cache.rs index 764fa3651df..c35f02a3d4d 100644 --- a/components/net/http_cache.rs +++ b/components/net/http_cache.rs @@ -35,7 +35,7 @@ pub struct CacheKey { impl CacheKey { fn new(request: Request) -> CacheKey { CacheKey { - url: request.url().clone() + url: request.current_url().clone() } } @@ -57,6 +57,7 @@ struct CachedResource { metadata: CachedMetadata, request_headers: Arc>, body: Arc>, + location_url: Option>, https_state: HttpsState, status: Option, raw_status: Option<(u16, Vec)>, @@ -273,6 +274,7 @@ fn create_cached_response(request: &Request, cached_resource: &CachedResource, c let mut response = Response::new(cached_resource.metadata.final_url.clone()); response.headers = cached_headers.clone(); response.body = cached_resource.body.clone(); + response.location_url = cached_resource.location_url.clone(); response.status = cached_resource.status.clone(); response.raw_status = cached_resource.raw_status.clone(); response.url_list = cached_resource.url_list.clone(); @@ -300,6 +302,7 @@ fn create_resource_with_bytes_from_resource(bytes: &[u8], resource: &CachedResou metadata: resource.metadata.clone(), request_headers: resource.request_headers.clone(), body: Arc::new(Mutex::new(ResponseBody::Done(bytes.to_owned()))), + location_url: resource.location_url.clone(), https_state: resource.https_state.clone(), status: Some(StatusCode::PartialContent), raw_status: Some((206, b"Partial Content".to_vec())), @@ -606,12 +609,6 @@ impl HttpCache { /// Storing Responses in Caches. /// pub fn store(&mut self, request: &Request, response: &Response) { - if let Some(status) = response.status { - // Not caching redirects, for simplicity, not per the spec. - if is_redirect_status(status) { - return - } - } if request.method != Method::Get { // Only Get requests are cached. return @@ -639,6 +636,7 @@ impl HttpCache { metadata: cacheable_metadata, request_headers: Arc::new(Mutex::new(request.headers.clone())), body: response.body.clone(), + location_url: response.location_url.clone(), https_state: response.https_state.clone(), status: response.status.clone(), raw_status: response.raw_status.clone(), From 6196a6e65d03e7e4f9ff857f0c2315413a41e44a Mon Sep 17 00:00:00 2001 From: Gregory Terzian Date: Sat, 18 Nov 2017 11:21:52 +0800 Subject: [PATCH 7/7] add pref to disable http cache, mozilla tests --- components/net/http_cache.rs | 4 + resources/prefs.json | 1 + tests/wpt/mozilla/meta/MANIFEST.json | 19 ++ .../mozilla/meta/mozilla/http-cache.html.ini | 3 + .../wpt/mozilla/tests/mozilla/http-cache.html | 50 +++ .../tests/mozilla/resources/http-cache.js | 286 ++++++++++++++++++ 6 files changed, 363 insertions(+) create mode 100644 tests/wpt/mozilla/meta/mozilla/http-cache.html.ini create mode 100644 tests/wpt/mozilla/tests/mozilla/http-cache.html create mode 100644 tests/wpt/mozilla/tests/mozilla/resources/http-cache.js diff --git a/components/net/http_cache.rs b/components/net/http_cache.rs index c35f02a3d4d..cd0211399a0 100644 --- a/components/net/http_cache.rs +++ b/components/net/http_cache.rs @@ -18,6 +18,7 @@ use hyper_serde::Serde; use net_traits::{Metadata, FetchMetadata}; use net_traits::request::Request; use net_traits::response::{HttpsState, Response, ResponseBody}; +use servo_config::prefs::PREFS; use servo_url::ServoUrl; use std::collections::HashMap; use std::str; @@ -609,6 +610,9 @@ impl HttpCache { /// Storing Responses in Caches. /// pub fn store(&mut self, request: &Request, response: &Response) { + if PREFS.get("network.http-cache.disabled").as_boolean().unwrap_or(false) { + return + } if request.method != Method::Get { // Only Get requests are cached. return diff --git a/resources/prefs.json b/resources/prefs.json index 1c5a5c92943..36469aebf7d 100644 --- a/resources/prefs.json +++ b/resources/prefs.json @@ -66,6 +66,7 @@ "layout.text-orientation.enabled": false, "layout.viewport.enabled": false, "layout.writing-mode.enabled": false, + "network.http-cache.disabled": false, "network.mime.sniff": false, "session-history.max-length": 20, "shell.builtin-key-shortcuts.enabled": true, diff --git a/tests/wpt/mozilla/meta/MANIFEST.json b/tests/wpt/mozilla/meta/MANIFEST.json index c2125ad4fab..fde870bcf23 100644 --- a/tests/wpt/mozilla/meta/MANIFEST.json +++ b/tests/wpt/mozilla/meta/MANIFEST.json @@ -12466,6 +12466,11 @@ {} ] ], + "mozilla/resources/http-cache.js": [ + [ + {} + ] + ], "mozilla/resources/iframe_contentDocument_inner.html": [ [ {} @@ -33212,6 +33217,12 @@ {} ] ], + "mozilla/http-cache.html": [ + [ + "/_mozilla/mozilla/http-cache.html", + {} + ] + ], "mozilla/iframe-unblock-onload.html": [ [ "/_mozilla/mozilla/iframe-unblock-onload.html", @@ -66299,6 +66310,10 @@ "592f69ee432ba5bc7a2f2649e72e083d21393496", "testharness" ], + "mozilla/http-cache.html": [ + "33827dc9bdb0efcbcae4f730086693be315cfc14", + "testharness" + ], "mozilla/iframe-unblock-onload.html": [ "8734756947d36b047df256f27adc56fce7e31f88", "testharness" @@ -71971,6 +71986,10 @@ "78686147f85e4146e7fc58c1f67a613f65b099a2", "support" ], + "mozilla/resources/http-cache.js": [ + "c6b1ee9def26d4e12a1b93e551c225f82b4717c2", + "support" + ], "mozilla/resources/iframe_contentDocument_inner.html": [ "eb1b1ae3bb7a437fc4fbdd1f537881890fe6347c", "support" diff --git a/tests/wpt/mozilla/meta/mozilla/http-cache.html.ini b/tests/wpt/mozilla/meta/mozilla/http-cache.html.ini new file mode 100644 index 00000000000..1510dd19f98 --- /dev/null +++ b/tests/wpt/mozilla/meta/mozilla/http-cache.html.ini @@ -0,0 +1,3 @@ +[http-cache.html] + type: testharness + prefs: [network.http-cache.disabled:true] diff --git a/tests/wpt/mozilla/tests/mozilla/http-cache.html b/tests/wpt/mozilla/tests/mozilla/http-cache.html new file mode 100644 index 00000000000..3254492e89a --- /dev/null +++ b/tests/wpt/mozilla/tests/mozilla/http-cache.html @@ -0,0 +1,50 @@ + + + + + + + + + + diff --git a/tests/wpt/mozilla/tests/mozilla/resources/http-cache.js b/tests/wpt/mozilla/tests/mozilla/resources/http-cache.js new file mode 100644 index 00000000000..34aaacf536f --- /dev/null +++ b/tests/wpt/mozilla/tests/mozilla/resources/http-cache.js @@ -0,0 +1,286 @@ +/** + * Each test run gets its own URL and randomized content and operates independently. + * + * Tests are an array of objects, each representing a request to make and check. + * The cache.py server script stashes an entry containing observed headers for + * each request it receives. When the test fetches have run, this state is retrieved + * and the expected_* lists are checked, including their length. + * + * Request object keys: + * - template - A template object for the request, by name -- see "templates" below. + * - request_method - A string containing the HTTP method to be used. + * - request_headers - An array of [header_name_string, header_value_string] arrays to + * emit in the request. + * - request_body - A string to use as the request body. + * - mode - The mode string to pass to fetch(). + * - credentials - The credentials string to pass to fetch(). + * - cache - The cache string to pass to fetch(). + * - pause_after - Boolean controlling a 3-second pause after the request completes. + * - response_status - A [number, string] array containing the HTTP status code + * and phrase to return. + * - response_headers - An array of [header_name_string, header_value_string] arrays to + * emit in the response. These values will also be checked like + * expected_response_headers, unless there is a third value that is + * false. + * - response_body - String to send as the response body. If not set, it will contain + * the test identifier. + * - expected_type - One of ["cached", "not_cached", "lm_validate", "etag_validate", "error"] + * - expected_status - A number representing a HTTP status code to check the response for. + * If not set, the value of response_status[0] will be used; if that + * is not set, 200 will be used. + * - expected_request_headers - An array of [header_name_string, header_value_string] representing + * headers to check the request for. + * - expected_response_headers - An array of [header_name_string, header_value_string] representing + * headers to check the response for. See also response_headers. + * - expected_response_text - A string to check the response body against. + */ + +function make_url(uuid, requests, idx) { + var arg = ""; + if ("query_arg" in requests[idx]) { + arg = "&target=" + requests[idx].query_arg; + } + return "/fetch/http-cache/resources/http-cache.py?token=" + uuid + "&info=" + btoa(JSON.stringify(requests)) + arg; +} + +function server_state(uuid) { + return fetch("/fetch/http-cache/resources/http-cache.py?querystate&token=" + uuid) + .then(function(response) { + return response.text(); + }).then(function(text) { + // null will be returned if the server never received any requests + // for the given uuid. Normalize that to an empty list consistent + // with our representation. + return JSON.parse(text) || []; + }); +} + + +templates = { + "fresh": { + "response_headers": [ + ['Expires', http_date(100000)], + ['Last-Modified', http_date(0)] + ] + }, + "stale": { + "response_headers": [ + ['Expires', http_date(-5000)], + ['Last-Modified', http_date(-100000)] + ] + }, + "lcl_response": { + "response_headers": [ + ['Location', "location_target"], + ['Content-Location', "content_location_target"] + ] + }, + "location": { + "query_arg": "location_target", + "response_headers": [ + ['Expires', http_date(100000)], + ['Last-Modified', http_date(0)] + ] + }, + "content_location": { + "query_arg": "content_location_target", + "response_headers": [ + ['Expires', http_date(100000)], + ['Last-Modified', http_date(0)] + ] + } +} + +function make_test(raw_requests) { + var requests = []; + for (var i = 0; i < raw_requests.length; i++) { + var request = raw_requests[i]; + if ("template" in request) { + var template = templates[request["template"]]; + for (var member in template) { + if (! request.hasOwnProperty(member)) { + request[member] = template[member]; + } + } + } + if ("expected_type" in request && request.expected_type === "cached") { + // requests after one that's expected to be cached will get out of sync + // with the server; not currently supported. + if (raw_requests.length > i + 1) { + assert_unreached("Making requests after something is expected to be cached."); + } + } + requests.push(request); + } + return function(test) { + var uuid = token(); + var fetch_functions = []; + for (var i = 0; i < requests.length; ++i) { + fetch_functions.push({ + code: function(idx) { + var init = {}; + var url = make_url(uuid, requests, idx); + var config = requests[idx]; + if ("request_method" in config) { + init.method = config["request_method"]; + } + if ("request_headers" in config) { + init.headers = config["request_headers"]; + } + if ("request_body" in config) { + init.body = config["request_body"]; + } + if ("mode" in config) { + init.mode = config["mode"]; + } + if ("credentials" in config) { + init.mode = config["credentials"]; + } + if ("cache" in config) { + init.cache = config["cache"]; + } + console.log(url, init) + return fetch(url, init.cache) + .then(function(response) { + var res_num = parseInt(response.headers.get("Server-Request-Count")); + var req_num = idx + 1; + if ("expected_type" in config) { + if (config.expected_type === "error") { + assert_true(false, "Request " + req_num + " should have been an error"); + return [response.text(), response_status]; + } + if (config.expected_type === "cached") { + assert_less_than(res_num, req_num, "Response used"); + } + if (config.expected_type === "not_cached") { + assert_equals(res_num, req_num, "Response used"); + } + } + if ("expected_status" in config) { + assert_equals(response.status, config.expected_status, "Response status"); + } else if ("response_status" in config) { + assert_equals(response.status, config.response_status[0], "Response status"); + } else { + assert_equals(response.status, 200, "Response status") + } + if ("response_headers" in config) { + config.response_headers.forEach(function(header) { + if (header.len < 3 || header[2] === true) { + assert_equals(response.headers.get(header[0]), header[1], "Response header") + } + }) + } + if ("expected_response_headers" in config) { + config.expected_response_headers.forEach(function(header) { + assert_equals(response.headers.get(header[0]), header[1], "Response header"); + }); + } + return response.text(); + }).then(function(res_body) { + if ("expected_response_text" in config) { + assert_equals(res_body, config.expected_response_text, "Response body"); + } else if ("response_body" in config) { + assert_equals(res_body, config.response_body, "Response body"); + } else { + assert_equals(res_body, uuid, "Response body"); + } + }, function(reason) { + if ("expected_type" in config && config.expected_type === "error") { + assert_throws(new TypeError(), function() { throw reason; }); + } else { + throw reason; + } + }); + }, + pause_after: "pause_after" in requests[i] && true || false + }); + } + + function pause() { + return new Promise(function(resolve, reject) { + step_timeout(function() { + return resolve() + }, 3000); + }); + } + + // TODO: it would be nice if this weren't serialised. + var idx = 0; + function run_next_step() { + if (fetch_functions.length) { + var fetch_function = fetch_functions.shift(); + if (fetch_function.pause_after > 0) { + return fetch_function.code(idx++) + .then(pause) + .then(run_next_step); + } else { + return fetch_function.code(idx++) + .then(run_next_step); + } + } else { + return Promise.resolve(); + } + } + + return run_next_step() + .then(function() { + // Now, query the server state + return server_state(uuid); + }).then(function(state) { + for (var i = 0; i < requests.length; ++i) { + var expected_validating_headers = [] + var req_num = i + 1; + if ("expected_type" in requests[i]) { + if (requests[i].expected_type === "cached") { + assert_true(state.length <= i, "cached response used for request " + req_num); + continue; // the server will not see the request, so we can't check anything else. + } + if (requests[i].expected_type === "not_cached") { + assert_false(state.length <= i, "cached response used for request " + req_num); + } + if (requests[i].expected_type === "etag_validated") { + expected_validating_headers.push('if-none-match') + } + if (requests[i].expected_type === "lm_validated") { + expected_validating_headers.push('if-modified-since') + } + } + for (var j in expected_validating_headers) { + var vhdr = expected_validating_headers[j]; + assert_own_property(state[i].request_headers, vhdr, " has " + vhdr + " request header"); + } + if ("expected_request_headers" in requests[i]) { + var expected_request_headers = requests[i].expected_request_headers; + for (var j = 0; j < expected_request_headers.length; ++j) { + var expected_header = expected_request_headers[j]; + assert_equals(state[i].request_headers[expected_header[0].toLowerCase()], + expected_header[1]); + } + } + } + }); + }; +} + + +function run_tests(tests) +{ + tests.forEach(function(info) { + promise_test(make_test(info.requests), info.name); + }); +} + +function http_date(delta) { + return new Date(Date.now() + (delta * 1000)).toGMTString(); +} + +var content_store = {}; +function http_content(cs_key) { + if (cs_key in content_store) { + return content_store[cs_key]; + } else { + var content = btoa(Math.random() * Date.now()); + content_store[cs_key] = content; + return content; + } +} \ No newline at end of file