diff --git a/components/net/http_cache.rs b/components/net/http_cache.rs index cfb2c8acc01..764fa3651df 100644 --- a/components/net/http_cache.rs +++ b/components/net/http_cache.rs @@ -4,497 +4,650 @@ #![deny(missing_docs)] -//! A non-validating memory cache that only evicts expired entries and grows -//! without bound. Implements the logic specified in http://tools.ietf.org/html/rfc7234 -//! and http://tools.ietf.org/html/rfc7232. - -use http_loader::send_error_direct; -use resource_task::{Metadata, ProgressMsg, LoadResponse, LoadData, Payload, Done, start_sending_opt}; - -use servo_util::time::parse_http_timestamp; - -use http::headers::etag::EntityTag; -use http::headers::HeaderEnum; -use http::headers::response::HeaderCollection as ResponseHeaderCollection; -use http::method::Get; -use http::status::Ok as StatusOk; +//! A memory cache implementing the logic specified in http://tools.ietf.org/html/rfc7234 +//! and . +use fetch::methods::DoneChannel; +use http_loader::is_redirect_status; +use hyper::header; +use hyper::header::ContentType; +use hyper::header::Headers; +use hyper::method::Method; +use hyper::status::StatusCode; +use hyper_serde::Serde; +use net_traits::{Metadata, FetchMetadata}; +use net_traits::request::Request; +use net_traits::response::{HttpsState, Response, ResponseBody}; +use servo_url::ServoUrl; use std::collections::HashMap; -use std::comm::Sender; -use std::iter::Map; -use std::mem; -use std::num::{Bounded, FromStrRadix}; -use std::str::CharSplits; +use std::str; use std::sync::{Arc, Mutex}; -use std::time::duration::{MAX, Duration}; use time; -use time::{Tm, Timespec}; -use url::Url; +use time::{Duration, Tm}; -//TODO: Store an Arc> instead? -//TODO: Cache HEAD requests -//TODO: Doom responses with network errors -//TODO: Send Err responses for doomed entries -//TODO: Enable forced eviction of a request instead of retrieving the cached response -//TODO: Doom incomplete entries -//TODO: Cache-Control: must-revalidate -//TODO: Last-Modified -//TODO: Range requests -//TODO: Revalidation rules for query strings -//TODO: Vary header /// The key used to differentiate requests in the cache. -#[deriving(Clone, Hash, PartialEq, Eq)] +#[derive(Clone, Eq, Hash, PartialEq)] pub struct CacheKey { - url: Url, - request_headers: Vec<(String, String)>, + url: ServoUrl } impl CacheKey { - fn new(load_data: LoadData) -> CacheKey { + fn new(request: Request) -> CacheKey { CacheKey { - url: load_data.url.clone(), - request_headers: load_data.headers - .iter() - .map(|header| (header.header_name(), header.header_value())) - .collect(), + url: request.url().clone() + } + } + + fn from_servo_url(servo_url: &ServoUrl) -> CacheKey { + CacheKey { + url: servo_url.clone() } } /// Retrieve the URL associated with this key - pub fn url(&self) -> Url { + pub fn url(&self) -> ServoUrl { self.url.clone() } } -/// The list of consumers waiting on this requests's response. -enum PendingConsumers { - /// Consumers awaiting the initial response metadata - AwaitingHeaders(Vec>), - /// Consumers awaiting the remaining response body. Incomplete body stored as Vec. - AwaitingBody(Metadata, Vec, Vec>), -} - -/// An unfulfilled request representing both the consumers waiting for the initial -/// metadata and the subsequent response body. If doomed, the entry will be removed -/// after the final payload. -struct PendingResource { - consumers: PendingConsumers, - expires: Duration, - last_validated: Tm, - doomed: bool, -} - /// A complete cached resource. +#[derive(Clone)] struct CachedResource { - metadata: Metadata, - body: Vec, + metadata: CachedMetadata, + request_headers: Arc>, + body: Arc>, + https_state: HttpsState, + status: Option, + raw_status: Option<(u16, Vec)>, + url_list: Vec, expires: Duration, - last_validated: Tm, - revalidating_consumers: Vec>, + last_validated: Tm } -/// A memory cache that tracks incomplete and complete responses, differentiated by -/// the initial request. -pub struct MemoryCache { - /// Complete cached responses. - complete_entries: HashMap, - /// Incomplete cached responses. - pending_entries: HashMap, - /// The time at which this cache was created for use by expiry checks. - base_time: Timespec, +/// Metadata about a loaded resource, such as is obtained from HTTP headers. +#[derive(Clone)] +struct CachedMetadata { + /// Final URL after redirects. + pub final_url: ServoUrl, + /// MIME type / subtype. + pub content_type: Option>, + /// Character set. + pub charset: Option, + /// Headers + pub headers: Arc>, + /// HTTP Status + pub status: Option<(u16, Vec)> } -/// Abstraction over the concept of a single target for HTTP response messages. -pub enum ResourceResponseTarget { - /// A response is being streamed into the cache. - CachedPendingResource(CacheKey, Arc>), - /// A response is being streamed directly to a consumer and skipping the cache. - UncachedPendingResource(Sender), +/// Wrapper around a cached response, including information on re-validation needs +pub struct CachedResponse { + /// The response constructed from the cached resource + pub response: Response, + /// The revalidation flag for the stored response + pub needs_validation: bool } -/// Abstraction over the concept of a single target for HTTP response payload messages. -pub enum ResourceProgressTarget { - /// A response is being streamed into the cache. - CachedInProgressResource(CacheKey, Arc>), - /// A response is being streamed directly to a consumer and skipping the cache. - UncachedInProgressResource(Sender), +/// A memory cache. +pub struct HttpCache { + /// cached responses. + entries: HashMap>, } -/// The result of matching a request against an HTTP cache. -pub enum CacheOperationResult { - /// The request cannot be cached for a given reason. - Uncacheable(&'static str), - /// The request is in the cache and the response data is forthcoming. - CachedContentPending, - /// The request is not present in the cache but will be cached with the given key. - NewCacheEntry(CacheKey), - /// The request is in the cache but requires revalidation. - Revalidate(CacheKey, RevalidationMethod), -} - -/// The means by which to revalidate stale cached content -pub enum RevalidationMethod { - /// The result of a stored Last-Modified or Expires header - ExpiryDate(Tm), - /// The result of a stored Etag header - Etag(EntityTag), -} - -/// Tokenize a header value. -fn split_header(header: &str) -> Map<&str, &str, CharSplits> { - header.split(',') - .map(|v| v.trim()) -} - -/// Match any header value token. -fn any_token_matches(header: &str, tokens: &[&str]) -> bool { - split_header(header).any(|token| tokens.iter().any(|&s| s == token)) -} /// Determine if a given response is cacheable based on the initial metadata received. -/// Based on http://tools.ietf.org/html/rfc7234#section-5 +/// Based on fn response_is_cacheable(metadata: &Metadata) -> bool { - if metadata.status != StatusOk { + // TODO: if we determine that this cache should be considered shared: + // 1. check for absence of private response directive + // 2. check for absence of the Authorization header field. + let mut is_cacheable = false; + let headers = metadata.headers.as_ref().unwrap(); + if headers.has::() || + headers.has::() || + headers.has::() { + is_cacheable = true; + } + if let Some(&header::CacheControl(ref directive)) = headers.get::() { + for directive in directive.iter() { + match *directive { + header::CacheDirective::NoStore => return false, + header::CacheDirective::Public | header::CacheDirective::SMaxAge(_) + | header::CacheDirective::MaxAge(_) | header::CacheDirective::NoCache => is_cacheable = true, + _ => {}, + } + } + } + if let Some(&header::Pragma::NoCache) = headers.get::() { return false; } - - if metadata.headers.is_none() { - return true; - } - - let headers = metadata.headers.as_ref().unwrap(); - match headers.cache_control { - Some(ref cache_control) => { - if any_token_matches(cache_control[], &["no-cache", "no-store", "max-age=0"]) { - return false; - } - } - None => () - } - - match headers.pragma { - Some(ref pragma) => { - if any_token_matches(pragma[], &["no-cache"]) { - return false; - } - } - None => () - } - - return true; + is_cacheable } -/// Determine the expiry date of the given response headers. -/// Returns a far-future date if the response does not expire. -fn get_response_expiry_from_headers(headers: &ResponseHeaderCollection) -> Duration { - headers.cache_control.as_ref().and_then(|cache_control| { - for token in split_header(cache_control[]) { - let mut parts = token.split('='); - if parts.next() == Some("max-age") { - return parts.next() - .and_then(|val| FromStrRadix::from_str_radix(val, 10)) - .map(|secs| Duration::seconds(secs)); +/// Calculating Age +/// +fn calculate_response_age(response: &Response) -> Duration { + // TODO: follow the spec more closely (Date headers, request/response lag, ...) + if let Some(secs) = response.headers.get_raw("Age") { + let seconds_string = String::from_utf8_lossy(&secs[0]); + if let Ok(secs) = seconds_string.parse::() { + return Duration::seconds(secs); + } + } + Duration::seconds(0i64) +} + +/// Determine the expiry date from relevant headers, +/// or uses a heuristic if none are present. +fn get_response_expiry(response: &Response) -> Duration { + // Calculating Freshness Lifetime + let age = calculate_response_age(&response); + if let Some(&header::CacheControl(ref directives)) = response.headers.get::() { + let has_no_cache_directive = directives.iter().any(|directive| { + header::CacheDirective::NoCache == *directive + }); + if has_no_cache_directive { + // Requires validation on first use. + return Duration::seconds(0i64); + } else { + for directive in directives { + match *directive { + header::CacheDirective::SMaxAge(secs) | header::CacheDirective::MaxAge(secs) => { + let max_age = Duration::seconds(secs as i64); + if max_age < age { + return Duration::seconds(0i64); + } + return max_age - age; + }, + _ => (), + } + } + } + } + if let Some(&header::Expires(header::HttpDate(t))) = response.headers.get::() { + // store the period of time from now until expiry + let desired = t.to_timespec(); + let current = time::now().to_timespec(); + if desired > current { + return desired - current; + } else { + return Duration::seconds(0i64); + } + } else { + if let Some(_) = response.headers.get_raw("Expires") { + // Malformed Expires header, shouldn't be used to construct a valid response. + return Duration::seconds(0i64); + } + } + // Calculating Heuristic Freshness + // + if let Some((ref code, _)) = response.raw_status { + // + // Since presently we do not generate a Warning header field with a 113 warn-code, + // 24 hours minus response age is the max for heuristic calculation. + let max_heuristic = Duration::hours(24) - age; + let heuristic_freshness = if let Some(&header::LastModified(header::HttpDate(t))) = + // If the response has a Last-Modified header field, + // caches are encouraged to use a heuristic expiration value + // that is no more than some fraction of the interval since that time. + response.headers.get::() { + let last_modified = t.to_timespec(); + let current = time::now().to_timespec(); + // A typical setting of this fraction might be 10%. + let raw_heuristic_calc = (current - last_modified) / 10; + let result = if raw_heuristic_calc < max_heuristic { + raw_heuristic_calc + } else { + max_heuristic + }; + result + } else { + max_heuristic + }; + match *code { + 200 | 203 | 204 | 206 | 300 | 301 | 404 | 405 | 410 | 414 | 501 => { + // Status codes that are cacheable by default + return heuristic_freshness + }, + _ => { + // Other status codes can only use heuristic freshness if the public cache directive is present. + if let Some(&header::CacheControl(ref directives)) = response.headers.get::() { + let has_public_directive = directives.iter().any(|directive| { + header::CacheDirective::Public == *directive + }); + if has_public_directive { + return heuristic_freshness; + } + } + }, + } + } + // Requires validation upon first use as default. + Duration::seconds(0i64) +} + +/// Request Cache-Control Directives +/// +fn get_expiry_adjustment_from_request_headers(request: &Request, expires: Duration) -> Duration { + let directive_data = match request.headers.get_raw("cache-control") { + Some(data) => data, + None => return expires, + }; + let directives_string = String::from_utf8_lossy(&directive_data[0]); + for directive in directives_string.split(",") { + let mut directive_info = directive.split("="); + match (directive_info.next(), directive_info.next()) { + (Some("max-stale"), Some(sec_str)) => { + if let Ok(secs) = sec_str.parse::() { + return expires + Duration::seconds(secs); + } + }, + (Some("max-age"), Some(sec_str)) => { + if let Ok(secs) = sec_str.parse::() { + let max_age = Duration::seconds(secs); + if expires > max_age { + return Duration::min_value(); + } + return expires - max_age; + } + }, + (Some("min-fresh"), Some(sec_str)) => { + if let Ok(secs) = sec_str.parse::() { + let min_fresh = Duration::seconds(secs); + if expires < min_fresh { + return Duration::min_value(); + } + return expires - min_fresh; + } + }, + (Some("no-cache"), _) | (Some("no-store"), _) => return Duration::min_value(), + _ => {} + } + } + expires +} + +/// Create a CachedResponse from a request and a CachedResource. +fn create_cached_response(request: &Request, cached_resource: &CachedResource, cached_headers: &Headers) + -> CachedResponse { + let mut response = Response::new(cached_resource.metadata.final_url.clone()); + response.headers = cached_headers.clone(); + response.body = cached_resource.body.clone(); + response.status = cached_resource.status.clone(); + response.raw_status = cached_resource.raw_status.clone(); + response.url_list = cached_resource.url_list.clone(); + response.https_state = cached_resource.https_state.clone(); + response.referrer = request.referrer.to_url().cloned(); + response.referrer_policy = request.referrer_policy.clone(); + let expires = cached_resource.expires; + let adjusted_expires = get_expiry_adjustment_from_request_headers(request, expires); + let now = Duration::seconds(time::now().to_timespec().sec); + let last_validated = Duration::seconds(cached_resource.last_validated.to_timespec().sec); + let time_since_validated = now - last_validated; + // TODO: take must-revalidate into account + // TODO: if this cache is to be considered shared, take proxy-revalidate into account + // + let has_expired = (adjusted_expires < time_since_validated) || + (adjusted_expires == time_since_validated); + CachedResponse { response: response, needs_validation: has_expired } +} + +/// Create a new resource, based on the bytes requested, and an existing resource, +/// with a status-code of 206. +fn create_resource_with_bytes_from_resource(bytes: &[u8], resource: &CachedResource) + -> CachedResource { + CachedResource { + metadata: resource.metadata.clone(), + request_headers: resource.request_headers.clone(), + body: Arc::new(Mutex::new(ResponseBody::Done(bytes.to_owned()))), + https_state: resource.https_state.clone(), + status: Some(StatusCode::PartialContent), + raw_status: Some((206, b"Partial Content".to_vec())), + url_list: resource.url_list.clone(), + expires: resource.expires.clone(), + last_validated: resource.last_validated.clone() + } +} + +/// Support for range requests . +fn handle_range_request(request: &Request, candidates: Vec<&CachedResource>, range_spec: &[header::ByteRangeSpec]) + -> Option { + let mut complete_cached_resources = candidates.iter().filter(|resource| { + match resource.raw_status { + Some((ref code, _)) => *code == 200, + None => false + } + }); + let partial_cached_resources = candidates.iter().filter(|resource| { + match resource.raw_status { + Some((ref code, _)) => *code == 206, + None => false + } + }); + match (range_spec.first().unwrap(), complete_cached_resources.next()) { + // TODO: take the full range spec into account. + // If we have a complete resource, take the request range from the body. + // When there isn't a complete resource available, we loop over cached partials, + // and see if any individual partial response can fulfill the current request for a bytes range. + // TODO: combine partials that in combination could satisfy the requested range? + // see . + // TODO: add support for complete and partial resources, + // whose body is in the ResponseBody::Receiving state. + (&header::ByteRangeSpec::FromTo(beginning, end), Some(ref complete_resource)) => { + if let ResponseBody::Done(ref body) = *complete_resource.body.lock().unwrap() { + let b = beginning as usize; + let e = end as usize + 1; + let requested = body.get(b..e); + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(bytes, complete_resource); + let cached_headers = new_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, &new_resource, &*cached_headers); + return Some(cached_response); + } + } + }, + (&header::ByteRangeSpec::FromTo(beginning, end), None) => { + for partial_resource in partial_cached_resources { + let headers = partial_resource.metadata.headers.lock().unwrap(); + let content_range = headers.get::(); + let (res_beginning, res_end) = match content_range { + Some(&header::ContentRange( + header::ContentRangeSpec::Bytes { + range: Some((res_beginning, res_end)), .. })) => (res_beginning, res_end), + _ => continue, + }; + if res_beginning - 1 < beginning && res_end + 1 > end { + let resource_body = &*partial_resource.body.lock().unwrap(); + let requested = match resource_body { + &ResponseBody::Done(ref body) => { + let b = beginning as usize - res_beginning as usize; + let e = end as usize - res_beginning as usize + 1; + body.get(b..e) + }, + _ => continue, + }; + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(&bytes, partial_resource); + let cached_response = create_cached_response(request, &new_resource, &*headers); + return Some(cached_response); + } + } + } + }, + (&header::ByteRangeSpec::AllFrom(beginning), Some(ref complete_resource)) => { + if let ResponseBody::Done(ref body) = *complete_resource.body.lock().unwrap() { + let b = beginning as usize; + let requested = body.get(b..); + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(bytes, complete_resource); + let cached_headers = new_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, &new_resource, &*cached_headers); + return Some(cached_response); + } + } + }, + (&header::ByteRangeSpec::AllFrom(beginning), None) => { + for partial_resource in partial_cached_resources { + let headers = partial_resource.metadata.headers.lock().unwrap(); + let content_range = headers.get::(); + let (res_beginning, res_end, total) = match content_range { + Some(&header::ContentRange( + header::ContentRangeSpec::Bytes { + range: Some((res_beginning, res_end)), + instance_length: Some(total) })) => (res_beginning, res_end, total), + _ => continue, + }; + if res_beginning < beginning && res_end == total - 1 { + let resource_body = &*partial_resource.body.lock().unwrap(); + let requested = match resource_body { + &ResponseBody::Done(ref body) => { + let from_byte = beginning as usize - res_beginning as usize; + body.get(from_byte..) + }, + _ => continue, + }; + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(&bytes, partial_resource); + let cached_response = create_cached_response(request, &new_resource, &*headers); + return Some(cached_response); + } + } + } + }, + (&header::ByteRangeSpec::Last(offset), Some(ref complete_resource)) => { + if let ResponseBody::Done(ref body) = *complete_resource.body.lock().unwrap() { + let from_byte = body.len() - offset as usize; + let requested = body.get(from_byte..); + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(bytes, complete_resource); + let cached_headers = new_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, &new_resource, &*cached_headers); + return Some(cached_response); + } + } + }, + (&header::ByteRangeSpec::Last(offset), None) => { + for partial_resource in partial_cached_resources { + let headers = partial_resource.metadata.headers.lock().unwrap(); + let content_range = headers.get::(); + let (res_beginning, res_end, total) = match content_range { + Some(&header::ContentRange( + header::ContentRangeSpec::Bytes { + range: Some((res_beginning, res_end)), + instance_length: Some(total) })) => (res_beginning, res_end, total), + _ => continue, + }; + if (total - res_beginning) > (offset - 1 ) && (total - res_end) < offset + 1 { + let resource_body = &*partial_resource.body.lock().unwrap(); + let requested = match resource_body { + &ResponseBody::Done(ref body) => { + let from_byte = body.len() - offset as usize; + body.get(from_byte..) + }, + _ => continue, + }; + if let Some(bytes) = requested { + let new_resource = create_resource_with_bytes_from_resource(&bytes, partial_resource); + let cached_response = create_cached_response(request, &new_resource, &*headers); + return Some(cached_response); + } + } + } + } + } + None +} + + +impl HttpCache { + /// Create a new memory cache instance. + pub fn new() -> HttpCache { + HttpCache { + entries: HashMap::new() + } + } + + /// Constructing Responses from Caches. + /// + pub fn construct_response(&self, request: &Request) -> Option { + // TODO: generate warning headers as appropriate + if request.method != Method::Get { + // Only Get requests are cached, avoid a url based match for others. + return None; + } + let entry_key = CacheKey::new(request.clone()); + let resources = match self.entries.get(&entry_key) { + Some(ref resources) => resources.clone(), + None => return None, + }; + let mut candidates = vec![]; + for cached_resource in resources.iter() { + let mut can_be_constructed = true; + let cached_headers = cached_resource.metadata.headers.lock().unwrap(); + let original_request_headers = cached_resource.request_headers.lock().unwrap(); + if let Some(vary_data) = cached_headers.get_raw("Vary") { + // Calculating Secondary Keys with Vary + let vary_data_string = String::from_utf8_lossy(&vary_data[0]); + let vary_values = vary_data_string.split(",").map(|val| val.trim()); + for vary_val in vary_values { + // For every header name found in the Vary header of the stored response. + if vary_val == "*" { + // A Vary header field-value of "*" always fails to match. + can_be_constructed = false; + break; + } + match request.headers.get_raw(vary_val) { + Some(header_data) => { + // If the header is present in the request. + let request_header_data_string = String::from_utf8_lossy(&header_data[0]); + if let Some(original_header_data) = original_request_headers.get_raw(vary_val) { + // Check that the value of the nominated header field, + // in the original request, matches the value in the current request. + let original_request_header_data_string = + String::from_utf8_lossy(&original_header_data[0]); + if original_request_header_data_string != request_header_data_string { + can_be_constructed = false; + break; + } + } + }, + None => { + // If a header field is absent from a request, + // it can only match a stored response if those headers, + // were also absent in the original request. + can_be_constructed = original_request_headers.get_raw(vary_val).is_none(); + }, + } + if !can_be_constructed { + break; + } + } + } + if can_be_constructed { + candidates.push(cached_resource); + } + } + // Support for range requests + if let Some(&header::Range::Bytes(ref range_spec)) = request.headers.get::() { + return handle_range_request(request, candidates, &range_spec); + } else { + // Not a Range request. + if let Some(ref cached_resource) = candidates.first() { + // Returning the first response that can be constructed + // TODO: select the most appropriate one, using a known mechanism from a selecting header field, + // or using the Date header to return the most recent one. + let cached_headers = cached_resource.metadata.headers.lock().unwrap(); + let cached_response = create_cached_response(request, cached_resource, &*cached_headers); + return Some(cached_response); } } None - }).or_else(|| { - headers.expires.as_ref().and_then(|expires| { - parse_http_timestamp(expires[]).map(|t| { - // store the period of time from now until expiry - let desired = t.to_timespec(); - let current = time::now().to_timespec(); - if desired > current { - desired - current - } else { - Bounded::min_value() - } - }) - }) - }).unwrap_or(Bounded::max_value()) -} - -/// Determine the expiry date of the given response. -/// Returns a far-future date if this response does not expire. -fn get_response_expiry(metadata: &Metadata) -> Duration { - metadata.headers.as_ref().map(|headers| { - get_response_expiry_from_headers(headers) - }).unwrap_or(Bounded::max_value()) -} - -impl MemoryCache { - /// Create a new memory cache instance. - pub fn new() -> MemoryCache { - MemoryCache { - complete_entries: HashMap::new(), - pending_entries: HashMap::new(), - base_time: time::now().to_timespec(), - } } - /// Process a revalidation that returned new content for an expired entry. - pub fn process_revalidation_failed(&mut self, key: &CacheKey) { - debug!("recreating entry for {} (cache entry expired)", key.url); - let resource = self.complete_entries.remove(key).unwrap(); - self.add_pending_cache_entry(key.clone(), resource.revalidating_consumers); - } - - /// Mark an incomplete cached request as doomed. Any waiting consumers will immediately - /// receive an error message or a final body payload. The cache entry is immediately - /// removed. - pub fn doom_request(&mut self, key: &CacheKey, err: String) { - debug!("dooming entry for {} ({})", key.url, err); - - assert!(!self.complete_entries.contains_key(key)); - - let resource = self.pending_entries.remove(key).unwrap(); - match resource.consumers { - AwaitingHeaders(ref consumers) => { - for consumer in consumers.iter() { - send_error_direct(key.url.clone(), err.clone(), consumer.clone()); - } + /// Freshening Stored Responses upon Validation. + /// + pub fn refresh(&mut self, request: &Request, response: Response, done_chan: &mut DoneChannel) -> Option { + assert!(response.status == Some(StatusCode::NotModified)); + let entry_key = CacheKey::new(request.clone()); + if let Some(cached_resources) = self.entries.get_mut(&entry_key) { + for cached_resource in cached_resources.iter_mut() { + let mut stored_headers = cached_resource.metadata.headers.lock().unwrap(); + // Received a response with 304 status code, in response to a request that matches a cached resource. + // 1. update the headers of the cached resource. + // 2. return a response, constructed from the cached resource. + stored_headers.extend(response.headers.iter()); + let mut constructed_response = Response::new(cached_resource.metadata.final_url.clone()); + constructed_response.headers = stored_headers.clone(); + constructed_response.body = cached_resource.body.clone(); + constructed_response.status = cached_resource.status.clone(); + constructed_response.https_state = cached_resource.https_state.clone(); + constructed_response.referrer = request.referrer.to_url().cloned(); + constructed_response.referrer_policy = request.referrer_policy.clone(); + constructed_response.raw_status = cached_resource.raw_status.clone(); + constructed_response.url_list = cached_resource.url_list.clone(); + // done_chan will have been set to Some by http_network_fetch, + // set it back to None since the response returned here replaces the 304 one from the network. + *done_chan = None; + cached_resource.expires = get_response_expiry(&constructed_response); + return Some(constructed_response); } - AwaitingBody(_, _, ref consumers) => { - for consumer in consumers.iter() { - let _ = consumer.send_opt(Done(Ok(()))); - } + } + None + } + + fn invalidate_for_url(&mut self, url: &ServoUrl) { + let entry_key = CacheKey::from_servo_url(url); + if let Some(cached_resources) = self.entries.get_mut(&entry_key) { + for cached_resource in cached_resources.iter_mut() { + cached_resource.expires = Duration::seconds(0i64); } } } - /// Handle a 304 response to a revalidation request. Updates the cached response - /// metadata with any new expiration data. - pub fn process_not_modified(&mut self, key: &CacheKey, headers: &ResponseHeaderCollection) { - debug!("updating metadata for {}", key.url); - let resource = self.complete_entries.get_mut(key).unwrap(); - resource.expires = get_response_expiry_from_headers(headers); - - for consumer in mem::replace(&mut resource.revalidating_consumers, vec!()).into_iter() { - MemoryCache::send_complete_resource(resource, consumer); + /// Invalidation. + /// + pub fn invalidate(&mut self, request: &Request, response: &Response) { + if let Some(&header::Location(ref location)) = response.headers.get::() { + if let Ok(url) = request.current_url().join(location) { + self.invalidate_for_url(&url); + } } + // TODO: update hyper to use typed getter. + if let Some(url_data) = response.headers.get_raw("Content-Location") { + if let Ok(content_location) = str::from_utf8(&url_data[0]) { + if let Ok(url) = request.current_url().join(content_location) { + self.invalidate_for_url(&url); + } + } + } + self.invalidate_for_url(&request.url()); } - /// Handle the initial response metadata for an incomplete cached request. - /// If the response should not be cached, the entry will be doomed and any - /// subsequent requests will not see the cached request. All waiting consumers - /// will see the new metadata. - pub fn process_metadata(&mut self, key: &CacheKey, metadata: Metadata) { - debug!("storing metadata for {}", key.url); - let resource = self.pending_entries.get_mut(key).unwrap(); - let chans: Vec>; - match resource.consumers { - AwaitingHeaders(ref consumers) => { - chans = consumers.iter() - .map(|chan| start_sending_opt(chan.clone(), metadata.clone())) - .take_while(|chan| chan.is_ok()) - .map(|chan| chan.unwrap()) - .collect(); + /// Storing Responses in Caches. + /// + pub fn store(&mut self, request: &Request, response: &Response) { + if let Some(status) = response.status { + // Not caching redirects, for simplicity, not per the spec. + if is_redirect_status(status) { + return } - AwaitingBody(..) => panic!("obtained headers for {} but awaiting body?", key.url) } - + if request.method != Method::Get { + // Only Get requests are cached. + return + } + let entry_key = CacheKey::new(request.clone()); + let metadata = match response.metadata() { + Ok(FetchMetadata::Filtered { + filtered: _, + unsafe_: metadata }) | + Ok(FetchMetadata::Unfiltered(metadata)) => metadata, + _ => return, + }; if !response_is_cacheable(&metadata) { - resource.doomed = true; - } - - resource.expires = get_response_expiry(&metadata); - resource.last_validated = time::now(); - resource.consumers = AwaitingBody(metadata, vec!(), chans); - } - - /// Handle a repsonse body payload for an incomplete cached response. - /// All waiting consumers will see the new payload addition. - pub fn process_payload(&mut self, key: &CacheKey, payload: Vec) { - debug!("storing partial response for {}", key.url); - let resource = self.pending_entries.get_mut(key).unwrap(); - match resource.consumers { - AwaitingBody(_, ref mut body, ref consumers) => { - body.push_all(payload.as_slice()); - for consumer in consumers.iter() { - //FIXME: maybe remove consumer on failure to avoid extra clones? - let _ = consumer.send_opt(Payload(payload.clone())); - } - } - AwaitingHeaders(_) => panic!("obtained body for {} but awaiting headers?", key.url) - } - } - - /// Handle a response body final payload for an incomplete cached response. - /// All waiting consumers will see the new message. If the cache entry is - /// doomed, it will not be transferred to the set of complete cache entries. - pub fn process_done(&mut self, key: &CacheKey) { - debug!("finished fetching {}", key.url); - let resource = self.pending_entries.remove(key).unwrap(); - match resource.consumers { - AwaitingHeaders(_) => panic!("saw Done for {} but awaiting headers?", key.url), - AwaitingBody(_, _, ref consumers) => { - for consumer in consumers.iter() { - let _ = consumer.send_opt(Done(Ok(()))); - } - } - } - - if resource.doomed { - debug!("completing dooming of {}", key.url); return; } - - let (metadata, body) = match resource.consumers { - AwaitingBody(metadata, body, _) => (metadata, body), - _ => panic!("expected consumer list awaiting bodies"), + let expiry = get_response_expiry(&response); + let cacheable_metadata = CachedMetadata { + final_url: metadata.final_url, + content_type: metadata.content_type, + charset: metadata.charset, + status: metadata.status, + headers: Arc::new(Mutex::new(response.headers.clone())) }; - - let complete = CachedResource { - metadata: metadata, - body: body, - expires: resource.expires, - last_validated: resource.last_validated, - revalidating_consumers: vec!(), + let entry_resource = CachedResource { + metadata: cacheable_metadata, + request_headers: Arc::new(Mutex::new(request.headers.clone())), + body: response.body.clone(), + https_state: response.https_state.clone(), + status: response.status.clone(), + raw_status: response.raw_status.clone(), + url_list: response.url_list.clone(), + expires: expiry, + last_validated: time::now() }; - self.complete_entries.insert(key.clone(), complete); + let entry = self.entries.entry(entry_key).or_insert(vec![]); + entry.push(entry_resource); } - /// Match a new request against the set of incomplete and complete cached requests. - /// If the request matches an existing, non-doomed entry, any existing response data will - /// be synchronously streamed to the consumer. If the request does not match but can be - /// cached, a new cache entry will be created and the request will be responsible for - /// notifying the cache of the subsequent HTTP response. If the request does not match - /// and cannot be cached, the request is responsible for handling its own response and - /// consumer. - pub fn process_pending_request(&mut self, load_data: &LoadData, start_chan: Sender) - -> CacheOperationResult { - fn revalidate(resource: &mut CachedResource, - key: &CacheKey, - start_chan: Sender, - method: RevalidationMethod) -> CacheOperationResult { - // Ensure that at most one revalidation is taking place at a time for a - // cached resource. - resource.revalidating_consumers.push(start_chan); - if resource.revalidating_consumers.len() > 1 { - CachedContentPending - } else { - Revalidate(key.clone(), method) - } - } - - if load_data.method != Get { - return Uncacheable("Only GET requests can be cached."); - } - - let key = CacheKey::new(load_data.clone()); - match self.complete_entries.get_mut(&key) { - Some(resource) => { - if self.base_time + resource.expires < time::now().to_timespec() { - debug!("entry for {} has expired", key.url()); - let expiry = time::at(self.base_time + resource.expires); - return revalidate(resource, &key, start_chan, ExpiryDate(expiry)); - } - - let must_revalidate = resource.metadata.headers.as_ref().and_then(|headers| { - headers.cache_control.as_ref().map(|header| { - any_token_matches(header[], &["must-revalidate"]) - }) - }).unwrap_or(false); - - if must_revalidate { - debug!("entry for {} must be revalidated", key.url()); - let last_validated = resource.last_validated; - return revalidate(resource, &key, start_chan, ExpiryDate(last_validated)); - } - - let etag = resource.metadata.headers.as_ref().and_then(|headers| headers.etag.clone()); - match etag { - Some(etag) => { - debug!("entry for {} has an Etag", key.url()); - return revalidate(resource, &key, start_chan, Etag(etag.clone())); - } - None => () - } - - //TODO: Revalidate once per session for response with no explicit expiry - } - - None => () - } - - if self.complete_entries.contains_key(&key) { - self.send_complete_entry(key, start_chan); - return CachedContentPending; - } - - let new_entry = match self.pending_entries.get(&key) { - Some(resource) if resource.doomed => return Uncacheable("Cache entry already doomed"), - Some(_) => false, - None => true, - }; - - if new_entry { - self.add_pending_cache_entry(key.clone(), vec!(start_chan)); - NewCacheEntry(key) - } else { - self.send_partial_entry(key, start_chan); - CachedContentPending - } - } - - /// Add a new pending request to the set of incomplete cache entries. - fn add_pending_cache_entry(&mut self, key: CacheKey, consumers: Vec>) { - let resource = PendingResource { - consumers: AwaitingHeaders(consumers), - expires: MAX, - last_validated: time::now(), - doomed: false, - }; - debug!("creating cache entry for {}", key.url); - self.pending_entries.insert(key, resource); - } - - /// Synchronously send the entire cached response body to the given consumer. - fn send_complete_resource(resource: &CachedResource, start_chan: Sender) { - let progress_chan = start_sending_opt(start_chan, resource.metadata.clone()); - match progress_chan { - Ok(chan) => { - let _ = chan.send_opt(Payload(resource.body.clone())); - let _ = chan.send_opt(Done(Ok(()))); - } - Err(_) => () - } - } - - /// Synchronously send the entire cached response body to the given consumer. - fn send_complete_entry(&self, key: CacheKey, start_chan: Sender) { - debug!("returning full cache body for {}", key.url); - let resource = self.complete_entries.get(&key).unwrap(); - MemoryCache::send_complete_resource(resource, start_chan) - } - - /// Synchronously send all partial stored response data for a cached request to the - /// given consumer. - fn send_partial_entry(&mut self, key: CacheKey, start_chan: Sender) { - debug!("returning partial cache data for {}", key.url); - - let resource = self.pending_entries.get_mut(&key).unwrap(); - - match resource.consumers { - AwaitingHeaders(ref mut consumers) => { - consumers.push(start_chan); - } - AwaitingBody(ref metadata, ref body, ref mut consumers) => { - debug!("headers available for {}", key.url); - let progress_chan = start_sending_opt(start_chan, metadata.clone()); - match progress_chan { - Ok(chan) => { - consumers.push(chan.clone()); - - if !body.is_empty() { - debug!("partial body available for {}", key.url); - let _ = chan.send_opt(Payload(body.clone())); - } - } - - Err(_) => () - } - } - } - } } diff --git a/components/net/http_loader.rs b/components/net/http_loader.rs index 05fb6dbf1ee..0b7187bb8a6 100644 --- a/components/net/http_loader.rs +++ b/components/net/http_loader.rs @@ -13,6 +13,7 @@ use fetch::methods::{Data, DoneChannel, FetchContext, Target}; use fetch::methods::{is_cors_safelisted_request_header, is_cors_safelisted_method, main_fetch}; use flate2::read::{DeflateDecoder, GzDecoder}; use hsts::HstsList; +use http_cache::HttpCache; use hyper::Error as HttpError; use hyper::LanguageTag; use hyper::client::{Pool, Request as HyperRequest, Response as HyperResponse}; @@ -22,7 +23,7 @@ use hyper::header::{AccessControlMaxAge, AccessControlRequestHeaders}; use hyper::header::{AccessControlRequestMethod, AcceptEncoding, AcceptLanguage}; use hyper::header::{Authorization, Basic, CacheControl, CacheDirective}; use hyper::header::{ContentEncoding, ContentLength, Encoding, Header, Headers}; -use hyper::header::{Host, Origin as HyperOrigin, IfMatch, IfRange}; +use hyper::header::{Host, HttpDate, Origin as HyperOrigin, IfMatch, IfRange}; use hyper::header::{IfUnmodifiedSince, IfModifiedSince, IfNoneMatch, Location}; use hyper::header::{Pragma, Quality, QualityItem, Referer, SetCookie}; use hyper::header::{UserAgent, q, qitem}; @@ -45,6 +46,7 @@ use std::io::{self, Read, Write}; use std::iter::FromIterator; use std::mem; use std::ops::Deref; +use std::str::FromStr; use std::sync::RwLock; use std::sync::mpsc::{channel, Sender}; use std::thread; @@ -69,6 +71,7 @@ fn read_block(reader: &mut R) -> Result { pub struct HttpState { pub hsts_list: RwLock, pub cookie_jar: RwLock, + pub http_cache: RwLock, pub auth_cache: RwLock, pub ssl_client: OpensslClient, pub connector: Pool, @@ -80,6 +83,7 @@ impl HttpState { hsts_list: RwLock::new(HstsList::new()), cookie_jar: RwLock::new(CookieStorage::new(150)), auth_cache: RwLock::new(AuthCache::new()), + http_cache: RwLock::new(HttpCache::new()), ssl_client: ssl_client.clone(), connector: create_http_connector(ssl_client), } @@ -893,34 +897,35 @@ fn http_network_or_cache_fetch(request: &mut Request, let mut revalidating_flag = false; // Step 21 - // TODO have a HTTP cache to check for a completed response - let complete_http_response_from_cache: Option = None; - if http_request.cache_mode != CacheMode::NoStore && - http_request.cache_mode != CacheMode::Reload && - complete_http_response_from_cache.is_some() { - // TODO Substep 1 and 2. Select a response from HTTP cache. + if let Ok(http_cache) = context.state.http_cache.read() { + if let Some(response_from_cache) = http_cache.construct_response(&http_request) { + let response_headers = response_from_cache.response.headers.clone(); + // Substep 1, 2, 3, 4 + let (cached_response, needs_revalidation) = match (http_request.cache_mode, &http_request.mode) { + (CacheMode::ForceCache, _) => (Some(response_from_cache.response), false), + (CacheMode::OnlyIfCached, &RequestMode::SameOrigin) => (Some(response_from_cache.response), false), + (CacheMode::OnlyIfCached, _) | (CacheMode::NoStore, _) | (CacheMode::Reload, _) => (None, false), + (_, _) => (Some(response_from_cache.response), response_from_cache.needs_validation) + }; + if needs_revalidation { + revalidating_flag = true; + // Substep 5 + // TODO: find out why the typed header getter return None from the headers of cached responses. + if let Some(date_slice) = response_headers.get_raw("Last-Modified") { + let date_string = String::from_utf8_lossy(&date_slice[0]); + if let Ok(http_date) = HttpDate::from_str(&date_string) { + http_request.headers.set(IfModifiedSince(http_date)); + } + } + if let Some(entity_tag) = + response_headers.get_raw("ETag") { + http_request.headers.set_raw("If-None-Match", entity_tag.to_vec()); - // Substep 3 - if let Some(ref response) = response { - revalidating_flag = response_needs_revalidation(&response); - }; - - // Substep 4 - if http_request.cache_mode == CacheMode::ForceCache || - http_request.cache_mode == CacheMode::OnlyIfCached { - // TODO pull response from HTTP cache - // response = http_request - } - - if revalidating_flag { - // Substep 5 - // TODO set If-None-Match and If-Modified-Since according to cached - // response headers. - } else { - // Substep 6 - // TODO pull response from HTTP cache - // response = http_request - // response.cache_state = CacheState::Local; + } + } else { + // Substep 6 + response = cached_response; + } } } @@ -931,26 +936,37 @@ fn http_network_or_cache_fetch(request: &mut Request, return Response::network_error( NetworkError::Internal("Couldn't find response in cache".into())) } + } + // More Step 22 + if response.is_none() { // Substep 2 let forward_response = http_network_fetch(http_request, credentials_flag, done_chan, context); // Substep 3 if let Some((200...399, _)) = forward_response.raw_status { if !http_request.method.safe() { - // TODO Invalidate HTTP cache response + if let Ok(mut http_cache) = context.state.http_cache.write() { + http_cache.invalidate(&http_request, &forward_response); + } } } // Substep 4 if revalidating_flag && forward_response.status.map_or(false, |s| s == StatusCode::NotModified) { - // TODO update forward_response headers with cached response headers + if let Ok(mut http_cache) = context.state.http_cache.write() { + response = http_cache.refresh(&http_request, forward_response.clone(), done_chan); + } } // Substep 5 if response.is_none() { + if http_request.cache_mode != CacheMode::NoStore { + // Subsubstep 2, doing it first to avoid a clone of forward_response. + if let Ok(mut http_cache) = context.state.http_cache.write() { + http_cache.store(&http_request, &forward_response); + } + } // Subsubstep 1 response = Some(forward_response); - // Subsubstep 2 - // TODO: store http_request and forward_response in cache } } @@ -1168,7 +1184,9 @@ fn http_network_fetch(request: &Request, // Step 14 if !response.is_network_error() && request.cache_mode != CacheMode::NoStore { - // TODO update response in the HTTP cache for request + if let Ok(mut http_cache) = context.state.http_cache.write() { + http_cache.store(&request, &response); + } } // TODO this step isn't possible yet @@ -1366,11 +1384,6 @@ fn is_no_store_cache(headers: &Headers) -> bool { headers.has::() } -fn response_needs_revalidation(_response: &Response) -> bool { - // TODO this function - false -} - /// pub fn is_redirect_status(status: StatusCode) -> bool { match status { diff --git a/components/net/resource_task.rs b/components/net/resource_task.rs deleted file mode 100644 index ec6d49f065a..00000000000 --- a/components/net/resource_task.rs +++ /dev/null @@ -1,291 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! A task that takes a URL and streams back the binary data. - -use about_loader; -use data_loader; -use file_loader; -use http_cache::MemoryCache; -use http_loader; -use sniffer_task; - -use std::comm::{channel, Receiver, Sender}; -use std::sync::{Arc, Mutex}; -use http::headers::content_type::MediaType; -use http::headers::response::HeaderCollection as ResponseHeaderCollection; -use http::headers::request::HeaderCollection as RequestHeaderCollection; -use http::method::{Method, Get}; -use url::Url; - -use http::status::Ok as StatusOk; -use http::status::Status; - -use servo_util::task::spawn_named; - -pub enum ControlMsg { - /// Request the data associated with a particular URL - Load(LoadData, Sender), - Exit -} - -#[deriving(Clone)] -pub struct LoadData { - pub url: Url, - pub method: Method, - pub headers: RequestHeaderCollection, - pub data: Option>, - pub cors: Option -} - -impl LoadData { - pub fn new(url: Url) -> LoadData { - LoadData { - url: url, - method: Get, - headers: RequestHeaderCollection::new(), - data: None, - cors: None - } - } -} - -#[deriving(Clone)] -pub struct ResourceCORSData { - /// CORS Preflight flag - pub preflight: bool, - /// Origin of CORS Request - pub origin: Url -} - -/// Metadata about a loaded resource, such as is obtained from HTTP headers. -#[deriving(Clone)] -pub struct Metadata { - /// Final URL after redirects. - pub final_url: Url, - - /// MIME type / subtype. - pub content_type: Option<(String, String)>, - - /// Character set. - pub charset: Option, - - /// Headers - pub headers: Option, - - /// HTTP Status - pub status: Status -} - -impl Metadata { - /// Metadata with defaults for everything optional. - pub fn default(url: Url) -> Metadata { - Metadata { - final_url: url, - content_type: None, - charset: None, - headers: None, - status: StatusOk // http://fetch.spec.whatwg.org/#concept-response-status-message - } - } - - /// Extract the parts of a MediaType that we care about. - pub fn set_content_type(&mut self, content_type: &Option) { - match *content_type { - None => (), - Some(MediaType { ref type_, - ref subtype, - ref parameters }) => { - self.content_type = Some((type_.clone(), subtype.clone())); - for &(ref k, ref v) in parameters.iter() { - if "charset" == k.as_slice() { - self.charset = Some(v.clone()); - } - } - } - } - } -} - -/// Message sent in response to `Load`. Contains metadata, and a port -/// for receiving the data. -/// -/// Even if loading fails immediately, we send one of these and the -/// progress_port will provide the error. -pub struct LoadResponse { - /// Metadata, such as from HTTP headers. - pub metadata: Metadata, - /// Port for reading data. - pub progress_port: Receiver, -} - -/// Messages sent in response to a `Load` message -#[deriving(PartialEq,Show)] -pub enum ProgressMsg { - /// Binary data - there may be multiple of these - Payload(Vec), - /// Indicates loading is complete, either successfully or not - Done(Result<(), String>) -} - -/// For use by loaders in responding to a Load message. -pub fn start_sending(start_chan: Sender, metadata: Metadata) -> Sender { - start_sending_opt(start_chan, metadata).ok().unwrap() -} - -/// For use by loaders in responding to a Load message. -pub fn start_sending_opt(start_chan: Sender, metadata: Metadata) -> Result, ()> { - let (progress_chan, progress_port) = channel(); - let result = start_chan.send_opt(LoadResponse { - metadata: metadata, - progress_port: progress_port, - }); - match result { - Ok(_) => Ok(progress_chan), - Err(_) => Err(()) - } -} - -/// Convenience function for synchronously loading a whole resource. -pub fn load_whole_resource(resource_task: &ResourceTask, url: Url) - -> Result<(Metadata, Vec), String> { - let (start_chan, start_port) = channel(); - resource_task.send(Load(LoadData::new(url), start_chan)); - let response = start_port.recv(); - - let mut buf = vec!(); - loop { - match response.progress_port.recv() { - Payload(data) => buf.push_all(data.as_slice()), - Done(Ok(())) => return Ok((response.metadata, buf)), - Done(Err(e)) => return Err(e) - } - } -} - -/// Handle to a resource task -pub type ResourceTask = Sender; - -/// Create a ResourceTask -pub fn new_resource_task(user_agent: Option) -> ResourceTask { - let (setup_chan, setup_port) = channel(); - spawn_named("ResourceManager", proc() { - ResourceManager::new(setup_port, user_agent).start(); - }); - setup_chan -} - -struct ResourceManager { - from_client: Receiver, - user_agent: Option, - memory_cache: Arc>, -} - -impl ResourceManager { - fn new(from_client: Receiver, user_agent: Option) -> ResourceManager { - ResourceManager { - from_client: from_client, - user_agent: user_agent, - memory_cache: Arc::new(Mutex::new(MemoryCache::new())), - } - } -} - - -impl ResourceManager { - fn start(&self) { - loop { - match self.from_client.recv() { - Load(load_data, start_chan) => { - self.load(load_data, start_chan) - } - Exit => { - break - } - } - } - } - - fn load(&self, load_data: LoadData, start_chan: Sender) { - let mut load_data = load_data; - load_data.headers.user_agent = self.user_agent.clone(); - - // Create new communication channel, create new sniffer task, - // send all the data to the new sniffer task with the send - // end of the pipe, receive all the data. - - let sniffer_task = sniffer_task::new_sniffer_task(start_chan.clone()); - - fn from_factory<'a>(factory: fn(LoadData, Sender)) - -> proc(LoadData, Sender): 'a { - proc(load_data: LoadData, start_chan: Sender) { - factory(load_data, start_chan) - } - } - - let loader = match load_data.url.scheme.as_slice() { - "file" => from_factory(file_loader::factory), - "http" | "https" => http_loader::factory(self.memory_cache.clone()), - "data" => from_factory(data_loader::factory), - "about" => from_factory(about_loader::factory), - _ => { - debug!("resource_task: no loader for scheme {:s}", load_data.url.scheme); - start_sending(start_chan, Metadata::default(load_data.url)) - .send(Done(Err("no loader for scheme".to_string()))); - return - } - }; - debug!("resource_task: loading url: {:s}", load_data.url.serialize()); - - loader(load_data, sniffer_task); - } -} - -/// Load a URL asynchronously and iterate over chunks of bytes from the response. -pub fn load_bytes_iter(resource_task: &ResourceTask, url: Url) -> (Metadata, ProgressMsgPortIterator) { - let (input_chan, input_port) = channel(); - resource_task.send(Load(LoadData::new(url), input_chan)); - - let response = input_port.recv(); - let iter = ProgressMsgPortIterator { progress_port: response.progress_port }; - (response.metadata, iter) -} - -/// Iterator that reads chunks of bytes from a ProgressMsg port -pub struct ProgressMsgPortIterator { - progress_port: Receiver -} - -impl Iterator> for ProgressMsgPortIterator { - fn next(&mut self) -> Option> { - match self.progress_port.recv() { - Payload(data) => Some(data), - Done(Ok(())) => None, - Done(Err(e)) => { - error!("error receiving bytes: {}", e); - None - } - } - } -} - -#[test] -fn test_exit() { - let resource_task = new_resource_task(None); - resource_task.send(Exit); -} - -#[test] -fn test_bad_scheme() { - let resource_task = new_resource_task(None); - let (start_chan, start) = channel(); - let url = Url::parse("bogus://whatever").unwrap(); - resource_task.send(Load(LoadData::new(url), start_chan)); - let response = start.recv(); - match response.progress_port.recv() { - Done(result) => { assert!(result.is_err()) } - _ => panic!("bleh") - } - resource_task.send(Exit); -} diff --git a/components/net/resource_thread.rs b/components/net/resource_thread.rs index 9e1a3c9eee7..22366c562cd 100644 --- a/components/net/resource_thread.rs +++ b/components/net/resource_thread.rs @@ -12,6 +12,7 @@ use fetch::cors_cache::CorsCache; use fetch::methods::{FetchContext, fetch}; use filemanager_thread::{FileManager, TFDProvider}; use hsts::HstsList; +use http_cache::HttpCache; use http_loader::{HttpState, http_redirect_fetch}; use hyper_serde::Serde; use ipc_channel::ipc::{self, IpcReceiver, IpcReceiverSet, IpcSender}; @@ -91,6 +92,7 @@ struct ResourceChannelManager { fn create_http_states(config_dir: Option<&Path>) -> (Arc, Arc) { let mut hsts_list = HstsList::from_servo_preload(); let mut auth_cache = AuthCache::new(); + let http_cache = HttpCache::new(); let mut cookie_jar = CookieStorage::new(150); if let Some(config_dir) = config_dir { read_json_from_file(&mut auth_cache, config_dir, "auth_cache.json"); @@ -109,6 +111,7 @@ fn create_http_states(config_dir: Option<&Path>) -> (Arc, Arc String { - parse_http_timestamp(timestamp).map(|t| { - t.to_local().strftime("%m/%d/%Y %H:%M:%S").unwrap() - }).unwrap_or(String::new()) -} - -trait SinkHelpers { - fn get_or_create(&self, child: NodeOrText) -> Temporary; -} - -impl SinkHelpers for servohtmlparser::Sink { - fn get_or_create(&self, child: NodeOrText) -> Temporary { - match child { - AppendNode(n) => Temporary::new(unsafe { JS::from_trusted_node_address(n) }), - AppendText(t) => { - let doc = self.document.root(); - let text = Text::new(t, *doc); - NodeCast::from_temporary(text) - } - } - } -} - -impl<'a> TreeSink for servohtmlparser::Sink { - fn get_document(&mut self) -> TrustedNodeAddress { - let doc = self.document.root(); - let node: JSRef = NodeCast::from_ref(*doc); - node.to_trusted_node_address() - } - - fn same_node(&self, x: TrustedNodeAddress, y: TrustedNodeAddress) -> bool { - x == y - } - - fn elem_name(&self, target: TrustedNodeAddress) -> QualName { - let node: Root = unsafe { JS::from_trusted_node_address(target).root() }; - let elem: JSRef = ElementCast::to_ref(*node) - .expect("tried to get name of non-Element in HTML parsing"); - QualName { - ns: elem.get_namespace().clone(), - local: elem.get_local_name().clone(), - } - } - - fn create_element(&mut self, name: QualName, attrs: Vec) - -> TrustedNodeAddress { - let doc = self.document.root(); - let elem = Element::create(name, None, *doc, ParserCreated).root(); - - for attr in attrs.into_iter() { - elem.set_attribute_from_parser(attr.name, attr.value, None); - } - - let node: JSRef = NodeCast::from_ref(*elem); - node.to_trusted_node_address() - } - - fn create_comment(&mut self, text: String) -> TrustedNodeAddress { - let doc = self.document.root(); - let comment = Comment::new(text, *doc); - let node: Root = NodeCast::from_temporary(comment).root(); - node.to_trusted_node_address() - } - - fn append_before_sibling(&mut self, - sibling: TrustedNodeAddress, - new_node: NodeOrText) -> Result<(), NodeOrText> { - // If there is no parent, return the node to the parser. - let sibling: Root = unsafe { JS::from_trusted_node_address(sibling).root() }; - let parent = match sibling.parent_node() { - Some(p) => p.root(), - None => return Err(new_node), - }; - - let child = self.get_or_create(new_node).root(); - assert!(parent.InsertBefore(*child, Some(*sibling)).is_ok()); - Ok(()) - } - - fn parse_error(&mut self, msg: MaybeOwned<'static>) { - debug!("Parse error: {:s}", msg); - } - - fn set_quirks_mode(&mut self, mode: QuirksMode) { - let doc = self.document.root(); - doc.set_quirks_mode(mode); - } - - fn append(&mut self, parent: TrustedNodeAddress, child: NodeOrText) { - let parent: Root = unsafe { JS::from_trusted_node_address(parent).root() }; - let child = self.get_or_create(child).root(); - - // FIXME(#3701): Use a simpler algorithm and merge adjacent text nodes - assert!(parent.AppendChild(*child).is_ok()); - } - - fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) { - let doc = self.document.root(); - let doc_node: JSRef = NodeCast::from_ref(*doc); - let doctype = DocumentType::new(name, Some(public_id), Some(system_id), *doc); - let node: Root = NodeCast::from_temporary(doctype).root(); - - assert!(doc_node.AppendChild(*node).is_ok()); - } - - fn add_attrs_if_missing(&mut self, target: TrustedNodeAddress, attrs: Vec) { - let node: Root = unsafe { JS::from_trusted_node_address(target).root() }; - let elem: JSRef = ElementCast::to_ref(*node) - .expect("tried to set attrs on non-Element in HTML parsing"); - for attr in attrs.into_iter() { - elem.set_attribute_from_parser(attr.name, attr.value, None); - } - } - - fn remove_from_parent(&mut self, _target: TrustedNodeAddress) { - error!("remove_from_parent not implemented!"); - } - - fn mark_script_already_started(&mut self, node: TrustedNodeAddress) { - let node: Root = unsafe { JS::from_trusted_node_address(node).root() }; - let script: Option> = HTMLScriptElementCast::to_ref(*node); - script.map(|script| script.mark_already_started()); - } - - fn complete_script(&mut self, node: TrustedNodeAddress) { - let node: Root = unsafe { JS::from_trusted_node_address(node).root() }; - let script: Option> = HTMLScriptElementCast::to_ref(*node); - script.map(|script| script.prepare()); - } -} - -// The url from msg_load_data is ignored here -pub fn parse_html(page: &Page, - document: JSRef, - input: HTMLInput, - resource_task: ResourceTask, - msg_load_data: Option) { - let (base_url, load_response) = match input { - InputUrl(ref url) => { - // Wait for the LoadResponse so that the parser knows the final URL. - let (input_chan, input_port) = channel(); - let mut load_data = LoadData::new(url.clone()); - msg_load_data.map(|m| { - load_data.headers = m.headers; - load_data.method = m.method; - load_data.data = m.data; - }); - resource_task.send(Load(load_data, input_chan)); - - let load_response = input_port.recv(); - - load_response.metadata.headers.as_ref().map(|headers| { - let header = headers.iter().find(|h| - h.header_name().as_slice().to_ascii_lower() == "last-modified".to_string() - ); - - match header { - Some(h) => document.set_last_modified( - parse_last_modified(h.header_value().as_slice())), - None => {}, - }; - }); - - let base_url = load_response.metadata.final_url.clone(); - - { - // Store the final URL before we start parsing, so that DOM routines - // (e.g. HTMLImageElement::update_image) can resolve relative URLs - // correctly. - *page.mut_url() = Some((base_url.clone(), true)); - } - - (Some(base_url), Some(load_response)) - }, - InputString(_) => { - match *page.url() { - Some((ref page_url, _)) => (Some(page_url.clone()), None), - None => (None, None), - } - }, - }; - - let parser = ServoHTMLParser::new(base_url.clone(), document).root(); - let parser: JSRef = *parser; - - task_state::enter(IN_HTML_PARSER); - - match input { - InputString(s) => { - parser.parse_chunk(s); - } - InputUrl(url) => { - let load_response = load_response.unwrap(); - match load_response.metadata.content_type { - Some((ref t, _)) if t.as_slice().eq_ignore_ascii_case("image") => { - let page = format!("", base_url.as_ref().unwrap().serialize()); - parser.parse_chunk(page); - }, - _ => { - for msg in load_response.progress_port.iter() { - match msg { - Payload(data) => { - // FIXME: use Vec (html5ever #34) - let data = UTF_8.decode(data.as_slice(), DecodeReplace).unwrap(); - parser.parse_chunk(data); - } - Done(Err(err)) => { - panic!("Failed to load page URL {:s}, error: {:s}", url.serialize(), err); - } - Done(Ok(())) => break, - } - } - } - } - } - } - - parser.finish(); - - task_state::exit(IN_HTML_PARSER); - - debug!("finished parsing"); -} diff --git a/components/util/time.rs b/components/util/time.rs deleted file mode 100644 index ffc537b4db2..00000000000 --- a/components/util/time.rs +++ /dev/null @@ -1,297 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -//! Timing functions. - -use collections::TreeMap; -use std::comm::{Sender, channel, Receiver}; -use std::f64; -use std::io::timer::sleep; -use std::iter::AdditiveIterator; -use std::time::duration::Duration; -use std_time::{Tm, precise_time_ns, strptime}; -use task::{spawn_named}; -use url::Url; - -// front-end representation of the profiler used to communicate with the profiler -#[deriving(Clone)] -pub struct TimeProfilerChan(pub Sender); - -impl TimeProfilerChan { - pub fn send(&self, msg: TimeProfilerMsg) { - let TimeProfilerChan(ref c) = *self; - c.send(msg); - } -} - -#[deriving(PartialEq, Clone, PartialOrd, Eq, Ord)] -pub struct TimerMetadata { - url: String, - iframe: bool, - incremental: bool, -} - -pub trait Formatable { - fn format(&self) -> String; -} - -impl Formatable for Option { - fn format(&self) -> String { - match self { - // TODO(cgaebel): Center-align in the format strings as soon as rustc supports it. - &Some(ref meta) => { - let url = meta.url.as_slice(); - let url = if url.len() > 30 { - url.slice_to(30) - } else { - url - }; - let incremental = if meta.incremental { " yes" } else { " no " }; - let iframe = if meta.iframe { " yes" } else { " no " }; - format!(" {:14} {:9} {:30}", incremental, iframe, url) - }, - &None => - format!(" {:14} {:9} {:30}", " N/A", " N/A", " N/A") - } - } -} - -#[deriving(Clone)] -pub enum TimeProfilerMsg { - /// Normal message used for reporting time - TimeMsg((TimeProfilerCategory, Option), f64), - /// Message used to force print the profiling metrics - PrintMsg, - /// Tells the profiler to shut down. - ExitMsg, -} - -#[repr(u32)] -#[deriving(PartialEq, Clone, PartialOrd, Eq, Ord)] -pub enum TimeProfilerCategory { - CompositingCategory, - LayoutPerformCategory, - LayoutStyleRecalcCategory, - LayoutRestyleDamagePropagation, - LayoutNonIncrementalReset, - LayoutSelectorMatchCategory, - LayoutTreeBuilderCategory, - LayoutDamagePropagateCategory, - LayoutMainCategory, - LayoutParallelWarmupCategory, - LayoutShapingCategory, - LayoutDispListBuildCategory, - PaintingPerTileCategory, - PaintingPrepBuffCategory, - PaintingCategory, -} - -impl Formatable for TimeProfilerCategory { - // some categories are subcategories of LayoutPerformCategory - // and should be printed to indicate this - fn format(&self) -> String { - let padding = match *self { - LayoutStyleRecalcCategory | - LayoutRestyleDamagePropagation | - LayoutNonIncrementalReset | - LayoutMainCategory | - LayoutDispListBuildCategory | - LayoutShapingCategory | - LayoutDamagePropagateCategory | - PaintingPerTileCategory | - PaintingPrepBuffCategory => "+ ", - LayoutParallelWarmupCategory | - LayoutSelectorMatchCategory | - LayoutTreeBuilderCategory => "| + ", - _ => "" - }; - let name = match *self { - CompositingCategory => "Compositing", - LayoutPerformCategory => "Layout", - LayoutStyleRecalcCategory => "Style Recalc", - LayoutRestyleDamagePropagation => "Restyle Damage Propagation", - LayoutNonIncrementalReset => "Non-incremental reset (temporary)", - LayoutSelectorMatchCategory => "Selector Matching", - LayoutTreeBuilderCategory => "Tree Building", - LayoutDamagePropagateCategory => "Damage Propagation", - LayoutMainCategory => "Primary Layout Pass", - LayoutParallelWarmupCategory => "Parallel Warmup", - LayoutShapingCategory => "Shaping", - LayoutDispListBuildCategory => "Display List Construction", - PaintingPerTileCategory => "Painting Per Tile", - PaintingPrepBuffCategory => "Buffer Prep", - PaintingCategory => "Painting", - }; - format!("{:s}{}", padding, name) - } -} - -type TimeProfilerBuckets = TreeMap<(TimeProfilerCategory, Option), Vec>; - -// back end of the profiler that handles data aggregation and performance metrics -pub struct TimeProfiler { - pub port: Receiver, - buckets: TimeProfilerBuckets, - pub last_msg: Option, -} - -impl TimeProfiler { - pub fn create(period: Option) -> TimeProfilerChan { - let (chan, port) = channel(); - match period { - Some(period) => { - let period = Duration::milliseconds((period * 1000f64) as i64); - let chan = chan.clone(); - spawn_named("Time profiler timer", proc() { - loop { - sleep(period); - if chan.send_opt(PrintMsg).is_err() { - break; - } - } - }); - // Spawn the time profiler. - spawn_named("Time profiler", proc() { - let mut profiler = TimeProfiler::new(port); - profiler.start(); - }); - } - None => { - // No-op to handle messages when the time profiler is inactive. - spawn_named("Time profiler", proc() { - loop { - match port.recv_opt() { - Err(_) | Ok(ExitMsg) => break, - _ => {} - } - } - }); - } - } - - TimeProfilerChan(chan) - } - - pub fn new(port: Receiver) -> TimeProfiler { - TimeProfiler { - port: port, - buckets: TreeMap::new(), - last_msg: None, - } - } - - pub fn start(&mut self) { - loop { - let msg = self.port.recv_opt(); - match msg { - Ok(msg) => { - if !self.handle_msg(msg) { - break - } - } - _ => break - } - } - } - - fn find_or_insert(&mut self, k: (TimeProfilerCategory, Option), t: f64) { - match self.buckets.get_mut(&k) { - None => {}, - Some(v) => { v.push(t); return; }, - } - - self.buckets.insert(k, vec!(t)); - } - - fn handle_msg(&mut self, msg: TimeProfilerMsg) -> bool { - match msg.clone() { - TimeMsg(k, t) => self.find_or_insert(k, t), - PrintMsg => match self.last_msg { - // only print if more data has arrived since the last printout - Some(TimeMsg(..)) => self.print_buckets(), - _ => () - }, - ExitMsg => return false, - }; - self.last_msg = Some(msg); - true - } - - fn print_buckets(&mut self) { - println!("{:35s} {:14} {:9} {:30} {:15s} {:15s} {:-15s} {:-15s} {:-15s}", - "_category_", "_incremental?_", "_iframe?_", - " _url_", " _mean (ms)_", " _median (ms)_", - " _min (ms)_", " _max (ms)_", " _events_"); - for (&(ref category, ref meta), ref mut data) in self.buckets.iter_mut() { - data.sort_by(|a, b| { - if a < b { - Less - } else { - Greater - } - }); - let data_len = data.len(); - if data_len > 0 { - let (mean, median, min, max) = - (data.iter().map(|&x|x).sum() / (data_len as f64), - data.as_slice()[data_len / 2], - data.iter().fold(f64::INFINITY, |a, &b| a.min(b)), - data.iter().fold(-f64::INFINITY, |a, &b| a.max(b))); - println!("{:-35s}{} {:15.4f} {:15.4f} {:15.4f} {:15.4f} {:15u}", - category.format(), meta.format(), mean, median, min, max, data_len); - } - } - println!(""); - } -} - - -pub fn profile(category: TimeProfilerCategory, - // url, iframe?, first reflow? - meta: Option<(&Url, bool, bool)>, - time_profiler_chan: TimeProfilerChan, - callback: || -> T) - -> T { - let start_time = precise_time_ns(); - let val = callback(); - let end_time = precise_time_ns(); - let ms = (end_time - start_time) as f64 / 1000000f64; - let meta = meta.map(|(url, iframe, first_reflow)| - TimerMetadata { - url: url.serialize(), - iframe: iframe, - incremental: !first_reflow, - }); - time_profiler_chan.send(TimeMsg((category, meta), ms)); - return val; -} - -pub fn time(msg: &str, callback: || -> T) -> T{ - let start_time = precise_time_ns(); - let val = callback(); - let end_time = precise_time_ns(); - let ms = (end_time - start_time) as f64 / 1000000f64; - if ms >= 5f64 { - debug!("{:s} took {} ms", msg, ms); - } - return val; -} - -// Parses an RFC 2616 compliant date/time string -pub fn parse_http_timestamp(timestamp: &str) -> Option { - // RFC 822, updated by RFC 1123 - match strptime(timestamp, "%a, %d %b %Y %T %Z") { - Ok(t) => return Some(t), - Err(_) => () - } - - // RFC 850, obsoleted by RFC 1036 - match strptime(timestamp, "%A, %d-%b-%y %T %Z") { - Ok(t) => return Some(t), - Err(_) => () - } - - // ANSI C's asctime() format - strptime(timestamp, "%c").ok() -} diff --git a/tests/content/harness.js b/tests/content/harness.js deleted file mode 100644 index 452c72fa67d..00000000000 --- a/tests/content/harness.js +++ /dev/null @@ -1,106 +0,0 @@ -function _oneline(x) { - var i = x.indexOf("\n"); - return (i == -1) ? x : (x.slice(0, i) + "..."); -} - -var _expectations = 0; -var _tests = 0; -function expect(num) { - _expectations = num; -} - -function _fail(s, m) { - _tests++; - // string split to avoid problems with tests that end up printing the value of window._fail. - window.alert(_oneline("TEST-UNEXPECTED" + "-FAIL | " + s + ": " + m)); -} - -function _pass(s, m) { - _tests++; - window.alert(_oneline("TEST-PASS | " + s + ": " + m)); -} - -function _printer(opstr, op) { - return function (a, b, msg) { - var f = op(a,b) ? _pass : _fail; - if (!msg) msg = ""; - f(a + " " + opstr + " " + b, msg); - }; -} - -var is = _printer("===", function (a,b) { return a === b; }); -var is_not = _printer("!==", function (a,b) { return a !== b; }); -var is_a = _printer("is a", function (a,b) { return a instanceof b; }); -var is_not_a = _printer("is not a", function (a,b) { return !(a instanceof b); }); -var is_in = _printer("is in", function (a,b) { return a in b; }); -var is_not_in = _printer("is not in", function (a,b) { return !(a in b); }); -var as_str_is = _printer("as string is", function (a,b) { return String(a) == b; }); -var lt = _printer("<", function (a,b) { return a < b; }); -var gt = _printer(">", function (a,b) { return a > b; }); -var leq = _printer("<=", function (a,b) { return a <= b; }); -var geq = _printer(">=", function (a,b) { return a >= b; }); -var starts_with = _printer("starts with", function (a,b) { return a.indexOf(b) == 0; }); - -function is_function(val, name) { - starts_with(String(val), "function " + name + "("); -} - -function should_throw(f) { - try { - f(); - _fail("operation should have thrown but did not"); - } catch (x) { - _pass("operation successfully threw an exception", x.toString()); - } -} - -function should_not_throw(f) { - try { - f(); - _pass("operation did not throw an exception"); - } catch (x) { - _fail("operation should have not thrown", x.toString()); - } -} - -function check_selector(elem, selector, matches) { - is(elem.matches(selector), matches); -} - -function check_disabled_selector(elem, disabled) { - check_selector(elem, ":disabled", disabled); - check_selector(elem, ":enabled", !disabled); -} - -var _test_complete = false; -var _test_timeout = 10000; //10 seconds -function finish() { - if (_test_complete) { - _fail('finish called multiple times'); - } - if (_expectations > _tests) { - _fail('expected ' + _expectations + ' tests, fullfilled ' + _tests); - } - _test_complete = true; - window.close(); -} - -function _test_timed_out() { - if (!_test_complete) { - _fail('test timed out (' + _test_timeout/1000 + 's)'); - finish(); - } -} - -setTimeout(_test_timed_out, _test_timeout); - -var _needs_finish = false; -function waitForExplicitFinish() { - _needs_finish = true; -} - -addEventListener('load', function() { - if (!_needs_finish) { - finish(); - } -}); diff --git a/tests/content/netharness.js b/tests/content/netharness.js deleted file mode 100644 index cc18fe1300d..00000000000 --- a/tests/content/netharness.js +++ /dev/null @@ -1,25 +0,0 @@ -function assert_requests_made(url, n) { - var x = new XMLHttpRequest(); - x.open('GET', 'stats?' + url, false); - x.send(); - is(parseInt(x.responseText), n, '# of requests for ' + url + ' should be ' + n); -} - -function reset_stats() { - var x = new XMLHttpRequest(); - x.open('POST', 'reset', false); - x.send(); - is(x.status, 200, 'resetting stats should succeed'); -} - -function fetch(url, headers) { - var x = new XMLHttpRequest(); - x.open('GET', url, false); - if (headers) { - for (var i = 0; i < headers.length; i++) { - x.setRequestHeader(headers[i][0], headers[i][1]); - } - } - x.send(); - is(x.status, 200, 'fetching ' + url + ' should succeed '); -} diff --git a/tests/content/resources/helper.html b/tests/content/resources/helper.html deleted file mode 100644 index 90531a4b3ed..00000000000 --- a/tests/content/resources/helper.html +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/content/resources/helper_must_revalidate.html b/tests/content/resources/helper_must_revalidate.html deleted file mode 100644 index 90531a4b3ed..00000000000 --- a/tests/content/resources/helper_must_revalidate.html +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/content/resources/helper_must_revalidate.html^headers b/tests/content/resources/helper_must_revalidate.html^headers deleted file mode 100644 index 5f4c23137e1..00000000000 --- a/tests/content/resources/helper_must_revalidate.html^headers +++ /dev/null @@ -1,2 +0,0 @@ -200 -Cache-Control: must-revalidate \ No newline at end of file diff --git a/tests/content/resources/helper_nocache.html b/tests/content/resources/helper_nocache.html deleted file mode 100644 index 90531a4b3ed..00000000000 --- a/tests/content/resources/helper_nocache.html +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/content/resources/helper_nocache.html^headers b/tests/content/resources/helper_nocache.html^headers deleted file mode 100644 index e510c1a6f9a..00000000000 --- a/tests/content/resources/helper_nocache.html^headers +++ /dev/null @@ -1,2 +0,0 @@ -200 -Cache-Control: no-cache \ No newline at end of file diff --git a/tests/content/test_cached_headers_differ.html b/tests/content/test_cached_headers_differ.html deleted file mode 100644 index ba0e005a8c9..00000000000 --- a/tests/content/test_cached_headers_differ.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/content/test_cached_request.html b/tests/content/test_cached_request.html deleted file mode 100644 index 978e783f220..00000000000 --- a/tests/content/test_cached_request.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/content/test_document_url.html b/tests/content/test_document_url.html deleted file mode 100644 index 99b2a602b36..00000000000 --- a/tests/content/test_document_url.html +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - diff --git a/tests/content/test_nocache.html b/tests/content/test_nocache.html deleted file mode 100644 index d360841b5e2..00000000000 --- a/tests/content/test_nocache.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/content/test_revalidate.html b/tests/content/test_revalidate.html deleted file mode 100644 index 1caa1562b4a..00000000000 --- a/tests/content/test_revalidate.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - diff --git a/tests/contenttest.rs b/tests/contenttest.rs deleted file mode 100644 index 95e017ee768..00000000000 --- a/tests/contenttest.rs +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2013 The Servo Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -#![deny(unused_imports)] -#![deny(unused_variables)] - -extern crate getopts; -extern crate regex; -extern crate test; - -use test::{AutoColor, TestOpts, run_tests_console, TestDesc, TestDescAndFn, DynTestFn, DynTestName}; -use getopts::{getopts, reqopt}; -use std::comm::channel; -use std::from_str::FromStr; -use std::{os, str}; -use std::io::fs; -use std::io::Reader; -use std::io::process::{Command, Ignored, CreatePipe, InheritFd, ExitStatus}; -use std::task; -use regex::Regex; - -#[deriving(Clone)] -struct Config { - source_dir: String, - filter: Option -} - -fn main() { - let args = os::args(); - let config = parse_config(args.into_iter().collect()); - let opts = test_options(&config); - let tests = find_tests(&config); - match run_tests_console(&opts, tests) { - Ok(false) => os::set_exit_status(1), // tests failed - Err(_) => os::set_exit_status(2), // I/O-related failure - _ => (), - } -} - -enum ServerMsg { - IsAlive(Sender), - Exit, -} - -fn run_http_server(source_dir: String) -> (Sender, u16) { - let (tx, rx) = channel(); - let (port_sender, port_receiver) = channel(); - task::spawn(proc() { - let mut prc = Command::new("python") - .args(["../httpserver.py"]) - .stdin(Ignored) - .stdout(CreatePipe(false, true)) - .stderr(Ignored) - .cwd(&Path::new(source_dir)) - .spawn() - .ok() - .expect("Unable to spawn server."); - - let mut bytes = vec!(); - loop { - let byte = prc.stdout.as_mut().unwrap().read_byte().unwrap(); - if byte == '\n' as u8 { - break; - } else { - bytes.push(byte); - } - } - - let mut words = str::from_utf8(bytes.as_slice()).unwrap().split(' '); - let port = FromStr::from_str(words.last().unwrap()).unwrap(); - port_sender.send(port); - - loop { - match rx.recv() { - IsAlive(reply) => reply.send(prc.signal(0).is_ok()), - Exit => { - let _ = prc.signal_exit(); - break; - } - } - } - }); - (tx, port_receiver.recv()) -} - -fn parse_config(args: Vec) -> Config { - let args = args.tail(); - let opts = vec!(reqopt("s", "source-dir", "source-dir", "source-dir")); - let matches = match getopts(args, opts.as_slice()) { - Ok(m) => m, - Err(f) => panic!(format!("{}", f)) - }; - - Config { - source_dir: matches.opt_str("source-dir").unwrap(), - filter: matches.free.as_slice().head().map(|s| Regex::new(s.as_slice()).unwrap()) - } -} - -fn test_options(config: &Config) -> TestOpts { - TestOpts { - filter: config.filter.clone(), - run_ignored: false, - run_tests: true, - run_benchmarks: false, - ratchet_metrics: None, - ratchet_noise_percent: None, - save_metrics: None, - test_shard: None, - logfile: None, - nocapture: false, - color: AutoColor - } -} - -fn find_tests(config: &Config) -> Vec { - let files_res = fs::readdir(&Path::new(config.source_dir.clone())); - let mut files = match files_res { - Ok(files) => files, - _ => panic!("Error reading directory."), - }; - files.retain(|file| file.extension_str() == Some("html") ); - return files.iter().map(|file| make_test(format!("{}", file.display()), - config.source_dir.clone())).collect(); -} - -fn make_test(file: String, source_dir: String) -> TestDescAndFn { - TestDescAndFn { - desc: TestDesc { - name: DynTestName(file.clone()), - ignore: false, - should_fail: false - }, - testfn: DynTestFn(proc() { run_test(file, source_dir) }) - } -} - -fn run_test(file: String, source_dir: String) { - let (server, port) = run_http_server(source_dir); - - let path = os::make_absolute(&Path::new(file)); - // FIXME (#1094): not the right way to transform a path - let infile = format!("http://localhost:{}/{}", port, path.filename_display()); - let stdout = CreatePipe(false, true); - let stderr = InheritFd(2); - let args = ["-z", "-f", infile.as_slice()]; - - let (tx, rx) = channel(); - server.send(IsAlive(tx)); - assert!(rx.recv(), "HTTP server must be running."); - - let mut prc = match Command::new("target/servo") - .args(args) - .stdin(Ignored) - .stdout(stdout) - .stderr(stderr) - .spawn() - { - Ok(p) => p, - _ => panic!("Unable to spawn process."), - }; - let mut output = Vec::new(); - loop { - let byte = prc.stdout.as_mut().unwrap().read_byte(); - match byte { - Ok(byte) => { - print!("{}", byte as char); - output.push(byte); - } - _ => break - } - } - - server.send(Exit); - - let out = str::from_utf8(output.as_slice()); - let lines: Vec<&str> = out.unwrap().split('\n').collect(); - for &line in lines.iter() { - if line.contains("TEST-UNEXPECTED-FAIL") { - panic!(line.to_string()); - } - } - - let retval = prc.wait(); - if retval != Ok(ExitStatus(0)) { - panic!("Servo exited with non-zero status {}", retval); - } -} diff --git a/tests/httpserver.py b/tests/httpserver.py deleted file mode 100644 index 689d29bef91..00000000000 --- a/tests/httpserver.py +++ /dev/null @@ -1,115 +0,0 @@ -from SimpleHTTPServer import SimpleHTTPRequestHandler -import SocketServer -import os -import sys -from collections import defaultdict - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 0 - -requests = defaultdict(int) - -class CountingRequestHandler(SimpleHTTPRequestHandler): - def __init__(self, req, client_addr, server): - SimpleHTTPRequestHandler.__init__(self, req, client_addr, server) - - def do_POST(self): - global requests - parts = self.path.split('/') - - if parts[1] == 'reset': - requests = defaultdict(int) - self.send_response(200) - self.send_header('Content-Type', 'text/plain') - self.send_header('Content-Length', 0) - self.end_headers() - self.wfile.write('') - return - - def do_GET(self): - global requests - parts = self.path.split('?') - if parts[0] == '/stats': - self.send_response(200) - self.send_header('Content-Type', 'text/plain') - if len(parts) > 1: - body = str(requests['/' + parts[1]]) - else: - body = '' - for key, value in requests.iteritems(): - body += key + ': ' + str(value) + '\n' - self.send_header('Content-Length', len(body)) - self.end_headers() - self.wfile.write(body) - return - - header_list = [] - status = None - - path = self.translate_path(self.path) - headers = path + '^headers' - - if os.path.isfile(headers): - try: - h = open(headers, 'rb') - except IOError: - self.send_error(404, "Header file not found") - return - - header_lines = h.readlines() - status = int(header_lines[0]) - for header in header_lines[1:]: - parts = map(lambda x: x.strip(), header.split(':')) - header_list += [parts] - - if self.headers.get('If-Modified-Since'): - self.send_response(304) - self.end_headers() - return - - if not status or status == 200: - requests[self.path] += 1 - - if status or header_list: - ctype = self.guess_type(path) - try: - # Always read in binary mode. Opening files in text mode may cause - # newline translations, making the actual size of the content - # transmitted *less* than the content-length! - f = open(path, 'rb') - except IOError: - self.send_error(404, "File not found") - return - - try: - self.send_response(status or 200) - self.send_header("Content-type", ctype) - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) - - for header in header_list: - self.send_header(header[0], header[1]) - - self.end_headers() - - try: - self.copyfile(f, self.wfile) - finally: - f.close() - except: - f.close() - raise - else: - SimpleHTTPRequestHandler.do_GET(self) - -class MyTCPServer(SocketServer.TCPServer): - request_queue_size = 2000 - allow_reuse_address = True - -httpd = MyTCPServer(("", PORT), CountingRequestHandler) -if not PORT: - ip, PORT = httpd.server_address - -print "serving at port", PORT -sys.stdout.flush() -httpd.serve_forever() diff --git a/tests/wpt/metadata/MANIFEST.json b/tests/wpt/metadata/MANIFEST.json index 7a2c8735267..180cb4feebb 100644 --- a/tests/wpt/metadata/MANIFEST.json +++ b/tests/wpt/metadata/MANIFEST.json @@ -524761,7 +524761,7 @@ "support" ], "fetch/http-cache/cc-request.html": [ - "d4417b8fd444362a3f217d1c95d37811a608e1a7", + "2002d341679139428e164cfe916dd39b9b664a3e", "testharness" ], "fetch/http-cache/freshness.html": [ @@ -524769,7 +524769,7 @@ "testharness" ], "fetch/http-cache/heuristic.html": [ - "5b0d55f891cb2e235456cd65f4e9f63e07999410", + "63837026eb6085fc7d6220c3dcab200b4bcd1eca", "testharness" ], "fetch/http-cache/http-cache.js": [ @@ -524781,7 +524781,7 @@ "testharness" ], "fetch/http-cache/partial.html": [ - "243e57c39f9e45e3e2acf845b36f3a140e3763bc", + "685057fe8876321a5d42bcf1e7582e6f0b745f85", "testharness" ], "fetch/http-cache/resources/http-cache.py": [ @@ -524793,7 +524793,7 @@ "testharness" ], "fetch/http-cache/vary.html": [ - "fa9a2e0554671bf2de5826e66ac0ea73de28d530", + "45f337270cfa90932c7469802655e313367ac92f", "testharness" ], "fetch/nosniff/image.html": [ diff --git a/tests/wpt/metadata/cors/304.htm.ini b/tests/wpt/metadata/cors/304.htm.ini deleted file mode 100644 index 9b7b2c0cf46..00000000000 --- a/tests/wpt/metadata/cors/304.htm.ini +++ /dev/null @@ -1,14 +0,0 @@ -[304.htm] - type: testharness - [A 304 response with no CORS headers inherits from the stored response] - expected: FAIL - - [A 304 can expand Access-Control-Expose-Headers] - expected: FAIL - - [A 304 can contract Access-Control-Expose-Headers] - expected: FAIL - - [A 304 can change Access-Control-Allow-Origin] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-default-conditional.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-default-conditional.html.ini deleted file mode 100644 index 919c03caf2a..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-default-conditional.html.ini +++ /dev/null @@ -1,3 +0,0 @@ -[request-cache-default-conditional.html] - type: testharness - disabled: https://github.com/servo/servo/issues/13441 diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-default.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-default.html.ini deleted file mode 100644 index 1363bbf5134..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-default.html.ini +++ /dev/null @@ -1,11 +0,0 @@ -[request-cache-default.html] - type: testharness - [RequestCache "default" mode checks the cache for previously cached content and avoids going to the network if a fresh response exists with Etag and fresh response] - expected: FAIL - - [RequestCache "default" mode checks the cache for previously cached content and avoids going to the network if a fresh response exists with date and fresh response] - expected: FAIL - - [RequestCache "default" mode checks the cache for previously cached content and avoids going to the network if a fresh response exists with Last-Modified and fresh response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-force-cache.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-force-cache.html.ini deleted file mode 100644 index 24d84797f31..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-force-cache.html.ini +++ /dev/null @@ -1,29 +0,0 @@ -[request-cache-force-cache.html] - type: testharness - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for stale responses with Etag and stale response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for stale responses with date and stale response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for fresh responses with Etag and fresh response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for fresh responses with date and fresh response] - expected: FAIL - - [RequestCache "force-cache" stores the response in the cache if it goes to the network with Etag and fresh response] - expected: FAIL - - [RequestCache "force-cache" stores the response in the cache if it goes to the network with date and fresh response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for stale responses with Last-Modified and stale response] - expected: FAIL - - [RequestCache "force-cache" mode checks the cache for previously cached content and avoid revalidation for fresh responses with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "force-cache" stores the response in the cache if it goes to the network with Last-Modified and fresh response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-only-if-cached.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-only-if-cached.html.ini deleted file mode 100644 index a3406a375cc..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-only-if-cached.html.ini +++ /dev/null @@ -1,65 +0,0 @@ -[request-cache-only-if-cached.html] - type: testharness - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for stale responses with Etag and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for stale responses with date and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for fresh responses with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for fresh responses with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and does not go to the network if a cached response is not found with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and does not go to the network if a cached response is not found with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Etag and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with date and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Etag and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with date and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Etag and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with date and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for stale responses with Last-Modified and stale response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and avoids revalidation for fresh responses with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" mode checks the cache for previously cached content and does not go to the network if a cached response is not found with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") uses cached same-origin redirects to same-origin content with Last-Modified and stale response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "only-if-cached" (with "same-origin") does not follow redirects across origins and rejects with Last-Modified and stale response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/api/request/request-cache-reload.html.ini b/tests/wpt/metadata/fetch/api/request/request-cache-reload.html.ini deleted file mode 100644 index 4538f3d560f..00000000000 --- a/tests/wpt/metadata/fetch/api/request/request-cache-reload.html.ini +++ /dev/null @@ -1,20 +0,0 @@ -[request-cache-reload.html] - type: testharness - [RequestCache "reload" mode does store the response in the cache with Etag and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache with date and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache even if a previous response is already stored with Etag and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache even if a previous response is already stored with date and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache with Last-Modified and fresh response] - expected: FAIL - - [RequestCache "reload" mode does store the response in the cache even if a previous response is already stored with Last-Modified and fresh response] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/304-update.html.ini b/tests/wpt/metadata/fetch/http-cache/304-update.html.ini deleted file mode 100644 index d6a1386dd4c..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/304-update.html.ini +++ /dev/null @@ -1,14 +0,0 @@ -[304-update.html] - type: testharness - [HTTP cache updates returned headers from a Last-Modified 304.] - expected: FAIL - - [HTTP cache updates stored headers from a Last-Modified 304.] - expected: FAIL - - [HTTP cache updates returned headers from a ETag 304.] - expected: FAIL - - [HTTP cache updates stored headers from a ETag 304.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini b/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini index 253204fe700..cc3c925009b 100644 --- a/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini +++ b/tests/wpt/metadata/fetch/http-cache/cc-request.html.ini @@ -1,11 +1,5 @@ [cc-request.html] type: testharness - [HTTP cache does use aged stale response when request contains Cache-Control: max-stale that permits its use.] - expected: FAIL - - [HTTP cache does reuse stale response with Age header when request contains Cache-Control: max-stale that permits its use.] - expected: FAIL - [HTTP cache generates 504 status code when nothing is in cache and request contains Cache-Control: only-if-cached.] expected: FAIL diff --git a/tests/wpt/metadata/fetch/http-cache/freshness.html.ini b/tests/wpt/metadata/fetch/http-cache/freshness.html.ini deleted file mode 100644 index 28ad1e13bd3..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/freshness.html.ini +++ /dev/null @@ -1,20 +0,0 @@ -[freshness.html] - type: testharness - [HTTP cache reuses a response with a future Expires.] - expected: FAIL - - [HTTP cache reuses a response with positive Cache-Control: max-age.] - expected: FAIL - - [HTTP cache reuses a response with positive Cache-Control: max-age and a past Expires.] - expected: FAIL - - [HTTP cache reuses a response with positive Cache-Control: max-age and an invalid Expires.] - expected: FAIL - - [HTTP cache stores a response with Cache-Control: no-cache, but revalidates upon use.] - expected: FAIL - - [HTTP cache stores a response with Cache-Control: no-cache, but revalidates upon use, even with max-age and Expires.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/heuristic.html.ini b/tests/wpt/metadata/fetch/http-cache/heuristic.html.ini deleted file mode 100644 index 9a1905eba23..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/heuristic.html.ini +++ /dev/null @@ -1,29 +0,0 @@ -[heuristic.html] - type: testharness - [HTTP cache reuses an unknown response with Last-Modified based upon heuristic freshness when Cache-Control: public is present.] - expected: FAIL - - [HTTP cache reuses a 200 OK response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 203 Non-Authoritative Information response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 204 No Content response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 404 Not Found response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 405 Method Not Allowed response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 410 Gone response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 414 URI Too Long response with Last-Modified based upon heuristic freshness.] - expected: FAIL - - [HTTP cache reuses a 501 Not Implemented response with Last-Modified based upon heuristic freshness.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini b/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini index ccf74fbef31..2ef2eb49c2b 100644 --- a/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini +++ b/tests/wpt/metadata/fetch/http-cache/invalidate.html.ini @@ -1,20 +1,11 @@ [invalidate.html] type: testharness - [HTTP cache does not invalidate after a failed response from an unsafe request] - expected: FAIL - [HTTP cache invalidates after a successful response from an unknown method] expected: FAIL - [HTTP cache does not invalidate Location URL after a failed response from an unsafe request] - expected: FAIL - [HTTP cache invalidates Location URL after a successful response from an unknown method] expected: FAIL - [HTTP cache does not invalidate Content-Location URL after a failed response from an unsafe request] - expected: FAIL - [HTTP cache invalidates Content-Location URL after a successful response from an unknown method] expected: FAIL diff --git a/tests/wpt/metadata/fetch/http-cache/partial.html.ini b/tests/wpt/metadata/fetch/http-cache/partial.html.ini index e4afd89f90b..65c7f3ebc0c 100644 --- a/tests/wpt/metadata/fetch/http-cache/partial.html.ini +++ b/tests/wpt/metadata/fetch/http-cache/partial.html.ini @@ -1,13 +1,5 @@ [partial.html] type: testharness - [HTTP cache stores partial content and reuses it.] - expected: FAIL - - [HTTP cache stores complete response and serves smaller ranges from it.] - expected: FAIL - - [HTTP cache stores partial response and serves smaller ranges from it.] - expected: FAIL [HTTP cache stores partial content and completes it.] expected: FAIL diff --git a/tests/wpt/metadata/fetch/http-cache/status.html.ini b/tests/wpt/metadata/fetch/http-cache/status.html.ini deleted file mode 100644 index c5990d83543..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/status.html.ini +++ /dev/null @@ -1,41 +0,0 @@ -[status.html] - type: testharness - [HTTP cache avoids going to the network if it has a fresh 200 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 203 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 204 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 299 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 400 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 404 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 410 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 499 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 500 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 502 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 503 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 504 response.] - expected: FAIL - - [HTTP cache avoids going to the network if it has a fresh 599 response.] - expected: FAIL - diff --git a/tests/wpt/metadata/fetch/http-cache/vary.html.ini b/tests/wpt/metadata/fetch/http-cache/vary.html.ini deleted file mode 100644 index fe697287a21..00000000000 --- a/tests/wpt/metadata/fetch/http-cache/vary.html.ini +++ /dev/null @@ -1,17 +0,0 @@ -[vary.html] - type: testharness - [HTTP cache reuses Vary response when request matches.] - expected: FAIL - - [HTTP cache doesn't invalidate existing Vary response.] - expected: FAIL - - [HTTP cache doesn't pay attention to headers not listed in Vary.] - expected: FAIL - - [HTTP cache reuses two-way Vary response when request matches.] - expected: FAIL - - [HTTP cache reuses three-way Vary response when request matches.] - expected: FAIL - diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html b/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html index 05d6f6b8c09..6ea8fbc92fd 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/cc-request.html @@ -201,7 +201,8 @@ request_headers: [ ["Cache-Control", "only-if-cached"] ], - expected_status: 504 + expected_status: 504, + expected_response_text: "" } ] } diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html b/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html index 429dddace6a..81deb1d0688 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/heuristic.html @@ -26,6 +26,7 @@ }, { expected_type: "cached", + response_status: [299, "Whatever"], } ] }, @@ -35,8 +36,7 @@ { response_status: [299, "Whatever"], response_headers: [ - ['Last-Modified', http_date(-3 * 100)], - ['Cache-Control', 'public'] + ['Last-Modified', http_date(-3 * 100)] ], }, { diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/partial.html b/tests/wpt/web-platform-tests/fetch/http-cache/partial.html index 3ad593375ac..8d5d61d46c4 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/partial.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/partial.html @@ -24,7 +24,7 @@ response_status: [206, "Partial Content"], response_headers: [ ['Cache-Control', 'max-age=3600'], - ['Content-Range', 'bytes 0-4/10'] + ['Content-Range', 'bytes 4-9/10'] ], response_body: "01234", expected_request_headers: [ @@ -36,12 +36,13 @@ ['Range', "bytes=-5"] ], expected_type: "cached", - expected_status: 206 + expected_status: 206, + expected_response_text: "01234" } ] }, { - name: 'HTTP cache stores complete response and serves smaller ranges from it.', + name: 'HTTP cache stores complete response and serves smaller ranges from it(byte-range-spec).', requests: [ { response_headers: [ @@ -51,15 +52,54 @@ }, { request_headers: [ - ['Range', "bytes=-1"] + ['Range', "bytes=0-1"] ], expected_type: "cached", + expected_status: 206, expected_response_text: "01" + }, + ] + }, + { + name: 'HTTP cache stores complete response and serves smaller ranges from it(absent last-byte-pos).', + requests: [ + { + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ], + response_body: "01234567890", + }, + { + request_headers: [ + ['Range', "bytes=1-"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "1234567890" } ] }, { - name: 'HTTP cache stores partial response and serves smaller ranges from it.', + name: 'HTTP cache stores complete response and serves smaller ranges from it(suffix-byte-range-spec).', + requests: [ + { + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ], + response_body: "0123456789A", + }, + { + request_headers: [ + ['Range', "bytes=-1"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "A" + } + ] + }, + { + name: 'HTTP cache stores partial response and serves smaller ranges from it(byte-range-spec).', requests: [ { request_headers: [ @@ -68,7 +108,55 @@ response_status: [206, "Partial Content"], response_headers: [ ['Cache-Control', 'max-age=3600'], - ['Content-Range', 'bytes 0-4/10'] + ['Content-Range', 'bytes 4-9/10'] + ], + response_body: "01234", + }, + { + request_headers: [ + ['Range', "bytes=6-8"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "234" + } + ] + }, + { + name: 'HTTP cache stores partial response and serves smaller ranges from it(absent last-byte-pos).', + requests: [ + { + request_headers: [ + ['Range', "bytes=-5"] + ], + response_status: [206, "Partial Content"], + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ['Content-Range', 'bytes 4-9/10'] + ], + response_body: "01234", + }, + { + request_headers: [ + ['Range', "bytes=6-"] + ], + expected_type: "cached", + expected_status: 206, + expected_response_text: "234" + } + ] + }, + { + name: 'HTTP cache stores partial response and serves smaller ranges from it(suffix-byte-range-spec).', + requests: [ + { + request_headers: [ + ['Range', "bytes=-5"] + ], + response_status: [206, "Partial Content"], + response_headers: [ + ['Cache-Control', 'max-age=3600'], + ['Content-Range', 'bytes 4-9/10'] ], response_body: "01234", }, @@ -77,7 +165,8 @@ ['Range', "bytes=-1"] ], expected_type: "cached", - expected_response_text: "01" + expected_status: 206, + expected_response_text: "4" } ] }, diff --git a/tests/wpt/web-platform-tests/fetch/http-cache/vary.html b/tests/wpt/web-platform-tests/fetch/http-cache/vary.html index 2f4b945b0af..dd42b14f27a 100644 --- a/tests/wpt/web-platform-tests/fetch/http-cache/vary.html +++ b/tests/wpt/web-platform-tests/fetch/http-cache/vary.html @@ -103,6 +103,7 @@ request_headers: [ ["Foo", "1"] ], + response_body: http_content('foo_1'), expected_type: "cached" } ] @@ -245,7 +246,32 @@ ] }, { - name: "HTTP cache doesn't use three-way Vary response when request omits variant header.", + name: "HTTP cache doesn't use three-way Vary response when request doesn't match, regardless of header order.", + requests: [ + { + request_headers: [ + ["Foo", "1"], + ["Bar", "abc4"], + ["Baz", "789"] + ], + response_headers: [ + ["Expires", http_date(5000)], + ["Last-Modified", http_date(-3000)], + ["Vary", "Foo, Bar, Baz"] + ] + }, + { + request_headers: [ + ["Foo", "1"], + ["Bar", "abc"], + ["Baz", "789"] + ], + expected_type: "not_cached" + } + ] + }, + { + name: "HTTP cache uses three-way Vary response when both request and the original request omited a variant header.", requests: [ { request_headers: [ @@ -259,6 +285,33 @@ ] }, { + request_headers: [ + ["Foo", "1"], + ["Baz", "789"] + ], + expected_type: "cached" + } + ] + }, + { + name: "HTTP cache doesn't use Vary response with a field value of '*'.", + requests: [ + { + request_headers: [ + ["Foo", "1"], + ["Baz", "789"] + ], + response_headers: [ + ["Expires", http_date(5000)], + ["Last-Modified", http_date(-3000)], + ["Vary", "*"] + ] + }, + { + request_headers: [ + ["*", "1"], + ["Baz", "789"] + ], expected_type: "not_cached" } ]