Beginnings of a http cache

Doom cache entries based on the initial response, and prevent matching against doomed cache enties.

Evict cache entries that have passed their expiry date instead of matching them.

Document the cache. Refactor incomplete entries to lessen Option-itis.

Revalidate expired cache entries instead of unconditionally evicting them.

Forbid missing docs in cache code.

Revalidate must-revalidate entries.

Fetch content tests from a local HTTP server.

Track requests made to the test HTTP server.

Add a simple test that a cached resource with no expiry is not revalidated. Correct inverted expiry check in revalidation code.

Fix incorrect revalidation logic that dropped the consumer channels on the floor.

Ensure that requests are cached based on their request headers.

Run a separate http server instance for each test to avoid intermittent failures due to concurrent cache tests.

Add a test for uncacheable responses.

Address review comments.

tests/content/harness.js

@@ -0,0 +1,106 @@
+function _oneline(x) {
+  var i = x.indexOf("\n");
+  return (i == -1) ? x : (x.slice(0, i) + "...");
+}
+
+var _expectations = 0;
+var _tests = 0;
+function expect(num) {
+  _expectations = num;
+}
+
+function _fail(s, m) {
+  _tests++;
+  // string split to avoid problems with tests that end up printing the value of window._fail.
+  window.alert(_oneline("TEST-UNEXPECTED" + "-FAIL | " + s + ": " + m));
+}
+
+function _pass(s, m) {
+  _tests++;
+  window.alert(_oneline("TEST-PASS | " + s + ": " + m));
+}
+
+function _printer(opstr, op) {
+  return function (a, b, msg) {
+    var f = op(a,b) ? _pass : _fail;
+    if (!msg) msg = "";
+    f(a + " " + opstr + " " + b, msg);
+  };
+}
+
+var is          = _printer("===",          function (a,b) { return a === b; });
+var is_not      = _printer("!==",          function (a,b) { return a !== b; });
+var is_a        = _printer("is a",         function (a,b) { return a instanceof b; });
+var is_not_a    = _printer("is not a",     function (a,b) { return !(a instanceof b); });
+var is_in       = _printer("is in",        function (a,b) { return a in b; });
+var is_not_in   = _printer("is not in",    function (a,b) { return !(a in b); });
+var as_str_is   = _printer("as string is", function (a,b) { return String(a) == b; });
+var lt          = _printer("<",            function (a,b) { return a <  b; });
+var gt          = _printer(">",            function (a,b) { return a >  b; });
+var leq         = _printer("<=",           function (a,b) { return a <= b; });
+var geq         = _printer(">=",           function (a,b) { return a >= b; });
+var starts_with = _printer("starts with",  function (a,b) { return a.indexOf(b) == 0; });
+
+function is_function(val, name) {
+  starts_with(String(val), "function " + name + "(");
+}
+
+function should_throw(f) {
+  try {
+    f();
+    _fail("operation should have thrown but did not");
+  } catch (x) {
+    _pass("operation successfully threw an exception", x.toString());
+  }
+}
+
+function should_not_throw(f) {
+  try {
+    f();
+    _pass("operation did not throw an exception");
+  } catch (x) {
+    _fail("operation should have not thrown", x.toString());
+  }
+}
+
+function check_selector(elem, selector, matches) {
+    is(elem.matches(selector), matches);
+}
+
+function check_disabled_selector(elem, disabled) {
+    check_selector(elem, ":disabled", disabled);
+    check_selector(elem, ":enabled", !disabled);
+}
+
+var _test_complete = false;
+var _test_timeout = 10000; //10 seconds
+function finish() {
+   if (_test_complete) {
+    _fail('finish called multiple times');
+  }
+  if (_expectations > _tests) {
+    _fail('expected ' + _expectations + ' tests, fullfilled ' + _tests);
+  }
+  _test_complete = true;
+  window.close();
+}
+
+function _test_timed_out() {
+  if (!_test_complete) {
+    _fail('test timed out (' + _test_timeout/1000 + 's)');
+    finish();
+  }
+}
+
+setTimeout(_test_timed_out, _test_timeout);
+
+var _needs_finish = false;
+function waitForExplicitFinish() {
+    _needs_finish = true;
+}
+
+addEventListener('load', function() {
+  if (!_needs_finish) {
+    finish();
+  }
+});

tests/content/netharness.js

@@ -0,0 +1,25 @@
+function assert_requests_made(url, n) {
+    var x = new XMLHttpRequest();
+    x.open('GET', 'stats?' + url, false);
+    x.send();
+    is(parseInt(x.responseText), n, '# of requests for ' + url + ' should be ' + n);
+}
+
+function reset_stats() {
+    var x = new XMLHttpRequest();
+    x.open('POST', 'reset', false);
+    x.send();
+    is(x.status, 200, 'resetting stats should succeed');    
+}
+
+function fetch(url, headers) {
+    var x = new XMLHttpRequest();
+    x.open('GET', url, false);
+    if (headers) {
+	for (var i = 0; i < headers.length; i++) {
+	    x.setRequestHeader(headers[i][0], headers[i][1]);
+	}
+    }
+    x.send();
+    is(x.status, 200, 'fetching ' + url + ' should succeed ');    
+}

tests/content/resources/helper.html

@@ -0,0 +1,2 @@
+<html>
+</html>

tests/content/resources/helper_must_revalidate.html

@@ -0,0 +1,2 @@
+<html>
+</html>

tests/content/resources/helper_must_revalidate.html^headers

@@ -0,0 +1,2 @@
+200
+Cache-Control: must-revalidate

tests/content/resources/helper_nocache.html

@@ -0,0 +1,2 @@
+<html>
+</html>

tests/content/resources/helper_nocache.html^headers

@@ -0,0 +1,2 @@
+200
+Cache-Control: no-cache

tests/content/test_cached_headers_differ.html

@@ -0,0 +1,14 @@
+<html>
+<head>
+<script src="harness.js"></script>
+<script src="netharness.js"></script>
+</head>
+<body>
+<script>
+  reset_stats();
+  fetch('resources/helper.html');
+  fetch('resources/helper.html', [['X-User', 'foo']]);
+  assert_requests_made('resources/helper.html', 2);
+</script>
+</body>
+</html>

tests/content/test_cached_request.html

@@ -0,0 +1,14 @@
+<html>
+<head>
+<script src="harness.js"></script>
+<script src="netharness.js"></script>
+</head>
+<body>
+<script>
+  reset_stats();
+  fetch('resources/helper.html');
+  fetch('resources/helper.html');
+  assert_requests_made('resources/helper.html', 1);
+</script>
+</body>
+</html>

tests/content/test_document_url.html

@@ -0,0 +1,30 @@
+<!doctype html>
+<html>
+    <head>
+        <title></title>
+        <script src="harness.js"></script>
+        <script>
+            // test1: URL & documentURI
+            {
+                is_not(document.URL, null, "test1-0, URL & documentURI");
+                is_not(document.documentURI, null, "test1-1, URL & documentURI");
+                is(document.URL, document.documentURI, "test1-2, URL & documentURI");
+            }
+
+            // test2: new document
+            {
+                var doc = new Document();
+                is(doc.URL, "about:blank", "test2-0, new document");
+            }
+
+            // test3: current document
+            {
+                var url = document.URL.split("/");
+                is(url[0], "http:", "test3-0, current document");
+                is(url[url.length-1], "test_document_url.html", "test3-1, current document");
+            }
+        </script>
+    </head>
+    <body>
+    </body>
+</html>

tests/content/test_nocache.html

@@ -0,0 +1,14 @@
+<html>
+<head>
+<script src="harness.js"></script>
+<script src="netharness.js"></script>
+</head>
+<body>
+<script>
+  reset_stats();
+  fetch('resources/helper_nocache.html');
+  fetch('resources/helper_nocache.html');
+  assert_requests_made('resources/helper_nocache.html', 2);
+</script>
+</body>
+</html>

tests/content/test_revalidate.html

@@ -0,0 +1,14 @@
+<html>
+<head>
+<script src="harness.js"></script>
+<script src="netharness.js"></script>
+</head>
+<body>
+<script>
+  reset_stats();
+  fetch('resources/helper_must_revalidate.html');
+  fetch('resources/helper_must_revalidate.html');
+  assert_requests_made('resources/helper_must_revalidate.html', 1);
+</script>
+</body>
+</html>

tests/contenttest.rs

@@ -0,0 +1,194 @@
+// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![deny(unused_imports)]
+#![deny(unused_variables)]
+
+extern crate getopts;
+extern crate regex;
+extern crate test;
+
+use test::{AutoColor, TestOpts, run_tests_console, TestDesc, TestDescAndFn, DynTestFn, DynTestName};
+use getopts::{getopts, reqopt};
+use std::comm::channel;
+use std::from_str::FromStr;
+use std::{os, str};
+use std::io::fs;
+use std::io::Reader;
+use std::io::process::{Command, Ignored, CreatePipe, InheritFd, ExitStatus};
+use std::task;
+use regex::Regex;
+
+#[deriving(Clone)]
+struct Config {
+    source_dir: String,
+    filter: Option<Regex>
+}
+
+fn main() {
+    let args = os::args();
+    let config = parse_config(args.into_iter().collect());
+    let opts = test_options(&config);
+    let tests = find_tests(&config);
+    match run_tests_console(&opts, tests) {
+        Ok(false) => os::set_exit_status(1), // tests failed
+        Err(_) => os::set_exit_status(2),    // I/O-related failure
+        _ => (),
+    }
+}
+
+enum ServerMsg {
+    IsAlive(Sender<bool>),
+    Exit,
+}
+
+fn run_http_server(source_dir: String) -> (Sender<ServerMsg>, u16) {
+    let (tx, rx) = channel();
+    let (port_sender, port_receiver) = channel();
+    task::spawn(proc() {
+        let mut prc = Command::new("python")
+            .args(["../httpserver.py"])
+            .stdin(Ignored)
+            .stdout(CreatePipe(false, true))
+            .stderr(Ignored)
+            .cwd(&Path::new(source_dir))
+            .spawn()
+            .ok()
+            .expect("Unable to spawn server.");
+
+        let mut bytes = vec!();
+        loop {
+            let byte = prc.stdout.as_mut().unwrap().read_byte().unwrap();
+            if byte == '\n' as u8 {
+                break;
+            } else {
+                bytes.push(byte);
+            }
+        }
+
+        let mut words = str::from_utf8(bytes.as_slice()).unwrap().split(' ');
+        let port = FromStr::from_str(words.last().unwrap()).unwrap();
+        port_sender.send(port);
+
+        loop {
+            match rx.recv() {
+                IsAlive(reply) => reply.send(prc.signal(0).is_ok()),
+                Exit => {
+                    let _ = prc.signal_exit();
+                    break;
+                }
+            }
+        }
+    });
+    (tx, port_receiver.recv())
+}
+
+fn parse_config(args: Vec<String>) -> Config {
+    let args = args.tail();
+    let opts = vec!(reqopt("s", "source-dir", "source-dir", "source-dir"));
+    let matches = match getopts(args, opts.as_slice()) {
+      Ok(m) => m,
+      Err(f) => panic!(format!("{}", f))
+    };
+
+    Config {
+        source_dir: matches.opt_str("source-dir").unwrap(),
+        filter: matches.free.as_slice().head().map(|s| Regex::new(s.as_slice()).unwrap())
+    }
+}
+
+fn test_options(config: &Config) -> TestOpts {
+    TestOpts {
+        filter: config.filter.clone(),
+        run_ignored: false,
+        run_tests: true,
+        run_benchmarks: false,
+        ratchet_metrics: None,
+        ratchet_noise_percent: None,
+        save_metrics: None,
+        test_shard: None,
+        logfile: None,
+        nocapture: false,
+        color: AutoColor
+    }
+}
+
+fn find_tests(config: &Config) -> Vec<TestDescAndFn> {
+    let files_res = fs::readdir(&Path::new(config.source_dir.clone()));
+    let mut files = match files_res {
+        Ok(files) => files,
+        _ => panic!("Error reading directory."),
+    };
+    files.retain(|file| file.extension_str() == Some("html") );
+    return files.iter().map(|file| make_test(format!("{}", file.display()),
+                                             config.source_dir.clone())).collect();
+}
+
+fn make_test(file: String, source_dir: String) -> TestDescAndFn {
+    TestDescAndFn {
+        desc: TestDesc {
+            name: DynTestName(file.clone()),
+            ignore: false,
+            should_fail: false
+        },
+        testfn: DynTestFn(proc() { run_test(file, source_dir) })
+    }
+}
+
+fn run_test(file: String, source_dir: String) {
+    let (server, port) = run_http_server(source_dir);
+
+    let path = os::make_absolute(&Path::new(file));
+    // FIXME (#1094): not the right way to transform a path
+    let infile = format!("http://localhost:{}/{}", port, path.filename_display());
+    let stdout = CreatePipe(false, true);
+    let stderr = InheritFd(2);
+    let args = ["-z", "-f", infile.as_slice()];
+
+    let (tx, rx) = channel();
+    server.send(IsAlive(tx));
+    assert!(rx.recv(), "HTTP server must be running.");
+
+    let mut prc = match Command::new("target/servo")
+        .args(args)
+        .stdin(Ignored)
+        .stdout(stdout)
+        .stderr(stderr)
+        .spawn()
+    {
+        Ok(p) => p,
+        _ => panic!("Unable to spawn process."),
+    };
+    let mut output = Vec::new();
+    loop {
+        let byte = prc.stdout.as_mut().unwrap().read_byte();
+        match byte {
+            Ok(byte) => {
+                print!("{}", byte as char);
+                output.push(byte);
+            }
+            _ => break
+        }
+    }
+
+    server.send(Exit);
+
+    let out = str::from_utf8(output.as_slice());
+    let lines: Vec<&str> = out.unwrap().split('\n').collect();
+    for &line in lines.iter() {
+        if line.contains("TEST-UNEXPECTED-FAIL") {
+            panic!(line.to_string());
+        }
+    }
+
+    let retval = prc.wait();
+    if retval != Ok(ExitStatus(0)) {
+        panic!("Servo exited with non-zero status {}", retval);
+    }
+}

tests/httpserver.py

@@ -0,0 +1,115 @@
+from SimpleHTTPServer import SimpleHTTPRequestHandler
+import SocketServer
+import os
+import sys
+from collections import defaultdict
+
+PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 0
+
+requests = defaultdict(int)
+
+class CountingRequestHandler(SimpleHTTPRequestHandler):
+    def __init__(self, req, client_addr, server):
+        SimpleHTTPRequestHandler.__init__(self, req, client_addr, server)
+
+    def do_POST(self):
+        global requests
+        parts = self.path.split('/')
+
+        if parts[1] == 'reset':
+            requests = defaultdict(int)
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain')
+            self.send_header('Content-Length', 0)
+            self.end_headers()
+            self.wfile.write('')
+            return
+
+    def do_GET(self):
+        global requests
+        parts = self.path.split('?')
+        if parts[0] == '/stats':
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain')
+            if len(parts) > 1:
+                body = str(requests['/' + parts[1]])
+            else:
+                body = ''
+                for key, value in requests.iteritems():
+                    body += key + ': ' + str(value) + '\n'
+            self.send_header('Content-Length', len(body))
+            self.end_headers()
+            self.wfile.write(body)
+            return
+
+        header_list = []
+        status = None
+
+        path = self.translate_path(self.path)
+        headers = path + '^headers'
+
+        if os.path.isfile(headers):
+            try:
+                h = open(headers, 'rb')
+            except IOError:
+                self.send_error(404, "Header file not found")
+                return
+
+            header_lines = h.readlines()
+            status = int(header_lines[0])
+            for header in header_lines[1:]:
+                parts = map(lambda x: x.strip(), header.split(':'))
+                header_list += [parts]
+
+        if self.headers.get('If-Modified-Since'):
+            self.send_response(304)
+            self.end_headers()
+            return
+
+        if not status or status == 200:
+            requests[self.path] += 1
+
+        if status or header_list:
+            ctype = self.guess_type(path)
+            try:
+                # Always read in binary mode. Opening files in text mode may cause
+                # newline translations, making the actual size of the content
+                # transmitted *less* than the content-length!
+                f = open(path, 'rb')
+            except IOError:
+                self.send_error(404, "File not found")
+                return
+
+            try:
+                self.send_response(status or 200)
+                self.send_header("Content-type", ctype)
+                fs = os.fstat(f.fileno())
+                self.send_header("Content-Length", str(fs[6]))
+                self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
+
+                for header in header_list:
+                    self.send_header(header[0], header[1])
+
+                self.end_headers()
+
+                try:
+                    self.copyfile(f, self.wfile)
+                finally:
+                    f.close()
+            except:
+                f.close()
+                raise
+        else:
+            SimpleHTTPRequestHandler.do_GET(self)
+
+class MyTCPServer(SocketServer.TCPServer):
+    request_queue_size = 2000
+    allow_reuse_address = True
+
+httpd = MyTCPServer(("", PORT), CountingRequestHandler)
+if not PORT:
+    ip, PORT = httpd.server_address
+
+print "serving at port", PORT
+sys.stdout.flush()
+httpd.serve_forever()