auto merge of #600 : metajack/servo/new-reftest, r=pcwalton

This does not port the existing src/test/html/ref tests to the new framework, as it appears to me that they aren't really reftests in the sense of Gecko's reftest. This new driver uses the Gecko methodology.

Currently this will pop a window for each test due to not having a headless driver yet, and #570 means that servo segfaults when it shuts down so we can't check the exit status.

There's plenty to improve in the future, but this should get us started.

mk/check.mk

@@ -22,7 +22,7 @@ servo-test: $(DEPS_servo)
 	$(RUSTC) $(RFLAGS_servo) --test -o $@ $<
 
 reftest: $(S)src/test/harness/reftest/reftest.rs servo
-	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
+	$(RUSTC) -o $@ $<
 
 contenttest: $(S)src/test/harness/contenttest/contenttest.rs servo
 	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
@@ -44,15 +44,15 @@ check-all: $(DEPS_CHECK_TARGETS_ALL) check-servo tidy
 
 .PHONY: check-servo
 check-servo: servo-test
-	./servo-test $(TESTNAME)
+	./servo-test
 
 .PHONY: check-ref
 check-ref: reftest
-	./reftest --source-dir=$(S)/src/test/html/ref --work-dir=src/test/html/ref $(TESTNAME)
+	./reftest $(S)src/test/ref/*.list
 
 .PHONY: check-content
 check-content: contenttest
-	./contenttest --source-dir=$(S)/src/test/html/content $(TESTNAME)
+	./contenttest --source-dir=$(S)src/test/html/content $(TESTNAME)
 
 .PHONY: tidy
 tidy: 

src/test/harness/reftest/reftest.rs

@@ -8,188 +8,141 @@
 // except according to those terms.
 
 extern mod std;
-extern mod servo;
+extern mod extra;
 
-use std::test::{TestOpts, run_tests_console, TestDesc};
+use std::cell::Cell;
-use std::getopts::{getopts, reqopt, opt_str, fail_str};
+use std::io;
-use os::list_dir_path;
+use std::os;
-use servo::run_pipeline_png;
+use std::run;
-use servo::image::base::Image;
+use extra::digest::{Digest, DigestUtil};
+use extra::sha1::Sha1;
+use extra::test::{DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn};
+use extra::test::run_tests_console;
 
-fn main(args: ~[~str]) {
+fn main() {
-    let config = parse_config(args);
+    let args = os::args();
-    let opts = test_options(config);
+    if args.len() < 2 {
-    let tests = find_tests(config);
+        println("error: at least one reftest list must be given");
-    install_rasterize_py(config);
+        os::set_exit_status(1);
-    run_tests_console(opts, tests);
+        return;
-}
-
-struct Config {
-    source_dir: ~str,
-    work_dir: ~str,
-    filter: Option<~str>
-}
-
-fn parse_config(args: ~[~str]) -> Config {
-    let args = args.tail();
-    let opts = ~[reqopt(~"source-dir"), reqopt(~"work-dir")];
-    let matches = match getopts(args, opts) {
-      Ok(m) => m,
-      Err(f) => fail fail_str(f)
-    };
-
-    Config {
-        source_dir: opt_str(matches, ~"source-dir"),
-        work_dir: opt_str(matches, ~"work-dir"),
-        filter: if matches.free.is_empty() {
-            None
-        } else {
-            Some(matches.free.head())
-        }
     }
-}
 
-fn test_options(config: Config) -> TestOpts {
+    let tests = parse_lists(args.tail());
-    {
+    let test_opts = TestOpts {
-        filter: config.filter,
+        filter: None,
         run_ignored: false,
-        logfile: None
+        logfile: None,
-    }
+        run_tests: true,
-}
+        run_benchmarks: false,
-
+        save_results: None,
-fn find_tests(config: Config) -> ~[TestDesc] {
+        compare_results: None,
-    let all_files = list_dir_path(&Path(config.source_dir));
-    let html_files = all_files.filter( |file| file.to_str().ends_with(".html") );
-    return html_files.map(|file| make_test(config, (*file).to_str()) );
-}
-
-fn make_test(config: Config, file: ~str) -> TestDesc {
-    let directives = load_test_directives(file);
-
-    {
-        name: file,
-        testfn: fn~() { run_test(config, file) },
-        ignore: directives.ignore,
-        should_fail: false
-    }
-}
-
-struct Directives {
-    ignore: bool
-}
-
-fn load_test_directives(file: ~str) -> Directives {
-    let data = match io::read_whole_file_str(&Path(file)) {
-      result::Ok(data) => data,
-      result::Err(e) => fail #fmt("unable to load directives for %s: %s", file, e)
     };
 
-    let mut ignore = false;
+    if !run_tests_console(&test_opts, tests) {
+        os::set_exit_status(1);
+    }
+}
 
-    for str::lines(data).each |line| {
+enum ReftestKind {
-        if is_comment(line) {
+    Same,
-            if line.contains("ignore") {
+    Different,
-                ignore = true;
+}
-                break;
+
+struct Reftest {
+    name: ~str,
+    kind: ReftestKind,
+    left: ~str,
+    right: ~str,
+}
+
+fn parse_lists(filenames: &[~str]) -> ~[TestDescAndFn] {
+    let mut tests: ~[TestDescAndFn] = ~[];
+    for filenames.iter().advance |file| {
+        let file_path = Path(*file);
+        let contents = match io::read_whole_file_str(&file_path) {
+            Ok(x) => x,
+            Err(s) => fail!(s)
+        };
+
+        for contents.line_iter().advance |line| {
+            let parts: ~[&str] = line.split_iter(' ').filter(|p| !p.is_empty()).collect();
+
+            if parts.len() != 3 {
+                fail!(fmt!("reftest line: '%s' doesn't match 'KIND LEFT RIGHT'", line));
             }
+
+            let kind = match parts[0] {
+                "==" => Same,
+                "!=" => Different,
+                _ => fail!(fmt!("reftest line: '%s' has invalid kind '%s'",
+                                line, parts[0]))
+            };
+            let src_dir = file_path.dirname();
+            let file_left = src_dir + "/" + parts[1];
+            let file_right = src_dir + "/" + parts[2];
+            
+            let reftest = Reftest {
+                name: parts[1] + " / " + parts[2],
+                kind: kind,
+                left: file_left,
+                right: file_right,
+            };
+
+            tests.push(make_test(reftest));
         }
     }
+    tests
+}
 
-    fn is_comment(line: ~str) -> bool {
+fn make_test(reftest: Reftest) -> TestDescAndFn {
-        line.starts_with("<!--")
+    let name = reftest.name.clone();
-    }
+    let reftest = Cell::new(reftest);
-
+    TestDescAndFn {
-    return Directives {
+        desc: TestDesc {
-        ignore: ignore
+            name: DynTestName(name),
+            ignore: false,
+            should_fail: false,
+        },
+        testfn: DynTestFn(|| {
+            check_reftest(reftest.take());
+        }),
     }
 }
 
-fn run_test(config: Config, file: ~str) {
+fn check_reftest(reftest: Reftest) {
-    let servo_image = render_servo(config, file);
+    let options = run::ProcessOptions::new();
-    let ref_image = render_ref(config, file);
+    let args = ~[~"-o", ~"/tmp/reftest-left.png", reftest.left.clone()];
+    let mut process = run::Process::new("./servo", args, options);
+    let _retval = process.finish();
+    // assert!(retval == 0);
 
-    assert servo_image.width == ref_image.width;
+    let args = ~[~"-o", ~"/tmp/reftest-right.png", reftest.right.clone()];
-    assert servo_image.height == ref_image.height;
+    let mut process = run::Process::new("./servo", args, options);
-    #debug("image depth: ref: %?, servo: %?", ref_image.depth, servo_image.depth);
+    let _retval = process.finish();
+    // assert!(retval == 0);
 
-    for uint::range(0, servo_image.height) |h| {
+    // check the pngs are bit equal
-        for uint::range(0, servo_image.width) |w| {
+    let left_sha = calc_hash(&Path("/tmp/reftest-left.png"));
-            let i = (h * servo_image.width + w) * 4;
+    let right_sha = calc_hash(&Path("/tmp/reftest-right.png"));
-            let servo_pixel = (
+    assert!(left_sha.is_some());
-                servo_image.data[i + 0],
+    assert!(right_sha.is_some());
-                servo_image.data[i + 1],
+    match reftest.kind {
-                servo_image.data[i + 2],
+        Same => assert!(left_sha == right_sha),
-                servo_image.data[i + 3]
+        Different => assert!(left_sha != right_sha),
-            );
+    }
-            let ref_pixel = (
+}
-                ref_image.data[i + 0],
-                ref_image.data[i + 1],
-                ref_image.data[i + 2],
-                ref_image.data[i + 3]
-            );
-            #debug("i: %?, x: %?, y: %?, ref: %?, servo: %?", i, w, h, ref_pixel, servo_pixel);
 
-            let (sr, sg, sb, sa) = servo_pixel;
+fn calc_hash(path: &Path) -> Option<~str> {
-            let (rr, rg, rb, ra) = ref_pixel;
+    match io::file_reader(path) {
-
+        Err(*) => None,
-            if sr != rr
+        Ok(reader) => {
-                || sg != rg
+            let mut sha = Sha1::new();
-                || sb != rb
+            loop {
-                || sa != ra {
+                let bytes = reader.read_bytes(4096);
-                fail #fmt("mismatched pixel. x: %?, y: %?, ref: %?, servo: %?", w, h, ref_pixel, servo_pixel)
+                sha.input(bytes);
+                if bytes.len() < 4096 { break; }
             }
+            Some(sha.result_str())
         }
     }
-}
+}
-
-const WIDTH: uint = 800;
-const HEIGHT: uint = 600;
-
-fn render_servo(config: Config, file: ~str) -> Image {
-    let infile = ~"file://" + os::make_absolute(&Path(file)).to_str();
-    let outfilename = Path(file).filename().get().to_str() + ".png";
-    let outfile = Path(config.work_dir).push(outfilename).to_str();
-    run_pipeline_png(infile, outfile);
-    return sanitize_image(outfile);
-}
-
-fn render_ref(config: Config, file: ~str) -> Image {
-    let infile = file;
-    let outfilename = Path(file).filename().get().to_str() + "ref..png";
-    let outfile = Path(config.work_dir).push(outfilename);
-    // After we've generated the reference image once, we don't need
-    // to keep launching Firefox
-    if !os::path_exists(&outfile) {
-        let rasterize_path = rasterize_path(config);
-        let prog = run::start_program("python", ~[rasterize_path, infile, outfile.to_str()]);
-        prog.finish();
-    }
-    return sanitize_image(outfile.to_str());
-}
-
-fn sanitize_image(file: ~str) -> Image {
-    let buf = io::read_whole_file(&Path(file)).get();
-    let image = servo::image::base::load_from_memory(buf).get();
-
-    // I don't know how to precisely control the rendered height of
-    // the Firefox output, so it is larger than we want. Trim it down.
-    assert image.width == WIDTH;
-    assert image.height >= HEIGHT;
-    let data = vec::slice(image.data, 0, image.width * HEIGHT * 4);
-
-    return Image(image.width, HEIGHT, image.depth, data);
-}
-
-fn install_rasterize_py(config: Config) {
-    use io::WriterUtil;
-    let path = rasterize_path(config);
-    let writer = io::file_writer(&Path(path), ~[io::Create, io::Truncate]).get();
-    writer.write_str(rasterize_py());
-}
-
-fn rasterize_path(config: Config) -> ~str {
-    Path(config.work_dir).push(~"rasterize.py").to_str()
-}
-
-// This is the script that uses phantom.js to render pages
-fn rasterize_py() -> ~str { #include_str("rasterize.py") }

src/test/ref/basic.list

@@ -0,0 +1 @@
+== hello_a.html hello_b.html

src/test/ref/hello_a.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>hello</title>
+  </head>
+  <body>
+    <strong>Hello!</strong>
+  </body>
+</html>

src/test/ref/hello_b.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>hello</title>
+  </head>
+  <body>
+    <b>Hello!</b>
+  </body>
+</html>