auto merge of #600 : metajack/servo/new-reftest, r=pcwalton

This does not port the existing src/test/html/ref tests to the new framework, as it appears to me that they aren't really reftests in the sense of Gecko's reftest. This new driver uses the Gecko methodology.

Currently this will pop a window for each test due to not having a headless driver yet, and #570 means that servo segfaults when it shuts down so we can't check the exit status.

There's plenty to improve in the future, but this should get us started.

mk/check.mk

@@ -22,7 +22,7 @@ servo-test: $(DEPS_servo)
 	$(RUSTC) $(RFLAGS_servo) --test -o $@ $<
 
 reftest: $(S)src/test/harness/reftest/reftest.rs servo
-	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
+	$(RUSTC) -o $@ $<
 
 contenttest: $(S)src/test/harness/contenttest/contenttest.rs servo
 	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
@@ -44,15 +44,15 @@ check-all: $(DEPS_CHECK_TARGETS_ALL) check-servo tidy
 
 .PHONY: check-servo
 check-servo: servo-test
-	./servo-test $(TESTNAME)
+	./servo-test
 
 .PHONY: check-ref
 check-ref: reftest
-	./reftest --source-dir=$(S)/src/test/html/ref --work-dir=src/test/html/ref $(TESTNAME)
+	./reftest $(S)src/test/ref/*.list
 
 .PHONY: check-content
 check-content: contenttest
-	./contenttest --source-dir=$(S)/src/test/html/content $(TESTNAME)
+	./contenttest --source-dir=$(S)src/test/html/content $(TESTNAME)
 
 .PHONY: tidy
 tidy: 

src/test/harness/reftest/reftest.rs

@@ -8,188 +8,141 @@
 // except according to those terms.
 
 extern mod std;
-extern mod servo;
+extern mod extra;
 
-use std::test::{TestOpts, run_tests_console, TestDesc};
+use std::cell::Cell;
-use std::getopts::{getopts, reqopt, opt_str, fail_str};
+use std::io;
-use os::list_dir_path;
+use std::os;
-use servo::run_pipeline_png;
+use std::run;
-use servo::image::base::Image;
+use extra::digest::{Digest, DigestUtil};
+use extra::sha1::Sha1;
+use extra::test::{DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn};
+use extra::test::run_tests_console;
 
-fn main(args: ~[~str]) {
+fn main() {
-    let config = parse_config(args);
+    let args = os::args();
-    let opts = test_options(config);
+    if args.len() < 2 {
-    let tests = find_tests(config);
+        println("error: at least one reftest list must be given");
-    install_rasterize_py(config);
+        os::set_exit_status(1);
-    run_tests_console(opts, tests);
+        return;
-}
-
-struct Config {
-    source_dir: ~str,
-    work_dir: ~str,
-    filter: Option<~str>
-}
-
-fn parse_config(args: ~[~str]) -> Config {
-    let args = args.tail();
-    let opts = ~[reqopt(~"source-dir"), reqopt(~"work-dir")];
-    let matches = match getopts(args, opts) {
-      Ok(m) => m,
-      Err(f) => fail fail_str(f)
-    };
-
-    Config {
-        source_dir: opt_str(matches, ~"source-dir"),
-        work_dir: opt_str(matches, ~"work-dir"),
-        filter: if matches.free.is_empty() {
-            None
-        } else {
-            Some(matches.free.head())
     }
-    }
-}
 
-fn test_options(config: Config) -> TestOpts {
+    let tests = parse_lists(args.tail());
-    {
+    let test_opts = TestOpts {
-        filter: config.filter,
+        filter: None,
         run_ignored: false,
-        logfile: None
+        logfile: None,
-    }
+        run_tests: true,
-}
+        run_benchmarks: false,
-
+        save_results: None,
-fn find_tests(config: Config) -> ~[TestDesc] {
+        compare_results: None,
-    let all_files = list_dir_path(&Path(config.source_dir));
-    let html_files = all_files.filter( |file| file.to_str().ends_with(".html") );
-    return html_files.map(|file| make_test(config, (*file).to_str()) );
-}
-
-fn make_test(config: Config, file: ~str) -> TestDesc {
-    let directives = load_test_directives(file);
-
-    {
-        name: file,
-        testfn: fn~() { run_test(config, file) },
-        ignore: directives.ignore,
-        should_fail: false
-    }
-}
-
-struct Directives {
-    ignore: bool
-}
-
-fn load_test_directives(file: ~str) -> Directives {
-    let data = match io::read_whole_file_str(&Path(file)) {
-      result::Ok(data) => data,
-      result::Err(e) => fail #fmt("unable to load directives for %s: %s", file, e)
     };
 
-    let mut ignore = false;
+    if !run_tests_console(&test_opts, tests) {
-
+        os::set_exit_status(1);
-    for str::lines(data).each |line| {
-        if is_comment(line) {
-            if line.contains("ignore") {
-                ignore = true;
-                break;
-            }
-        }
-    }
-
-    fn is_comment(line: ~str) -> bool {
-        line.starts_with("<!--")
-    }
-
-    return Directives {
-        ignore: ignore
     }
 }
 
-fn run_test(config: Config, file: ~str) {
+enum ReftestKind {
-    let servo_image = render_servo(config, file);
+    Same,
-    let ref_image = render_ref(config, file);
+    Different,
+}
 
-    assert servo_image.width == ref_image.width;
+struct Reftest {
-    assert servo_image.height == ref_image.height;
+    name: ~str,
-    #debug("image depth: ref: %?, servo: %?", ref_image.depth, servo_image.depth);
+    kind: ReftestKind,
+    left: ~str,
+    right: ~str,
+}
 
-    for uint::range(0, servo_image.height) |h| {
+fn parse_lists(filenames: &[~str]) -> ~[TestDescAndFn] {
-        for uint::range(0, servo_image.width) |w| {
+    let mut tests: ~[TestDescAndFn] = ~[];
-            let i = (h * servo_image.width + w) * 4;
+    for filenames.iter().advance |file| {
-            let servo_pixel = (
+        let file_path = Path(*file);
-                servo_image.data[i + 0],
+        let contents = match io::read_whole_file_str(&file_path) {
-                servo_image.data[i + 1],
+            Ok(x) => x,
-                servo_image.data[i + 2],
+            Err(s) => fail!(s)
-                servo_image.data[i + 3]
+        };
-            );
-            let ref_pixel = (
-                ref_image.data[i + 0],
-                ref_image.data[i + 1],
-                ref_image.data[i + 2],
-                ref_image.data[i + 3]
-            );
-            #debug("i: %?, x: %?, y: %?, ref: %?, servo: %?", i, w, h, ref_pixel, servo_pixel);
 
-            let (sr, sg, sb, sa) = servo_pixel;
+        for contents.line_iter().advance |line| {
-            let (rr, rg, rb, ra) = ref_pixel;
+            let parts: ~[&str] = line.split_iter(' ').filter(|p| !p.is_empty()).collect();
 
-            if sr != rr
+            if parts.len() != 3 {
-                || sg != rg
+                fail!(fmt!("reftest line: '%s' doesn't match 'KIND LEFT RIGHT'", line));
-                || sb != rb
+            }
-                || sa != ra {
+
-                fail #fmt("mismatched pixel. x: %?, y: %?, ref: %?, servo: %?", w, h, ref_pixel, servo_pixel)
+            let kind = match parts[0] {
+                "==" => Same,
+                "!=" => Different,
+                _ => fail!(fmt!("reftest line: '%s' has invalid kind '%s'",
+                                line, parts[0]))
+            };
+            let src_dir = file_path.dirname();
+            let file_left = src_dir + "/" + parts[1];
+            let file_right = src_dir + "/" + parts[2];
+            
+            let reftest = Reftest {
+                name: parts[1] + " / " + parts[2],
+                kind: kind,
+                left: file_left,
+                right: file_right,
+            };
+
+            tests.push(make_test(reftest));
         }
     }
+    tests
+}
+
+fn make_test(reftest: Reftest) -> TestDescAndFn {
+    let name = reftest.name.clone();
+    let reftest = Cell::new(reftest);
+    TestDescAndFn {
+        desc: TestDesc {
+            name: DynTestName(name),
+            ignore: false,
+            should_fail: false,
+        },
+        testfn: DynTestFn(|| {
+            check_reftest(reftest.take());
+        }),
     }
 }
 
-const WIDTH: uint = 800;
+fn check_reftest(reftest: Reftest) {
-const HEIGHT: uint = 600;
+    let options = run::ProcessOptions::new();
+    let args = ~[~"-o", ~"/tmp/reftest-left.png", reftest.left.clone()];
+    let mut process = run::Process::new("./servo", args, options);
+    let _retval = process.finish();
+    // assert!(retval == 0);
 
-fn render_servo(config: Config, file: ~str) -> Image {
+    let args = ~[~"-o", ~"/tmp/reftest-right.png", reftest.right.clone()];
-    let infile = ~"file://" + os::make_absolute(&Path(file)).to_str();
+    let mut process = run::Process::new("./servo", args, options);
-    let outfilename = Path(file).filename().get().to_str() + ".png";
+    let _retval = process.finish();
-    let outfile = Path(config.work_dir).push(outfilename).to_str();
+    // assert!(retval == 0);
-    run_pipeline_png(infile, outfile);
-    return sanitize_image(outfile);
-}
 
-fn render_ref(config: Config, file: ~str) -> Image {
+    // check the pngs are bit equal
-    let infile = file;
+    let left_sha = calc_hash(&Path("/tmp/reftest-left.png"));
-    let outfilename = Path(file).filename().get().to_str() + "ref..png";
+    let right_sha = calc_hash(&Path("/tmp/reftest-right.png"));
-    let outfile = Path(config.work_dir).push(outfilename);
+    assert!(left_sha.is_some());
-    // After we've generated the reference image once, we don't need
+    assert!(right_sha.is_some());
-    // to keep launching Firefox
+    match reftest.kind {
-    if !os::path_exists(&outfile) {
+        Same => assert!(left_sha == right_sha),
-        let rasterize_path = rasterize_path(config);
+        Different => assert!(left_sha != right_sha),
-        let prog = run::start_program("python", ~[rasterize_path, infile, outfile.to_str()]);
-        prog.finish();
     }
-    return sanitize_image(outfile.to_str());
 }
 
-fn sanitize_image(file: ~str) -> Image {
+fn calc_hash(path: &Path) -> Option<~str> {
-    let buf = io::read_whole_file(&Path(file)).get();
+    match io::file_reader(path) {
-    let image = servo::image::base::load_from_memory(buf).get();
+        Err(*) => None,
-
+        Ok(reader) => {
-    // I don't know how to precisely control the rendered height of
+            let mut sha = Sha1::new();
-    // the Firefox output, so it is larger than we want. Trim it down.
+            loop {
-    assert image.width == WIDTH;
+                let bytes = reader.read_bytes(4096);
-    assert image.height >= HEIGHT;
+                sha.input(bytes);
-    let data = vec::slice(image.data, 0, image.width * HEIGHT * 4);
+                if bytes.len() < 4096 { break; }
-
+            }
-    return Image(image.width, HEIGHT, image.depth, data);
+            Some(sha.result_str())
+        }
+    }
 }
-
-fn install_rasterize_py(config: Config) {
-    use io::WriterUtil;
-    let path = rasterize_path(config);
-    let writer = io::file_writer(&Path(path), ~[io::Create, io::Truncate]).get();
-    writer.write_str(rasterize_py());
-}
-
-fn rasterize_path(config: Config) -> ~str {
-    Path(config.work_dir).push(~"rasterize.py").to_str()
-}
-
-// This is the script that uses phantom.js to render pages
-fn rasterize_py() -> ~str { #include_str("rasterize.py") }

src/test/ref/basic.list

@@ -0,0 +1 @@
+== hello_a.html hello_b.html

src/test/ref/hello_a.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>hello</title>
+  </head>
+  <body>
+    <strong>Hello!</strong>
+  </body>
+</html>

src/test/ref/hello_b.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>hello</title>
+  </head>
+  <body>
+    <b>Hello!</b>
+  </body>
+</html>