auto merge of #600 : metajack/servo/new-reftest, r=pcwalton

This does not port the existing src/test/html/ref tests to the new framework, as it appears to me that they aren't really reftests in the sense of Gecko's reftest. This new driver uses the Gecko methodology.

Currently this will pop a window for each test due to not having a headless driver yet, and #570 means that servo segfaults when it shuts down so we can't check the exit status.

There's plenty to improve in the future, but this should get us started.

mk/check.mk

@@ -22,7 +22,7 @@ servo-test: $(DEPS_servo)
 	$(RUSTC) $(RFLAGS_servo) --test -o $@ $<
 
 reftest: $(S)src/test/harness/reftest/reftest.rs servo
-	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
+	$(RUSTC) -o $@ $<
 
 contenttest: $(S)src/test/harness/contenttest/contenttest.rs servo
 	$(RUSTC) $(RFLAGS_servo) -o $@ $< -L .
@@ -44,15 +44,15 @@ check-all: $(DEPS_CHECK_TARGETS_ALL) check-servo tidy
 
 .PHONY: check-servo
 check-servo: servo-test
-	./servo-test $(TESTNAME)
+	./servo-test
 
 .PHONY: check-ref
 check-ref: reftest
-	./reftest --source-dir=$(S)/src/test/html/ref --work-dir=src/test/html/ref $(TESTNAME)
+	./reftest $(S)src/test/ref/*.list
 
 .PHONY: check-content
 check-content: contenttest
-	./contenttest --source-dir=$(S)/src/test/html/content $(TESTNAME)
+	./contenttest --source-dir=$(S)src/test/html/content $(TESTNAME)
 
 .PHONY: tidy
 tidy: 

src/test/harness/reftest/reftest.rs

@@ -8,188 +8,141 @@
 // except according to those terms.
 
 extern mod std;
-extern mod servo;
+extern mod extra;
 
-use std::test::{TestOpts, run_tests_console, TestDesc};
-use std::getopts::{getopts, reqopt, opt_str, fail_str};
-use os::list_dir_path;
-use servo::run_pipeline_png;
-use servo::image::base::Image;
+use std::cell::Cell;
+use std::io;
+use std::os;
+use std::run;
+use extra::digest::{Digest, DigestUtil};
+use extra::sha1::Sha1;
+use extra::test::{DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn};
+use extra::test::run_tests_console;
 
-fn main(args: ~[~str]) {
-    let config = parse_config(args);
-    let opts = test_options(config);
-    let tests = find_tests(config);
-    install_rasterize_py(config);
-    run_tests_console(opts, tests);
-}
-
-struct Config {
-    source_dir: ~str,
-    work_dir: ~str,
-    filter: Option<~str>
-}
-
-fn parse_config(args: ~[~str]) -> Config {
-    let args = args.tail();
-    let opts = ~[reqopt(~"source-dir"), reqopt(~"work-dir")];
-    let matches = match getopts(args, opts) {
-      Ok(m) => m,
-      Err(f) => fail fail_str(f)
-    };
-
-    Config {
-        source_dir: opt_str(matches, ~"source-dir"),
-        work_dir: opt_str(matches, ~"work-dir"),
-        filter: if matches.free.is_empty() {
-            None
-        } else {
-            Some(matches.free.head())
-        }
+fn main() {
+    let args = os::args();
+    if args.len() < 2 {
+        println("error: at least one reftest list must be given");
+        os::set_exit_status(1);
+        return;
     }
-}
 
-fn test_options(config: Config) -> TestOpts {
-    {
-        filter: config.filter,
+    let tests = parse_lists(args.tail());
+    let test_opts = TestOpts {
+        filter: None,
         run_ignored: false,
-        logfile: None
-    }
-}
-
-fn find_tests(config: Config) -> ~[TestDesc] {
-    let all_files = list_dir_path(&Path(config.source_dir));
-    let html_files = all_files.filter( |file| file.to_str().ends_with(".html") );
-    return html_files.map(|file| make_test(config, (*file).to_str()) );
-}
-
-fn make_test(config: Config, file: ~str) -> TestDesc {
-    let directives = load_test_directives(file);
-
-    {
-        name: file,
-        testfn: fn~() { run_test(config, file) },
-        ignore: directives.ignore,
-        should_fail: false
-    }
-}
-
-struct Directives {
-    ignore: bool
-}
-
-fn load_test_directives(file: ~str) -> Directives {
-    let data = match io::read_whole_file_str(&Path(file)) {
-      result::Ok(data) => data,
-      result::Err(e) => fail #fmt("unable to load directives for %s: %s", file, e)
+        logfile: None,
+        run_tests: true,
+        run_benchmarks: false,
+        save_results: None,
+        compare_results: None,
     };
 
-    let mut ignore = false;
+    if !run_tests_console(&test_opts, tests) {
+        os::set_exit_status(1);
+    }
+}
 
-    for str::lines(data).each |line| {
-        if is_comment(line) {
-            if line.contains("ignore") {
-                ignore = true;
-                break;
+enum ReftestKind {
+    Same,
+    Different,
+}
+
+struct Reftest {
+    name: ~str,
+    kind: ReftestKind,
+    left: ~str,
+    right: ~str,
+}
+
+fn parse_lists(filenames: &[~str]) -> ~[TestDescAndFn] {
+    let mut tests: ~[TestDescAndFn] = ~[];
+    for filenames.iter().advance |file| {
+        let file_path = Path(*file);
+        let contents = match io::read_whole_file_str(&file_path) {
+            Ok(x) => x,
+            Err(s) => fail!(s)
+        };
+
+        for contents.line_iter().advance |line| {
+            let parts: ~[&str] = line.split_iter(' ').filter(|p| !p.is_empty()).collect();
+
+            if parts.len() != 3 {
+                fail!(fmt!("reftest line: '%s' doesn't match 'KIND LEFT RIGHT'", line));
             }
+
+            let kind = match parts[0] {
+                "==" => Same,
+                "!=" => Different,
+                _ => fail!(fmt!("reftest line: '%s' has invalid kind '%s'",
+                                line, parts[0]))
+            };
+            let src_dir = file_path.dirname();
+            let file_left = src_dir + "/" + parts[1];
+            let file_right = src_dir + "/" + parts[2];
+            
+            let reftest = Reftest {
+                name: parts[1] + " / " + parts[2],
+                kind: kind,
+                left: file_left,
+                right: file_right,
+            };
+
+            tests.push(make_test(reftest));
         }
     }
+    tests
+}
 
-    fn is_comment(line: ~str) -> bool {
-        line.starts_with("<!--")
-    }
-
-    return Directives {
-        ignore: ignore
+fn make_test(reftest: Reftest) -> TestDescAndFn {
+    let name = reftest.name.clone();
+    let reftest = Cell::new(reftest);
+    TestDescAndFn {
+        desc: TestDesc {
+            name: DynTestName(name),
+            ignore: false,
+            should_fail: false,
+        },
+        testfn: DynTestFn(|| {
+            check_reftest(reftest.take());
+        }),
     }
 }
 
-fn run_test(config: Config, file: ~str) {
-    let servo_image = render_servo(config, file);
-    let ref_image = render_ref(config, file);
+fn check_reftest(reftest: Reftest) {
+    let options = run::ProcessOptions::new();
+    let args = ~[~"-o", ~"/tmp/reftest-left.png", reftest.left.clone()];
+    let mut process = run::Process::new("./servo", args, options);
+    let _retval = process.finish();
+    // assert!(retval == 0);
 
-    assert servo_image.width == ref_image.width;
-    assert servo_image.height == ref_image.height;
-    #debug("image depth: ref: %?, servo: %?", ref_image.depth, servo_image.depth);
+    let args = ~[~"-o", ~"/tmp/reftest-right.png", reftest.right.clone()];
+    let mut process = run::Process::new("./servo", args, options);
+    let _retval = process.finish();
+    // assert!(retval == 0);
 
-    for uint::range(0, servo_image.height) |h| {
-        for uint::range(0, servo_image.width) |w| {
-            let i = (h * servo_image.width + w) * 4;
-            let servo_pixel = (
-                servo_image.data[i + 0],
-                servo_image.data[i + 1],
-                servo_image.data[i + 2],
-                servo_image.data[i + 3]
-            );
-            let ref_pixel = (
-                ref_image.data[i + 0],
-                ref_image.data[i + 1],
-                ref_image.data[i + 2],
-                ref_image.data[i + 3]
-            );
-            #debug("i: %?, x: %?, y: %?, ref: %?, servo: %?", i, w, h, ref_pixel, servo_pixel);
+    // check the pngs are bit equal
+    let left_sha = calc_hash(&Path("/tmp/reftest-left.png"));
+    let right_sha = calc_hash(&Path("/tmp/reftest-right.png"));
+    assert!(left_sha.is_some());
+    assert!(right_sha.is_some());
+    match reftest.kind {
+        Same => assert!(left_sha == right_sha),
+        Different => assert!(left_sha != right_sha),
+    }
+}
 
-            let (sr, sg, sb, sa) = servo_pixel;
-            let (rr, rg, rb, ra) = ref_pixel;
-
-            if sr != rr
-                || sg != rg
-                || sb != rb
-                || sa != ra {
-                fail #fmt("mismatched pixel. x: %?, y: %?, ref: %?, servo: %?", w, h, ref_pixel, servo_pixel)
+fn calc_hash(path: &Path) -> Option<~str> {
+    match io::file_reader(path) {
+        Err(*) => None,
+        Ok(reader) => {
+            let mut sha = Sha1::new();
+            loop {
+                let bytes = reader.read_bytes(4096);
+                sha.input(bytes);
+                if bytes.len() < 4096 { break; }
             }
+            Some(sha.result_str())
         }
     }
-}
-
-const WIDTH: uint = 800;
-const HEIGHT: uint = 600;
-
-fn render_servo(config: Config, file: ~str) -> Image {
-    let infile = ~"file://" + os::make_absolute(&Path(file)).to_str();
-    let outfilename = Path(file).filename().get().to_str() + ".png";
-    let outfile = Path(config.work_dir).push(outfilename).to_str();
-    run_pipeline_png(infile, outfile);
-    return sanitize_image(outfile);
-}
-
-fn render_ref(config: Config, file: ~str) -> Image {
-    let infile = file;
-    let outfilename = Path(file).filename().get().to_str() + "ref..png";
-    let outfile = Path(config.work_dir).push(outfilename);
-    // After we've generated the reference image once, we don't need
-    // to keep launching Firefox
-    if !os::path_exists(&outfile) {
-        let rasterize_path = rasterize_path(config);
-        let prog = run::start_program("python", ~[rasterize_path, infile, outfile.to_str()]);
-        prog.finish();
-    }
-    return sanitize_image(outfile.to_str());
-}
-
-fn sanitize_image(file: ~str) -> Image {
-    let buf = io::read_whole_file(&Path(file)).get();
-    let image = servo::image::base::load_from_memory(buf).get();
-
-    // I don't know how to precisely control the rendered height of
-    // the Firefox output, so it is larger than we want. Trim it down.
-    assert image.width == WIDTH;
-    assert image.height >= HEIGHT;
-    let data = vec::slice(image.data, 0, image.width * HEIGHT * 4);
-
-    return Image(image.width, HEIGHT, image.depth, data);
-}
-
-fn install_rasterize_py(config: Config) {
-    use io::WriterUtil;
-    let path = rasterize_path(config);
-    let writer = io::file_writer(&Path(path), ~[io::Create, io::Truncate]).get();
-    writer.write_str(rasterize_py());
-}
-
-fn rasterize_path(config: Config) -> ~str {
-    Path(config.work_dir).push(~"rasterize.py").to_str()
-}
-
-// This is the script that uses phantom.js to render pages
-fn rasterize_py() -> ~str { #include_str("rasterize.py") }
+}

src/test/ref/basic.list

@@ -0,0 +1 @@
+== hello_a.html hello_b.html

src/test/ref/hello_a.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>hello</title>
+  </head>
+  <body>
+    <strong>Hello!</strong>
+  </body>
+</html>

src/test/ref/hello_b.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>hello</title>
+  </head>
+  <body>
+    <b>Hello!</b>
+  </body>
+</html>