diff --git a/src/test/harness/reftest/reftest.rs b/src/test/harness/reftest/reftest.rs index f0ec11ddc98..57738bbf517 100644 --- a/src/test/harness/reftest/reftest.rs +++ b/src/test/harness/reftest/reftest.rs @@ -114,6 +114,7 @@ struct Reftest { servo_args: Vec, render_mode: RenderMode, is_flaky: bool, + experimental: bool, } struct TestLine<'a> { @@ -166,12 +167,14 @@ fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode, id_of let mut conditions_list = test_line.conditions.split(','); let mut flakiness = RenderMode::empty(); + let mut experimental = false; for condition in conditions_list { match condition { "flaky_cpu" => flakiness.insert(CpuRendering), "flaky_gpu" => flakiness.insert(GpuRendering), "flaky_linux" => flakiness.insert(LinuxTarget), "flaky_macos" => flakiness.insert(MacOsTarget), + "experimental" => experimental = true, _ => (), } } @@ -184,6 +187,7 @@ fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode, id_of render_mode: render_mode, servo_args: servo_args.iter().map(|x| x.clone()).collect(), is_flaky: render_mode.intersects(flakiness), + experimental: experimental, }; tests.push(make_test(reftest)); @@ -212,7 +216,11 @@ fn capture(reftest: &Reftest, side: uint) -> png::Image { if reftest.render_mode.contains(CpuRendering) { args.push("-c".to_string()); } - args.push_all_move(vec!("-f".to_string(), "-o".to_string(), filename.clone(), reftest.files[side].clone())); + if reftest.experimental { + args.push("--experimental".to_string()); + } + args.push_all(["-f".to_string(), "-o".to_string(), filename.clone(), + reftest.files[side].clone()]); let retval = match Command::new("./servo").args(args.as_slice()).status() { Ok(status) => status, diff --git a/src/test/harness/reftest/reftest.rs.orig b/src/test/harness/reftest/reftest.rs.orig new file mode 100644 index 00000000000..645a8d55307 --- /dev/null +++ b/src/test/harness/reftest/reftest.rs.orig @@ -0,0 +1,415 @@ +// Copyright 2013 The Servo Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +extern crate png; +extern crate std; +extern crate test; +extern crate regex; + +use std::ascii::StrAsciiExt; +use std::io; +use std::io::{File, Reader, Command}; +use std::io::process::ExitStatus; +use std::os; +use test::{AutoColor, DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn}; +use test::run_tests_console; +use regex::Regex; + + +bitflags!( + flags RenderMode: u32 { + static CpuRendering = 0x00000001, + static GpuRendering = 0x00000010, + static LinuxTarget = 0x00000100, + static MacOsTarget = 0x00001000, + static AndroidTarget = 0x00010000 + } +) + + +fn main() { + let args = os::args(); + let mut parts = args.tail().split(|e| "--" == e.as_slice()); + + let harness_args = parts.next().unwrap(); // .split() is never empty + let servo_args = parts.next().unwrap_or(&[]); + + let (render_mode_string, base_path, testname) = match harness_args { + [] | [_] => fail!("USAGE: cpu|gpu base_path [testname regex]"), + [ref render_mode_string, ref base_path] => (render_mode_string, base_path, None), + [ref render_mode_string, ref base_path, ref testname, ..] => (render_mode_string, base_path, Some(Regex::new(testname.as_slice()).unwrap())), + }; + + let mut render_mode = match render_mode_string.as_slice() { + "cpu" => CpuRendering, + "gpu" => GpuRendering, + _ => fail!("First argument must specify cpu or gpu as rendering mode") + }; + if cfg!(target_os = "linux") { + render_mode.insert(LinuxTarget); + } + if cfg!(target_os = "macos") { + render_mode.insert(MacOsTarget); + } + if cfg!(target_os = "android") { + render_mode.insert(AndroidTarget); + } + + let mut all_tests = vec!(); + println!("Scanning {} for manifests\n", base_path); + + for file in io::fs::walk_dir(&Path::new(base_path.as_slice())).unwrap() { + let maybe_extension = file.extension_str(); + match maybe_extension { + Some(extension) => { + if extension.to_ascii_lower().as_slice() == "list" && file.is_file() { + let manifest = file.as_str().unwrap(); + let tests = parse_lists(manifest, servo_args, render_mode); + println!("\t{} [{} tests]", manifest, tests.len()); + all_tests.push_all_move(tests); + } + } + _ => {} + } + } + + let test_opts = TestOpts { + filter: testname, + run_ignored: false, + logfile: None, + run_tests: true, + run_benchmarks: false, + ratchet_noise_percent: None, + ratchet_metrics: None, + save_metrics: None, + test_shard: None, + nocapture: false, + color: AutoColor + }; + + match run_tests_console(&test_opts, all_tests) { + Ok(false) => os::set_exit_status(1), // tests failed + Err(_) => os::set_exit_status(2), // I/O-related failure + _ => (), + } +} + +#[deriving(PartialEq)] +enum ReftestKind { + Same, + Different, +} + +struct Reftest { + name: String, + kind: ReftestKind, + files: [String, ..2], + id: uint, + servo_args: Vec, + render_mode: RenderMode, +<<<<<<< HEAD + is_flaky: bool, +||||||| merged common ancestors + flakiness: uint, +======= + flakiness: uint, + experimental: bool, +>>>>>>> Reftests can opt into --experimental +} + +struct TestLine<'a> { + conditions: &'a str, + kind: &'a str, + file_left: &'a str, + file_right: &'a str, +} + +fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode) -> Vec { + let mut tests = Vec::new(); + let mut next_id = 0; + let file_path = Path::new(file); + let contents = File::open_mode(&file_path, io::Open, io::Read) + .and_then(|mut f| f.read_to_string()) + .ok().expect("Could not read file"); + + for line in contents.as_slice().lines() { +<<<<<<< HEAD + // ignore comments or empty lines + if line.starts_with("#") || line.is_empty() { + continue; + } + + let parts: Vec<&str> = line.split(' ').filter(|p| !p.is_empty()).collect(); + + let test_line = match parts.len() { + 3 => TestLine { + conditions: "", + kind: parts[0], + file_left: parts[1], + file_right: parts[2], +||||||| merged common ancestors + // ignore comments or empty lines + if line.starts_with("#") || line.is_empty() { + continue; + } + + let parts: Vec<&str> = line.split(' ').filter(|p| !p.is_empty()).collect(); + + let test_line = match parts.len() { + 3 => { + TestLine { + conditions: "", + kind: parts[0], + file_left: parts[1], + file_right: parts[2], + } + }, + 4 => { + TestLine { + conditions: parts[0], + kind: parts[1], + file_left: parts[2], + file_right: parts[3], + } + }, + _ => { + fail!("reftest line: '{:s}' doesn't match '[CONDITIONS] KIND LEFT RIGHT'", line); + } + }; + + let kind = match test_line.kind { + "==" => Same, + "!=" => Different, + part => fail!("reftest line: '{:s}' has invalid kind '{:s}'", line, part) + }; + let src_path = file_path.dir_path(); + let src_dir = src_path.display().to_string(); + let file_left = src_dir.clone().append("/").append(test_line.file_left); + let file_right = src_dir.append("/").append(test_line.file_right); + + let mut conditions_list = test_line.conditions.split(','); + let mut flakiness = 0; + for condition in conditions_list { + match condition { + "flaky_cpu" => { + flakiness |= CpuRendering as uint; +======= + // ignore comments or empty lines + if line.starts_with("#") || line.is_empty() { + continue; + } + + let parts: Vec<&str> = line.split(' ').filter(|p| !p.is_empty()).collect(); + + let test_line = match parts.len() { + 3 => { + TestLine { + conditions: "", + kind: parts[0], + file_left: parts[1], + file_right: parts[2], + } + }, + 4 => { + TestLine { + conditions: parts[0], + kind: parts[1], + file_left: parts[2], + file_right: parts[3], + } + }, + _ => { + fail!("reftest line: '{:s}' doesn't match '[CONDITIONS] KIND LEFT RIGHT'", line); + } + }; + + let kind = match test_line.kind { + "==" => Same, + "!=" => Different, + part => fail!("reftest line: '{:s}' has invalid kind '{:s}'", line, part) + }; + let src_path = file_path.dir_path(); + let src_dir = src_path.display().to_string(); + let file_left = src_dir.clone().append("/").append(test_line.file_left); + let file_right = src_dir.append("/").append(test_line.file_right); + + let mut conditions_list = test_line.conditions.split(','); + let mut flakiness = 0; + let mut experimental = false; + for condition in conditions_list { + match condition { + "flaky_cpu" => { + flakiness |= CpuRendering as uint; +>>>>>>> Reftests can opt into --experimental + }, + 4 => TestLine { + conditions: parts[0], + kind: parts[1], + file_left: parts[2], + file_right: parts[3], + }, +<<<<<<< HEAD + _ => fail!("reftest line: '{:s}' doesn't match '[CONDITIONS] KIND LEFT RIGHT'", line), + }; + + let kind = match test_line.kind { + "==" => Same, + "!=" => Different, + part => fail!("reftest line: '{:s}' has invalid kind '{:s}'", line, part) + }; + let src_path = file_path.dir_path(); + let src_dir = src_path.display().to_string(); + let file_left = src_dir.clone().append("/").append(test_line.file_left); + let file_right = src_dir.append("/").append(test_line.file_right); + + let mut conditions_list = test_line.conditions.split(','); + let mut flakiness = RenderMode::empty(); + for condition in conditions_list { + match condition { + "flaky_cpu" => flakiness.insert(CpuRendering), + "flaky_gpu" => flakiness.insert(GpuRendering), + "flaky_linux" => flakiness.insert(LinuxTarget), + "flaky_macos" => flakiness.insert(MacOsTarget), + _ => (), + } + } + + let reftest = Reftest { + name: format!("{} {} {}", test_line.file_left, test_line.kind, test_line.file_right), + kind: kind, + files: [file_left, file_right], + id: next_id, + render_mode: render_mode, + servo_args: servo_args.iter().map(|x| x.clone()).collect(), + is_flaky: render_mode.intersects(flakiness), + }; + + next_id += 1; + + tests.push(make_test(reftest)); +||||||| merged common ancestors + _ => {} + } + } + + let reftest = Reftest { + name: test_line.file_left.to_string().append(" / ").append(test_line.file_right), + kind: kind, + files: [file_left, file_right], + id: next_id, + render_mode: render_mode, + servo_args: servo_args.iter().map(|x| x.clone()).collect(), + flakiness: flakiness, + }; + + next_id += 1; + + tests.push(make_test(reftest)); +======= + "experimental" => { + experimental = true; + }, + _ => {} + } + } + + let reftest = Reftest { + name: test_line.file_left.to_string().append(" / ").append(test_line.file_right), + kind: kind, + files: [file_left, file_right], + id: next_id, + render_mode: render_mode, + servo_args: servo_args.iter().map(|x| x.clone()).collect(), + flakiness: flakiness, + experimental: experimental, + }; + + next_id += 1; + + tests.push(make_test(reftest)); +>>>>>>> Reftests can opt into --experimental + } + tests +} + +fn make_test(reftest: Reftest) -> TestDescAndFn { + let name = reftest.name.clone(); + TestDescAndFn { + desc: TestDesc { + name: DynTestName(name), + ignore: false, + should_fail: false, + }, + testfn: DynTestFn(proc() { + check_reftest(reftest); + }), + } +} + +fn capture(reftest: &Reftest, side: uint) -> png::Image { + let filename = format!("/tmp/servo-reftest-{:06u}-{:u}.png", reftest.id, side); + let mut args = reftest.servo_args.clone(); + // GPU rendering is the default + if reftest.render_mode.contains(CpuRendering) { + args.push("-c".to_string()); + } + if reftest.experimental { + args.push("--experimental".to_string()); + } + args.push_all(["-f".to_string(), "-o".to_string(), filename.clone(), + reftest.files[side].clone()]); + + let retval = match Command::new("./servo").args(args.as_slice()).status() { + Ok(status) => status, + Err(e) => fail!("failed to execute process: {}", e), + }; + assert!(retval == ExitStatus(0)); + + png::load_png(&from_str::(filename.as_slice()).unwrap()).unwrap() +} + +fn check_reftest(reftest: Reftest) { + let left = capture(&reftest, 0); + let right = capture(&reftest, 1); + + let pixels = left.pixels.iter().zip(right.pixels.iter()).map(|(&a, &b)| { + if a as i8 - b as i8 == 0 { + // White for correct + 0xFF + } else { + // "1100" in the RGBA channel with an error for an incorrect value + // This results in some number of C0 and FFs, which is much more + // readable (and distinguishable) than the previous difference-wise + // scaling but does not require reconstructing the actual RGBA pixel. + 0xC0 + } + }).collect::>(); + + if pixels.iter().any(|&a| a < 255) { + let output_str = format!("/tmp/servo-reftest-{:06u}-diff.png", reftest.id); + let output = from_str::(output_str.as_slice()).unwrap(); + + let mut img = png::Image { + width: left.width, + height: left.height, + color_type: png::RGBA8, + pixels: pixels, + }; + let res = png::store_png(&mut img, &output); + assert!(res.is_ok()); + + match (reftest.kind, reftest.is_flaky) { + (Same, true) => println!("flaky test - rendering difference: {}", output_str), + (Same, false) => fail!("rendering difference: {}", output_str), + (Different, _) => {} // Result was different and that's what was expected + } + } else { + assert!(reftest.is_flaky || reftest.kind == Same); + } +}