Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-08-06 22:15:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Allow marking tests as flaky by target OS.

Browse source
This commit is contained in:
Simon Sapin 2014-08-21 21:59:31 +01:00
parent 04c5080892
commit c497fe6b2c
1 changed files with 33 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

50
src/test/harness/reftest/reftest.rs
View file

@ -21,10 +21,17 @@ use test::{AutoColor, DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn}
use test::run_tests_console; use test::run_tests_console;
use regex::Regex; use regex::Regex;
enum RenderMode {
CpuRendering = 1, bitflags!(
GpuRendering = 2, flags RenderMode: u32 {
} static CpuRendering = 0x00000001,
static GpuRendering = 0x00000010,
static LinuxTarget = 0x00000100,
static MacOsTarget = 0x00001000,
static AndroidTarget = 0x00010000
}
)
fn main() { fn main() {
let args = os::args(); let args = os::args();
@ -39,11 +46,20 @@ fn main() {
[ref render_mode_string, ref base_path, ref testname, ..] => (render_mode_string, base_path, Some(Regex::new(testname.as_slice()).unwrap())), [ref render_mode_string, ref base_path, ref testname, ..] => (render_mode_string, base_path, Some(Regex::new(testname.as_slice()).unwrap())),
}; };
let render_mode = match render_mode_string.as_slice() { let mut render_mode = match render_mode_string.as_slice() {
"cpu" => CpuRendering, "cpu" => CpuRendering,
"gpu" => GpuRendering, "gpu" => GpuRendering,
_ => fail!("First argument must specify cpu or gpu as rendering mode") _ => fail!("First argument must specify cpu or gpu as rendering mode")
}; };
if cfg!(target_os = "linux") {
render_mode.insert(LinuxTarget);
}
if cfg!(target_os = "macos") {
render_mode.insert(MacOsTarget);
}
if cfg!(target_os = "android") {
render_mode.insert(AndroidTarget);
}
let mut all_tests = vec!(); let mut all_tests = vec!();
println!("Scanning {} for manifests\n", base_path); println!("Scanning {} for manifests\n", base_path);
@ -97,7 +113,7 @@ struct Reftest {
id: uint, id: uint,
servo_args: Vec<String>, servo_args: Vec<String>,
render_mode: RenderMode, render_mode: RenderMode,
flakiness: uint, is_flaky: bool,
} }
struct TestLine<'a> { struct TestLine<'a> {
@ -150,11 +166,13 @@ fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode) -> Ve
let file_right = src_dir.append("/").append(test_line.file_right); let file_right = src_dir.append("/").append(test_line.file_right);
let mut conditions_list = test_line.conditions.split(','); let mut conditions_list = test_line.conditions.split(',');
let mut flakiness = 0; let mut flakiness = RenderMode::empty();
for condition in conditions_list { for condition in conditions_list {
match condition { match condition {
"flaky_cpu" => flakiness |= CpuRendering as uint, "flaky_cpu" => flakiness.insert(CpuRendering),
"flaky_gpu" => flakiness |= GpuRendering as uint, "flaky_gpu" => flakiness.insert(GpuRendering),
"flaky_linux" => flakiness.insert(LinuxTarget),
"flaky_macos" => flakiness.insert(MacOsTarget),
_ => (), _ => (),
} }
} }
@ -166,7 +184,7 @@ fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode) -> Ve
id: next_id, id: next_id,
render_mode: render_mode, render_mode: render_mode,
servo_args: servo_args.iter().map(|x| x.clone()).collect(), servo_args: servo_args.iter().map(|x| x.clone()).collect(),
flakiness: flakiness, is_flaky: render_mode.intersects(flakiness),
}; };
next_id += 1; next_id += 1;
@ -193,9 +211,9 @@ fn make_test(reftest: Reftest) -> TestDescAndFn {
fn capture(reftest: &Reftest, side: uint) -> png::Image { fn capture(reftest: &Reftest, side: uint) -> png::Image {
let filename = format!("/tmp/servo-reftest-{:06u}-{:u}.png", reftest.id, side); let filename = format!("/tmp/servo-reftest-{:06u}-{:u}.png", reftest.id, side);
let mut args = reftest.servo_args.clone(); let mut args = reftest.servo_args.clone();
match reftest.render_mode { // GPU rendering is the default
CpuRendering => args.push("-c".to_string()), if reftest.render_mode.contains(CpuRendering) {
_ => {} // GPU rendering is the default args.push("-c".to_string());
} }
args.push_all_move(vec!("-f".to_string(), "-o".to_string(), filename.clone(), reftest.files[side].clone())); args.push_all_move(vec!("-f".to_string(), "-o".to_string(), filename.clone(), reftest.files[side].clone()));
@ -225,8 +243,6 @@ fn check_reftest(reftest: Reftest) {
} }
}).collect::<Vec<u8>>(); }).collect::<Vec<u8>>();
let test_is_flaky = (reftest.render_mode as uint & reftest.flakiness) != 0;
if pixels.iter().any(|&a| a < 255) { if pixels.iter().any(|&a| a < 255) {
let output_str = format!("/tmp/servo-reftest-{:06u}-diff.png", reftest.id); let output_str = format!("/tmp/servo-reftest-{:06u}-diff.png", reftest.id);
let output = from_str::<Path>(output_str.as_slice()).unwrap(); let output = from_str::<Path>(output_str.as_slice()).unwrap();
@ -240,12 +256,12 @@ fn check_reftest(reftest: Reftest) {
let res = png::store_png(&mut img, &output); let res = png::store_png(&mut img, &output);
assert!(res.is_ok()); assert!(res.is_ok());
match (reftest.kind, test_is_flaky) { match (reftest.kind, reftest.is_flaky) {
(Same, true) => println!("flaky test - rendering difference: {}", output_str), (Same, true) => println!("flaky test - rendering difference: {}", output_str),
(Same, false) => fail!("rendering difference: {}", output_str), (Same, false) => fail!("rendering difference: {}", output_str),
(Different, _) => {} // Result was different and that's what was expected (Different, _) => {} // Result was different and that's what was expected
} }
} else { } else {
assert!(test_is_flaky || reftest.kind == Same); assert!(reftest.is_flaky || reftest.kind == Same);
} }
} }