mirror of
https://github.com/servo/servo.git
synced 2025-08-05 13:40:08 +01:00
Remove the legacy reftest framework.
This commit is contained in:
Ms2ger
parent
671cfd9940
commit
d8511c8b8d
15 changed files with 5 additions and 503 deletions
|
|
@ -201,52 +201,11 @@ class MachCommands(CommandBase):
|
|||
@Command('test-ref',
|
||||
description='Run the reference tests',
|
||||
category='testing')
|
||||
@CommandArgument('--kind', '-k', default=DEFAULT_RENDER_MODE,
|
||||
help=HELP_RENDER_MODE)
|
||||
@CommandArgument('--release', '-r', action='store_true',
|
||||
help='Run with a release build of Servo')
|
||||
@CommandArgument('--include', default=None, nargs='+',
|
||||
help="Only run tests that match this pattern. If the "
|
||||
"path to the ref test directory is included, it "
|
||||
"will automatically be trimmed out.")
|
||||
@CommandArgument(
|
||||
'servo_params', default=None, nargs=argparse.REMAINDER,
|
||||
help="Command-line arguments to be passed through to Servo")
|
||||
def test_ref(self, kind=DEFAULT_RENDER_MODE, include=None, servo_params=None,
|
||||
release=False):
|
||||
self.ensure_bootstrapped()
|
||||
self.ensure_built_tests(release=release)
|
||||
assert kind is not None, 'kind cannot be None, see help'
|
||||
|
||||
kinds = ["cpu", "gpu"] if kind == 'both' else [kind]
|
||||
test_path = path.join(self.context.topdir, "tests", "ref")
|
||||
error = False
|
||||
|
||||
test_start = time()
|
||||
for k in kinds:
|
||||
print("Running %s reftests..." % k)
|
||||
test_args = [k, test_path]
|
||||
if include is not None:
|
||||
ref_path = path.join("tests", "ref")
|
||||
for name in include:
|
||||
# Check to see if we were passed something leading with the
|
||||
# path to the ref test directory, and trim it so that reftest
|
||||
# knows how to filter it.
|
||||
maybe_path = path.normpath(name)
|
||||
if ref_path in maybe_path:
|
||||
test_args.append(path.relpath(maybe_path, ref_path))
|
||||
else:
|
||||
test_args.append(name)
|
||||
if servo_params is not None:
|
||||
test_args += ["--"] + servo_params
|
||||
ret = self.run_test("reftest", test_args, release=release)
|
||||
error = error or ret != 0
|
||||
elapsed = time() - test_start
|
||||
|
||||
print("Reference tests completed in %0.2fs" % elapsed)
|
||||
|
||||
if error:
|
||||
return 1
|
||||
@CommandArgument('params', default=None, nargs=argparse.REMAINDER)
|
||||
def test_ref(self, params=None):
|
||||
print("Ref tests have been replaced by web-platform-tests under "
|
||||
"tests/wpt/mozilla/.")
|
||||
return 0
|
||||
|
||||
@Command('test-content',
|
||||
description='Run the content tests',
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 215 B |
Binary file not shown.
|
Before Width: | Height: | Size: 171 B |
Binary file not shown.
|
Before Width: | Height: | Size: 2.2 KiB |
|
|
@ -1,5 +0,0 @@
|
|||
# This file must be sorted alphabetically.
|
||||
# Please run `./mach test-tidy` to check your changes.
|
||||
|
||||
# This file must be sorted alphabetically.
|
||||
# Please run `./mach test-tidy` to check your changes.
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
body {
|
||||
font-family: 'ahem';
|
||||
font-size: 100px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
body {
|
||||
display: none;
|
||||
}
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 510 B |
|
|
@ -1,48 +0,0 @@
|
|||
/* http://meyerweb.com/eric/tools/css/reset/
|
||||
v2.0 | 20110126
|
||||
License: none (public domain)
|
||||
*/
|
||||
|
||||
html, body, div, span, applet, object, iframe,
|
||||
h1, h2, h3, h4, h5, h6, p, blockquote, pre,
|
||||
a, abbr, acronym, address, big, cite, code,
|
||||
del, dfn, em, img, ins, kbd, q, s, samp,
|
||||
small, strike, strong, sub, sup, tt, var,
|
||||
b, u, i, center,
|
||||
dl, dt, dd, ol, ul, li,
|
||||
fieldset, form, label, legend,
|
||||
table, caption, tbody, tfoot, thead, tr, th, td,
|
||||
article, aside, canvas, details, embed,
|
||||
figure, figcaption, footer, header, hgroup,
|
||||
menu, nav, output, ruby, section, summary,
|
||||
time, mark, audio, video {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
border: 0;
|
||||
font-size: 100%;
|
||||
font: inherit;
|
||||
vertical-align: baseline;
|
||||
}
|
||||
/* HTML5 display-role reset for older browsers */
|
||||
article, aside, details, figcaption, figure,
|
||||
footer, header, hgroup, menu, nav, section {
|
||||
display: block;
|
||||
}
|
||||
body {
|
||||
line-height: 1;
|
||||
}
|
||||
ol, ul {
|
||||
list-style: none;
|
||||
}
|
||||
blockquote, q {
|
||||
quotes: none;
|
||||
}
|
||||
blockquote:before, blockquote:after,
|
||||
q:before, q:after {
|
||||
content: '';
|
||||
content: none;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
border-spacing: 0;
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 253 B |
Binary file not shown.
|
Before Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 86 KiB |
395
tests/reftest.rs
vendored
395
tests/reftest.rs
vendored
|
|
@ -1,395 +0,0 @@
|
|||
// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
#![feature(fs_walk)]
|
||||
#![feature(slice_patterns)]
|
||||
#![feature(test)]
|
||||
|
||||
#[macro_use] extern crate bitflags;
|
||||
extern crate image;
|
||||
extern crate test;
|
||||
extern crate url;
|
||||
extern crate util;
|
||||
|
||||
use image::{DynamicImage, GenericImage, ImageFormat, RgbImage};
|
||||
use std::env;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::{File, walk_dir};
|
||||
use std::io::{self, Read, Result, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process;
|
||||
use std::process::{Command};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
use test::run_tests_console;
|
||||
use test::{AutoColor, DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn, ShouldPanic};
|
||||
use url::Url;
|
||||
|
||||
bitflags!(
|
||||
flags RenderMode: u32 {
|
||||
const CPU_RENDERING = 0x00000001,
|
||||
const GPU_RENDERING = 0x00000010,
|
||||
const LINUX_TARGET = 0x00000100,
|
||||
const MACOS_TARGET = 0x00001000,
|
||||
const ANDROID_TARGET = 0x00010000
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = env::args().collect();
|
||||
let mut parts = args[1..].split(|e| &**e == "--");
|
||||
|
||||
let harness_args = parts.next().unwrap(); // .split() is never empty
|
||||
let servo_args = parts.next().unwrap_or(&[]);
|
||||
|
||||
let (render_mode_string, base_path, testnames) = match harness_args {
|
||||
[ref render_mode_string, ref base_path, testnames..] =>
|
||||
(render_mode_string, base_path, testnames),
|
||||
_ => panic!("USAGE: cpu|gpu base_path [testname ...]"),
|
||||
};
|
||||
|
||||
let mut render_mode = match &**render_mode_string {
|
||||
"cpu" => CPU_RENDERING,
|
||||
"gpu" => GPU_RENDERING,
|
||||
_ => panic!("First argument must specify cpu or gpu as rendering mode")
|
||||
};
|
||||
if cfg!(target_os = "linux") {
|
||||
render_mode.insert(LINUX_TARGET);
|
||||
}
|
||||
if cfg!(target_os = "macos") {
|
||||
render_mode.insert(MACOS_TARGET);
|
||||
}
|
||||
if cfg!(target_os = "android") {
|
||||
render_mode.insert(ANDROID_TARGET);
|
||||
}
|
||||
|
||||
let mut all_tests = vec!();
|
||||
println!("Scanning {} for manifests\n", base_path);
|
||||
|
||||
for file in walk_dir(base_path).unwrap() {
|
||||
let file = file.unwrap().path();
|
||||
let maybe_extension = file.extension();
|
||||
match maybe_extension {
|
||||
Some(extension) => {
|
||||
if extension == OsStr::new("list") && file.is_file() {
|
||||
let len = all_tests.len();
|
||||
let mut tests = parse_lists(&file, testnames, servo_args, render_mode, len);
|
||||
println!("\t{} [{} tests]", file.display(), tests.len());
|
||||
all_tests.append(&mut tests);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let test_opts = TestOpts {
|
||||
filter: None,
|
||||
run_ignored: false,
|
||||
logfile: None,
|
||||
run_tests: true,
|
||||
bench_benchmarks: false,
|
||||
nocapture: false,
|
||||
color: AutoColor,
|
||||
};
|
||||
|
||||
match run(test_opts,
|
||||
all_tests,
|
||||
servo_args.iter().cloned().collect()) {
|
||||
Ok(false) => process::exit(1), // tests failed
|
||||
Err(_) => process::exit(2), // I/O-related failure
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn run(test_opts: TestOpts, all_tests: Vec<TestDescAndFn>,
|
||||
servo_args: Vec<String>) -> io::Result<bool> {
|
||||
// Verify that we're passing in valid servo arguments. Otherwise, servo
|
||||
// will exit before we've run any tests, and it will appear to us as if
|
||||
// all the tests are failing.
|
||||
let mut command = Command::new(&servo_path());
|
||||
command
|
||||
.args(&servo_args)
|
||||
.arg("-z")
|
||||
.arg("about:blank");
|
||||
|
||||
let mut child = match command.spawn() {
|
||||
Ok(p) => p,
|
||||
Err(e) => panic!("failed to execute process: {}", e),
|
||||
};
|
||||
|
||||
// Wait for the shell to launch or to fail
|
||||
thread::sleep(Duration::from_secs(1));
|
||||
child.kill().unwrap();
|
||||
let output = try!(child.wait_with_output());
|
||||
|
||||
let stderr = String::from_utf8(output.stderr).unwrap();
|
||||
|
||||
if stderr.contains("Unrecognized") {
|
||||
println!("Servo: {}", stderr);
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
run_tests_console(&test_opts, all_tests)
|
||||
}
|
||||
|
||||
#[derive(PartialEq)]
|
||||
enum ReftestKind {
|
||||
Same,
|
||||
Different,
|
||||
}
|
||||
|
||||
struct Reftest {
|
||||
name: String,
|
||||
kind: ReftestKind,
|
||||
files: [PathBuf; 2],
|
||||
id: usize,
|
||||
servo_args: Vec<String>,
|
||||
render_mode: RenderMode,
|
||||
is_flaky: bool,
|
||||
prefs: Vec<String>,
|
||||
fragment_identifier: Option<String>,
|
||||
resolution: Option<String>,
|
||||
pixel_ratio: Option<f32>,
|
||||
}
|
||||
|
||||
struct TestLine<'a> {
|
||||
conditions: &'a str,
|
||||
kind: &'a str,
|
||||
file_left: &'a str,
|
||||
file_right: &'a str,
|
||||
}
|
||||
|
||||
fn parse_lists(file: &Path,
|
||||
filters: &[String],
|
||||
servo_args: &[String],
|
||||
render_mode: RenderMode,
|
||||
id_offset: usize)
|
||||
-> Vec<TestDescAndFn> {
|
||||
let mut tests = Vec::new();
|
||||
let contents = {
|
||||
let mut f = File::open(file).unwrap();
|
||||
let mut contents = String::new();
|
||||
f.read_to_string(&mut contents).unwrap();
|
||||
contents
|
||||
};
|
||||
|
||||
for line in contents.lines() {
|
||||
// ignore comments or empty lines
|
||||
if line.starts_with("#") || line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let parts: Vec<&str> = line.split(' ').filter(|p| !p.is_empty()).collect();
|
||||
|
||||
let test_line = match parts.len() {
|
||||
3 => TestLine {
|
||||
conditions: "",
|
||||
kind: parts[0],
|
||||
file_left: parts[1],
|
||||
file_right: parts[2],
|
||||
},
|
||||
4 => TestLine {
|
||||
conditions: parts[0],
|
||||
kind: parts[1],
|
||||
file_left: parts[2],
|
||||
file_right: parts[3],
|
||||
},
|
||||
_ => panic!("reftest line: '{}' doesn't match '[CONDITIONS] KIND LEFT RIGHT'", line),
|
||||
};
|
||||
|
||||
let kind = match test_line.kind {
|
||||
"==" => ReftestKind::Same,
|
||||
"!=" => ReftestKind::Different,
|
||||
part => panic!("reftest line: '{}' has invalid kind '{}'", line, part)
|
||||
};
|
||||
|
||||
let base = env::current_dir().unwrap().join(file.parent().unwrap());
|
||||
|
||||
let file_left = base.join(test_line.file_left);
|
||||
let file_right = base.join(test_line.file_right);
|
||||
|
||||
let conditions_list = test_line.conditions.split(',');
|
||||
let mut flakiness = RenderMode::empty();
|
||||
let mut prefs = vec![];
|
||||
let mut fragment_identifier = None;
|
||||
let mut resolution = None;
|
||||
let mut pixel_ratio = None;
|
||||
for condition in conditions_list {
|
||||
match condition {
|
||||
"flaky_cpu" => flakiness.insert(CPU_RENDERING),
|
||||
"flaky_gpu" => flakiness.insert(GPU_RENDERING),
|
||||
"flaky_linux" => flakiness.insert(LINUX_TARGET),
|
||||
"flaky_macos" => flakiness.insert(MACOS_TARGET),
|
||||
_ => ()
|
||||
}
|
||||
if condition.starts_with("prefs:\"") {
|
||||
if let Some(joined) = condition.split("\"").nth(1) {
|
||||
prefs.extend(joined.split(",").map(str::to_owned));
|
||||
}
|
||||
}
|
||||
if condition.starts_with("fragment=") {
|
||||
fragment_identifier = Some(condition["fragment=".len()..].to_string());
|
||||
}
|
||||
if condition.starts_with("resolution=") {
|
||||
resolution = Some(condition["resolution=".len() ..].to_string());
|
||||
}
|
||||
if condition.starts_with("device-pixel-ratio=") {
|
||||
pixel_ratio = Some(condition["device-pixel-ratio=".len() ..].to_string()
|
||||
.parse().expect("Invalid device-pixel-ratio"));
|
||||
}
|
||||
}
|
||||
|
||||
let reftest = Reftest {
|
||||
name: format!("{} {} {}", test_line.file_left, test_line.kind, test_line.file_right),
|
||||
kind: kind,
|
||||
files: [file_left, file_right],
|
||||
id: id_offset + tests.len(),
|
||||
render_mode: render_mode,
|
||||
servo_args: servo_args.to_vec(),
|
||||
is_flaky: render_mode.intersects(flakiness),
|
||||
prefs: prefs,
|
||||
fragment_identifier: fragment_identifier,
|
||||
resolution: resolution,
|
||||
pixel_ratio: pixel_ratio,
|
||||
};
|
||||
|
||||
if filters.is_empty() || filters.iter().any(|pattern| reftest.name.contains(pattern)) {
|
||||
tests.push(make_test(reftest));
|
||||
}
|
||||
}
|
||||
tests
|
||||
}
|
||||
|
||||
fn make_test(reftest: Reftest) -> TestDescAndFn {
|
||||
let name = reftest.name.clone();
|
||||
TestDescAndFn {
|
||||
desc: TestDesc {
|
||||
name: DynTestName(name),
|
||||
ignore: false,
|
||||
should_panic: ShouldPanic::No,
|
||||
},
|
||||
testfn: DynTestFn(Box::new(move || {
|
||||
check_reftest(reftest);
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
fn capture(reftest: &Reftest, side: usize) -> (u32, u32, Vec<u8>) {
|
||||
let png_filename = format!("/tmp/servo-reftest-{:06}-{}.png", reftest.id, side);
|
||||
let mut command = Command::new(&servo_path());
|
||||
command
|
||||
.args(&reftest.servo_args[..])
|
||||
.arg("--user-stylesheet").arg(util::resource_files::resources_dir_path().join("ahem.css"))
|
||||
// Allows pixel perfect rendering of Ahem font and the HTML canvas for reftests.
|
||||
.arg("-Z")
|
||||
.arg("disable-text-aa,disable-canvas-aa")
|
||||
.args(&["-f", "-o"])
|
||||
.arg(&png_filename)
|
||||
.arg(&{
|
||||
let mut url = Url::from_file_path(&*reftest.files[side]).unwrap();
|
||||
url.fragment = reftest.fragment_identifier.clone();
|
||||
url.to_string()
|
||||
});
|
||||
// CPU rendering is the default
|
||||
if reftest.render_mode.contains(CPU_RENDERING) {
|
||||
command.arg("-c");
|
||||
}
|
||||
if reftest.render_mode.contains(GPU_RENDERING) {
|
||||
command.arg("-g");
|
||||
}
|
||||
for pref in &reftest.prefs {
|
||||
command.arg("--pref");
|
||||
command.arg(pref);
|
||||
}
|
||||
if let Some(ref resolution) = reftest.resolution {
|
||||
command.arg("--resolution");
|
||||
command.arg(resolution);
|
||||
}
|
||||
if let Some(pixel_ratio) = reftest.pixel_ratio {
|
||||
command.arg("--device-pixel-ratio");
|
||||
command.arg(pixel_ratio.to_string());
|
||||
}
|
||||
let (exit_status, stderr, stdout) = match command.output() {
|
||||
Ok(output) => (output.status, output.stderr, output.stdout),
|
||||
Err(e) => panic!("failed to execute process: {}", e),
|
||||
};
|
||||
|
||||
if !stdout.is_empty() {
|
||||
let stdout_filename = format!("/tmp/servo-reftest-{:06}-{}-stdout.txt", reftest.id, side);
|
||||
let mut stdout_file = File::create(stdout_filename).unwrap();
|
||||
stdout_file.write_all(&stdout[..]).unwrap();
|
||||
}
|
||||
|
||||
if !stderr.is_empty() {
|
||||
let stderr_filename = format!("/tmp/servo-reftest-{:06}-{}-stderr.txt", reftest.id, side);
|
||||
let mut stderr_file = File::create(stderr_filename).unwrap();
|
||||
stderr_file.write_all(&stderr[..]).unwrap();
|
||||
}
|
||||
|
||||
assert!(exit_status.success());
|
||||
|
||||
let image = match image::open(&png_filename) {
|
||||
Ok(DynamicImage::ImageRgb8(image)) => image,
|
||||
Ok(image) => image.to_rgb(),
|
||||
_ => panic!(),
|
||||
};
|
||||
(image.width(), image.height(), image.into_raw())
|
||||
}
|
||||
|
||||
fn servo_path() -> PathBuf {
|
||||
let current_exe = env::current_exe().ok().expect("Could not locate current executable");
|
||||
current_exe.parent().unwrap().join("servo")
|
||||
}
|
||||
|
||||
fn check_reftest(reftest: Reftest) {
|
||||
let (left_width, left_height, left_bytes) = capture(&reftest, 0);
|
||||
let (right_width, right_height, right_bytes) = capture(&reftest, 1);
|
||||
|
||||
// TODO(gw): This is a workaround for https://github.com/servo/servo/issues/7730
|
||||
if !reftest.is_flaky {
|
||||
assert_eq!(left_width, right_width);
|
||||
assert_eq!(left_height, right_height);
|
||||
|
||||
let left_all_white = left_bytes.iter().all(|&p| p == 255);
|
||||
let right_all_white = right_bytes.iter().all(|&p| p == 255);
|
||||
|
||||
if left_all_white && right_all_white {
|
||||
panic!("Both renderings are empty")
|
||||
}
|
||||
}
|
||||
|
||||
let pixels = left_bytes.iter().zip(right_bytes.iter()).map(|(&a, &b)| {
|
||||
if a == b {
|
||||
// White for correct
|
||||
0xFF
|
||||
} else {
|
||||
// "1100" in the RGBA channel with an error for an incorrect value
|
||||
// This results in some number of C0 and FFs, which is much more
|
||||
// readable (and distinguishable) than the previous difference-wise
|
||||
// scaling but does not require reconstructing the actual RGBA pixel.
|
||||
0xC0
|
||||
}
|
||||
}).collect::<Vec<u8>>();
|
||||
|
||||
if pixels.iter().any(|&a| a < 255) {
|
||||
let output = format!("/tmp/servo-reftest-{:06}-diff.png", reftest.id);
|
||||
let mut file = File::create(&output).unwrap();
|
||||
let img_buf = RgbImage::from_raw(left_width, left_height, pixels).expect("foo");
|
||||
DynamicImage::ImageRgb8(img_buf).save(&mut file, ImageFormat::PNG).unwrap();
|
||||
|
||||
match (reftest.kind, reftest.is_flaky) {
|
||||
(ReftestKind::Same, true) => println!("flaky test - rendering difference: {}", output),
|
||||
(ReftestKind::Same, false) => panic!("rendering difference: {}", output),
|
||||
(ReftestKind::Different, _) => {} // Result was different and that's what was expected
|
||||
}
|
||||
} else {
|
||||
assert!(reftest.is_flaky || reftest.kind == ReftestKind::Same);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue