From c497fe6b2c8303e9a3904cf28056b8007d5fbb08 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 21 Aug 2014 21:59:31 +0100 Subject: [PATCH 1/2] Allow marking tests as flaky by target OS. --- src/test/harness/reftest/reftest.rs | 50 +++++++++++++++++++---------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/src/test/harness/reftest/reftest.rs b/src/test/harness/reftest/reftest.rs index 6046cd62483..919130b76f0 100644 --- a/src/test/harness/reftest/reftest.rs +++ b/src/test/harness/reftest/reftest.rs @@ -21,10 +21,17 @@ use test::{AutoColor, DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn} use test::run_tests_console; use regex::Regex; -enum RenderMode { - CpuRendering = 1, - GpuRendering = 2, -} + +bitflags!( + flags RenderMode: u32 { + static CpuRendering = 0x00000001, + static GpuRendering = 0x00000010, + static LinuxTarget = 0x00000100, + static MacOsTarget = 0x00001000, + static AndroidTarget = 0x00010000 + } +) + fn main() { let args = os::args(); @@ -39,11 +46,20 @@ fn main() { [ref render_mode_string, ref base_path, ref testname, ..] => (render_mode_string, base_path, Some(Regex::new(testname.as_slice()).unwrap())), }; - let render_mode = match render_mode_string.as_slice() { + let mut render_mode = match render_mode_string.as_slice() { "cpu" => CpuRendering, "gpu" => GpuRendering, _ => fail!("First argument must specify cpu or gpu as rendering mode") }; + if cfg!(target_os = "linux") { + render_mode.insert(LinuxTarget); + } + if cfg!(target_os = "macos") { + render_mode.insert(MacOsTarget); + } + if cfg!(target_os = "android") { + render_mode.insert(AndroidTarget); + } let mut all_tests = vec!(); println!("Scanning {} for manifests\n", base_path); @@ -97,7 +113,7 @@ struct Reftest { id: uint, servo_args: Vec, render_mode: RenderMode, - flakiness: uint, + is_flaky: bool, } struct TestLine<'a> { @@ -150,11 +166,13 @@ fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode) -> Ve let file_right = src_dir.append("/").append(test_line.file_right); let mut conditions_list = test_line.conditions.split(','); - let mut flakiness = 0; + let mut flakiness = RenderMode::empty(); for condition in conditions_list { match condition { - "flaky_cpu" => flakiness |= CpuRendering as uint, - "flaky_gpu" => flakiness |= GpuRendering as uint, + "flaky_cpu" => flakiness.insert(CpuRendering), + "flaky_gpu" => flakiness.insert(GpuRendering), + "flaky_linux" => flakiness.insert(LinuxTarget), + "flaky_macos" => flakiness.insert(MacOsTarget), _ => (), } } @@ -166,7 +184,7 @@ fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode) -> Ve id: next_id, render_mode: render_mode, servo_args: servo_args.iter().map(|x| x.clone()).collect(), - flakiness: flakiness, + is_flaky: render_mode.intersects(flakiness), }; next_id += 1; @@ -193,9 +211,9 @@ fn make_test(reftest: Reftest) -> TestDescAndFn { fn capture(reftest: &Reftest, side: uint) -> png::Image { let filename = format!("/tmp/servo-reftest-{:06u}-{:u}.png", reftest.id, side); let mut args = reftest.servo_args.clone(); - match reftest.render_mode { - CpuRendering => args.push("-c".to_string()), - _ => {} // GPU rendering is the default + // GPU rendering is the default + if reftest.render_mode.contains(CpuRendering) { + args.push("-c".to_string()); } args.push_all_move(vec!("-f".to_string(), "-o".to_string(), filename.clone(), reftest.files[side].clone())); @@ -225,8 +243,6 @@ fn check_reftest(reftest: Reftest) { } }).collect::>(); - let test_is_flaky = (reftest.render_mode as uint & reftest.flakiness) != 0; - if pixels.iter().any(|&a| a < 255) { let output_str = format!("/tmp/servo-reftest-{:06u}-diff.png", reftest.id); let output = from_str::(output_str.as_slice()).unwrap(); @@ -240,12 +256,12 @@ fn check_reftest(reftest: Reftest) { let res = png::store_png(&mut img, &output); assert!(res.is_ok()); - match (reftest.kind, test_is_flaky) { + match (reftest.kind, reftest.is_flaky) { (Same, true) => println!("flaky test - rendering difference: {}", output_str), (Same, false) => fail!("rendering difference: {}", output_str), (Different, _) => {} // Result was different and that's what was expected } } else { - assert!(test_is_flaky || reftest.kind == Same); + assert!(reftest.is_flaky || reftest.kind == Same); } } From 4d4c0346009a44268575e0bf14a146e53aa2954f Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Tue, 1 Jul 2014 19:48:25 +0100 Subject: [PATCH 2/2] Add a (broken) Acid2 reftest. Fix #1993 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rendering is incorrect and the test is marked as flaky on Linux or on GPU rendering, but this should help catch further regressions. The test is modified to add the following CSS: .intro { display: none } html #top { margin-top: 0 } To disable the "scrolling" part of the test. The reference is modified to match the current rendering of the test: * Rows 4 and 5 (eyes) have a red background. * The nose is not quite where it should be. * Row 12’s height is 3px more than it should be. (See http://www.webstandards.org/action/acid2/guide/ for row numbers.) The nose issue seems be related to rounding vs truncating when pixel-snapping borders, but this is only a guess. --- src/test/harness/reftest/reftest.rs | 2 +- src/test/html/acid2.html | 1 - src/test/ref/acid2_noscroll.html | 152 ++++++++++++++++++++++++++++ src/test/ref/acid2_ref.html | 17 ++++ src/test/ref/acid2_ref.png | Bin 0 -> 2261 bytes src/test/ref/acid2_ref_broken.html | 17 ++++ src/test/ref/acid2_ref_broken.png | Bin 0 -> 2174 bytes src/test/ref/basic.list | 8 ++ 8 files changed, 195 insertions(+), 2 deletions(-) create mode 100755 src/test/ref/acid2_noscroll.html create mode 100644 src/test/ref/acid2_ref.html create mode 100644 src/test/ref/acid2_ref.png create mode 100644 src/test/ref/acid2_ref_broken.html create mode 100644 src/test/ref/acid2_ref_broken.png diff --git a/src/test/harness/reftest/reftest.rs b/src/test/harness/reftest/reftest.rs index 919130b76f0..be9bd308603 100644 --- a/src/test/harness/reftest/reftest.rs +++ b/src/test/harness/reftest/reftest.rs @@ -178,7 +178,7 @@ fn parse_lists(file: &str, servo_args: &[String], render_mode: RenderMode) -> Ve } let reftest = Reftest { - name: test_line.file_left.to_string().append(" / ").append(test_line.file_right), + name: format!("{} {} {}", test_line.file_left, test_line.kind, test_line.file_right), kind: kind, files: [file_left, file_right], id: next_id, diff --git a/src/test/html/acid2.html b/src/test/html/acid2.html index 411c4d7134a..b6432099e4d 100755 --- a/src/test/html/acid2.html +++ b/src/test/html/acid2.html @@ -1,4 +1,3 @@ - diff --git a/src/test/ref/acid2_noscroll.html b/src/test/ref/acid2_noscroll.html new file mode 100755 index 00000000000..5c6e6ff1112 --- /dev/null +++ b/src/test/ref/acid2_noscroll.html @@ -0,0 +1,152 @@ + + + + The Second Acid Test + + + + + + +
+

Standards compliant?

+

Take The Acid2 Test and compare it to the reference rendering.

+
+

Hello World!

+
+

+

+
                              
+
ERROR
+
+
+
+
 
+
+
    +
  • +
  • +
  • +
  • +
+
+
+ + diff --git a/src/test/ref/acid2_ref.html b/src/test/ref/acid2_ref.html new file mode 100644 index 00000000000..a6cc0c01b28 --- /dev/null +++ b/src/test/ref/acid2_ref.html @@ -0,0 +1,17 @@ + + + + The Second Acid Test (Reference Rendering) + + + +

Hello World!

+

Follow this link to view the reference image, which should be rendered below the text "Hello World!" on the test page in the same way that this paragraph is rendered below that text on this page.

+ + diff --git a/src/test/ref/acid2_ref.png b/src/test/ref/acid2_ref.png new file mode 100644 index 0000000000000000000000000000000000000000..7aee7609d6ade6e39ad53b04a9f61e55f3b00c76 GIT binary patch literal 2261 zcmZvecT^L|7KbC!qzedn8x+GTMFL1!kN}}~hzJQ5qzY1$B7_hUP>7)g2uchVq&MjT zA}G>U5=5FTi$JJ?6lqDQK|mhvThDRd**P<{*5-xn{(f!$G1n+5=QpIVw2A~5vj3~L+E;0aoDm2>m^PE}dWVqeG!qLa-4I$DN6 zL?{qHJ6aAv&VbR*&;3$gE-JrXQHnk$>J@XZS|HH^NwGjkSQ+>gOs(Uy?}-;tRc)k# zS~O{}uvXq&jW40Qw7H!Q{?4@Jz@FZaS+@RM(lIb62}I0AATpy-a7lpPu@mwpaB;ma5!3ej5iOawKwbzhwoC?nSmui*SU7#)vs)V{dc)<}VW+ z1WZpK8jI-N9fX#Jr`ulCWx*tD^BpOz>7dyW?wO-!1LoFWBy>VL;<__et zDY7L*d8+5QE;hf`!c}@aoJUqHV1hjmLN(d+ z8#-OY?b+`1iF=aG#)X(0<{Do?m1 z0C2$x`uj7n?rNV{*OEvc?uRBK!-g4Kr3xEG8`4EaTKK;0AQ^!cU-Z|T&XS{&y{~$9 z_P*_GG4AYb4D?-ivlDQ%9UES_ie)ju5|a0=z!Jvt1g-IRrn^@@IMmei ztE;ow(;i$s>PTBb-4kUMG87}V1RrrHCm)|E7nRF1aNeaABjo)KfPt;x5_-AV00063 zfxzb@%bisP#UGNrN4D-!mtzjRnW5CN0BIm_XvpBCU0cg`QT0e|AX@e?lDZP0?VXio z8zFnwgafNbh-!_Znf*E7%k!La>=c=~9YM?;#fb`-&AD#Mw$AT1${Ctoh=94OB~U0b z+&$`)-0TXW>2?rjB>0mF z+s)0ZW|;7SPr+1XAK|!o-`6x69*4RDV@dGa$j+na-F^dR;Yb?@cj<-?*|sn^jrl$t zT~Wxb2_i&^#8XLU);2cm#7*-0?oL?1Yu3CNlNvhC5`%^;lM8mPu5B;2i`0N3iM}H@ zYADa95D8Pqg~~P3tqhS>HF<*QD^BieV4wgW7z&DjISqD>w|Xxw++-dB<6E;k2p>Pn z6n3jqM23c=OJpd2SiKU8E1N2J)X>T+;4=F))v~O@ZwiO|6y)aNVT2smq?7np61^AR z>_==rhBCOesGD>lBvMXRR!V4nQ$p20vzXAba3<9fnC#eLcGY6y37^`E&kd)vS<={nhfy-6EJ@)k|QgSzgbOhd(Rp>wITj6Lbz5 z=-5RjlcQTuVOm%$5%^mc!pJR9*xEY2-MuEKTu}MqV;(6ln9ksbs7>@KGQEo*X&Ryq zf7&v6QbJNyO>K#V*}c6lw{l(w6rrL>A|(mn8i7)f6$N^P{gFe+7%E5H;pJtYKyzc` zS8-2d_^ToBKBKJ*M1-yd2!$~`a69v?D%JWiCI&7nC}=8k9|g?_k3~U`Cz!;>#_DGF z2WZh~!NN?dx~C5QfzvC_UEN?QvojQ26kc1Y0Rz6*;pT0gj=#lrgTz*;qR>W_=B}<2 zY&LszbaeHvap17o7+2RUmy{lOQu6WSw8Y3xZ6q|=f>TH5o#!f_9db%IeKt#Y{B?}= z!%6hpo3N5idA_de^<(v@BXt23i|}v7F7?qfqOJ*KrL1cqVqq(9Fsc2+H9j{Xz2NGr z#U-9H+zJmdJ*5vf_#e!$(h-*{Tl8G)mmf^{!5bAFBA#PU@+?(aaj|zS2a{0N zx#S}ED%5vu(Z?CS1-p@}U_N20R zII`KV?CYJLnQXh*TD-k@kc(yhDMb@6h2iUwN=Gc|#)V{!{fi}t($*QM(vFIsR|9tx zN=(&7{?hM?`tQzdENFKJyD*IZ=tzsJE@*t8c&J)tJ0`L|8~<-*kcm0@`*!!oniUbg z#*#iG+rq~PKa;TNZ@iLA^77Xf_>qY)5`wPHyTV#p_=~y!YzmnOE)Czf+;|~+cHCMi z&md&>Rr>*$zl;8vJh)8$QaP+tSn)&twpHo2q@R+3f^$%c_r?Eb`uD2{fz#y!k`Ppk rv;5q-m*6c7dgE{6|D4^D%G5vlSW`2yTA=9Qy8^H@g_~3ux#9l{WcO5+ literal 0 HcmV?d00001 diff --git a/src/test/ref/acid2_ref_broken.html b/src/test/ref/acid2_ref_broken.html new file mode 100644 index 00000000000..aecefd1ef02 --- /dev/null +++ b/src/test/ref/acid2_ref_broken.html @@ -0,0 +1,17 @@ + + + + The Second Acid Test (Reference Rendering) + + + +

Hello World!

+

Follow this link to view the reference image, which should be rendered below the text "Hello World!" on the test page in the same way that this paragraph is rendered below that text on this page.

+ + diff --git a/src/test/ref/acid2_ref_broken.png b/src/test/ref/acid2_ref_broken.png new file mode 100644 index 0000000000000000000000000000000000000000..91c803bdadc33ee0abd2a05d05dc2ef3f6ddb1f7 GIT binary patch literal 2174 zcmaJ@dpOf=AD^?fa(Xt3Ne&|(Pc?@q)tam|G-A=5vm~{gR+*kKdmRd;R|s1*Q6nP7 zq7CbS9HRp{%qCe<7*RZPcz(RS{qr8K`?~MX=Xc+q`*U5t-}if6pWh`{Ja!#e8w>)0 z*5M8~xB*xM%o1fqV4W8BXak@a=!A6uEw2n#E&Cd`zA)5wbPq5uy3_FJBv% z#q(8&^&7wEh7K5QdvNVWMZJsmn$p>CJmCfD;AH#cI9bD9nY<~($aH2t>_KaTu>+Lg zfX1LPzhIR>31F<}Ab#@~ujnqP-rgn?f0;;m7@MlD08@Y+a{URbZ7PllXF%MUpLrRWyNEsYG#xUc%%aqVXZr{|L2Qvt2Ww=LSY0ZMd zTz_8i*U#BsMZO}aWTQm?!GOhC1YcA1hX^FG6q)Q8pDdU$yj1ZX?tTj+67F3$w@2d3 z;_mI)g_G{a-Dp9h6qaO|N@WfFFvW8X5G##u<&SAXS*`egBz?7`Rg+$EUFxR&=6 zdz3ii@sa0_Cmj-f;gtCku1q2`w?X{9ph;p+B=b>b+sq)odlEi{acjh{VxW`Dv;`)E+m2nAcvr3`^U zjs^xAtehr&m`q~A$49Y-jbefm&o3Rx>uRJNoZPn)jHGXoyG}#BSh$wsq$?Y%#(okN z?-_4rXvjk%(RbXAnwm!hO<8D#dgOBLl=z}<^!M4U-r5+|-8j|Y4(8B{so|N9j%fzm z`N#OzCJRY$fbrdZFLupD2ZW`04`Ms*KeF_i6uJTzmThR04VbR%>Z`Ue_H+N*X>gWf zVqdNHWLs_f%fLJ2ybPA{Q9z|r?0fmfeUciUQa|RBEhlek$ z?;nE_XA9~_+IbqJ=;Rrj5!ejmNPe_?EM@HpPu@JQ0I^)7jZKQIan3OEsV3g>LM`(- zC9~fEj=*WOj99c*+{gbDgUVQ6!VdgupElyws^f>&qofEcjW_t`6A{D_kF^V9f8^9~ zDL*Jx<$DLi&c$4b%Xx+-|wGp}M@dVY^GG&!zx&FcL?* zUSr>o49}^SQa%WMM{Me`ORb8EhzWlv+X5N9` zHw;o^6V{h_r3p@&o?+8D>3*VqE2p!92>mH^DocsPH!(3Oes}WgU|n*-zNYtF(u`C8 z0GE`x@&NQ$mDlCwX56_(&Q6TxqB7E>6?yJCGP@Ss=cgDl$lTghn&NYpcuUnP>Kh!? z(b2Ze+8X2X`uWgPLnd{?+KlTLP#u*saZqq3Z0tC|TFSI+rv^^JH}0|rRBzuxgM*LT zssXhcpTJ&30%i9|LBh>lmwa*r&N2K6TaD6Cd|5-62$#o=-Xarrb64}rmn$u~(mekD z^YMshHZC`hS#i3W$D>v)`n0iP!PMyQufgS;85r4wKh?zgJk_Cewb0-P&4IOKP2%p* zrX#g_q^R=tPbmycccm7)#N~v?0WV5sli)7LkJEqJoARn!@q*=*!to`8hEkhUD893s zA4im3+@=!~^>EL2bCfg6!@`HWdDL)Py?s2|fA-L&Pa)Z$Z_b0#d{)M(hnjC+H8@QO zLHYtaKC79ed_D5zS@|rV0@iJo0oV*|_6t_YZI(ae61M_sH5qAEOc%#~B)BQGFIJWn zg7c(IcO1;>2gCqzxO1SqJBxs({LaoWJ25e`qN!y%-ec8VUzMy}O`(UvAD%ywsQT2k zbD*V=3C{DzWw`@aiX+WVgdz^(7259?wiL5!vIsk1R*y5x*mSXn+X$g^!13O{#Xap~ zem}$EKeMM>*H=Gi$`&_Y8DClTxl+~g!FNF~^==qSpcnpsQph*jF2D5DQ=QAVCl3R< zm7w~WP`?sOT_H<0BjM`ls_kE1bvbd%wM@%BIa+Q1GpkmE5lS%9JGL~+D6JP8FU~`y z^q|F}TUQ@}cF