Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-08-07 06:25:32 +01:00
Code Issues Projects Releases Packages Wiki Activity

wptrunner: detect solid color screenshots for failing reftests

Browse source 
The output looks like this:

```
  ▶ FAIL [expected PASS] /css/CSS2/box-display/root-box-003.xht
  │   → /css/CSS2/box-display/root-box-003.xht 54a9df64f1476dd12020019d7cf22ac34d727bc0
  │   → /css/CSS2/box-display/root-box-003-ref.xht 636eb693bc214b6e1c64e6566c48e69e6777b946
  └   → Screenshot is solid color 0xFFFFFF for /css/CSS2/box-display/root-box-003.xht
```
This commit is contained in:
Simon Sapin 2019-12-01 20:06:25 +01:00
parent 40ce8c7105
commit 8dc703f1df
2 changed files with 16 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/wpt/metadata/MANIFEST.json
View file

@ -711776,7 +711776,7 @@
"support" "support"
], ],
"tools/wptrunner/wptrunner/executors/base.py": [ "tools/wptrunner/wptrunner/executors/base.py": [
"713d85001135d0cdf23c7a06583bd03d4355d58e", "06b1012ec95f552d104b6f416342aa973512c160",
"support" "support"
], ],
"tools/wptrunner/wptrunner/executors/executorchrome.py": [ "tools/wptrunner/wptrunner/executors/executorchrome.py": [

20
tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/executors/base.py
View file

@ -358,17 +358,17 @@ class RefTestImplementation(object):
def reset(self): def reset(self):
self.screenshot_cache.clear() self.screenshot_cache.clear()
def is_pass(self, hashes, screenshots, relation, fuzzy): def is_pass(self, hashes, screenshots, urls, relation, fuzzy):
assert relation in ("==", "!=") assert relation in ("==", "!=")
if not fuzzy or fuzzy == ((0,0), (0,0)): if not fuzzy or fuzzy == ((0,0), (0,0)):
equal = hashes[0] == hashes[1] equal = hashes[0] == hashes[1]
# sometimes images can have different hashes, but pixels can be identical. # sometimes images can have different hashes, but pixels can be identical.
if not equal: if not equal:
self.logger.info("Image hashes didn't match, checking pixel differences") self.logger.info("Image hashes didn't match, checking pixel differences")
max_per_channel, pixels_different = self.get_differences(screenshots) max_per_channel, pixels_different = self.get_differences(screenshots, urls)
equal = pixels_different == 0 and max_per_channel == 0 equal = pixels_different == 0 and max_per_channel == 0
else: else:
max_per_channel, pixels_different = self.get_differences(screenshots) max_per_channel, pixels_different = self.get_differences(screenshots, urls)
allowed_per_channel, allowed_different = fuzzy allowed_per_channel, allowed_different = fuzzy
self.logger.info("Allowed %s pixels different, maximum difference per channel %s" % self.logger.info("Allowed %s pixels different, maximum difference per channel %s" %
("-".join(str(item) for item in allowed_different), ("-".join(str(item) for item in allowed_different),
@ -379,11 +379,13 @@ class RefTestImplementation(object):
allowed_different[0] <= pixels_different <= allowed_different[1])) allowed_different[0] <= pixels_different <= allowed_different[1]))
return equal if relation == "==" else not equal return equal if relation == "==" else not equal
def get_differences(self, screenshots): def get_differences(self, screenshots, urls):
from PIL import Image, ImageChops, ImageStat from PIL import Image, ImageChops, ImageStat
lhs = Image.open(io.BytesIO(base64.b64decode(screenshots[0]))).convert("RGB") lhs = Image.open(io.BytesIO(base64.b64decode(screenshots[0]))).convert("RGB")
rhs = Image.open(io.BytesIO(base64.b64decode(screenshots[1]))).convert("RGB") rhs = Image.open(io.BytesIO(base64.b64decode(screenshots[1]))).convert("RGB")
self.check_if_solid_color(lhs, urls[0])
self.check_if_solid_color(rhs, urls[1])
diff = ImageChops.difference(lhs, rhs) diff = ImageChops.difference(lhs, rhs)
minimal_diff = diff.crop(diff.getbbox()) minimal_diff = diff.crop(diff.getbbox())
mask = minimal_diff.convert("L", dither=None) mask = minimal_diff.convert("L", dither=None)
@ -394,6 +396,12 @@ class RefTestImplementation(object):
(count, per_channel)) (count, per_channel))
return per_channel, count return per_channel, count
def check_if_solid_color(self, image, url):
extrema = image.getextrema()
if all(min == max for min, max in extrema):
color = ''.join('%02X' % value for value, _ in extrema)
self.message.append("Screenshot is solid color 0x%s for %s\n" % (color, url))
def run_test(self, test): def run_test(self, test):
viewport_size = test.viewport_size viewport_size = test.viewport_size
dpi = test.dpi dpi = test.dpi
@ -406,6 +414,7 @@ class RefTestImplementation(object):
while stack: while stack:
hashes = [None, None] hashes = [None, None]
screenshots = [None, None] screenshots = [None, None]
urls = [None, None]
nodes, relation = stack.pop() nodes, relation = stack.pop()
fuzzy = self.get_fuzzy(test, nodes, relation) fuzzy = self.get_fuzzy(test, nodes, relation)
@ -416,8 +425,9 @@ class RefTestImplementation(object):
return {"status": data[0], "message": data[1]} return {"status": data[0], "message": data[1]}
hashes[i], screenshots[i] = data hashes[i], screenshots[i] = data
urls[i] = node.url
if self.is_pass(hashes, screenshots, relation, fuzzy): if self.is_pass(hashes, screenshots, urls, relation, fuzzy):
fuzzy = self.get_fuzzy(test, nodes, relation) fuzzy = self.get_fuzzy(test, nodes, relation)
if nodes[1].references: if nodes[1].references:
stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references))) stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))