From 8dc703f1df419b1089d1826666663f9b6d59e905 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sun, 1 Dec 2019 20:06:25 +0100 Subject: [PATCH] wptrunner: detect solid color screenshots for failing reftests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The output looks like this: ``` ▶ FAIL [expected PASS] /css/CSS2/box-display/root-box-003.xht │ → /css/CSS2/box-display/root-box-003.xht 54a9df64f1476dd12020019d7cf22ac34d727bc0 │ → /css/CSS2/box-display/root-box-003-ref.xht 636eb693bc214b6e1c64e6566c48e69e6777b946 └ → Screenshot is solid color 0xFFFFFF for /css/CSS2/box-display/root-box-003.xht ``` --- tests/wpt/metadata/MANIFEST.json | 2 +- .../wptrunner/wptrunner/executors/base.py | 20 ++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/wpt/metadata/MANIFEST.json b/tests/wpt/metadata/MANIFEST.json index f874a593c60..fc5b8b39759 100644 --- a/tests/wpt/metadata/MANIFEST.json +++ b/tests/wpt/metadata/MANIFEST.json @@ -711776,7 +711776,7 @@ "support" ], "tools/wptrunner/wptrunner/executors/base.py": [ - "713d85001135d0cdf23c7a06583bd03d4355d58e", + "06b1012ec95f552d104b6f416342aa973512c160", "support" ], "tools/wptrunner/wptrunner/executors/executorchrome.py": [ diff --git a/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/executors/base.py b/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/executors/base.py index 713d8500113..06b1012ec95 100644 --- a/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/executors/base.py +++ b/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/executors/base.py @@ -358,17 +358,17 @@ class RefTestImplementation(object): def reset(self): self.screenshot_cache.clear() - def is_pass(self, hashes, screenshots, relation, fuzzy): + def is_pass(self, hashes, screenshots, urls, relation, fuzzy): assert relation in ("==", "!=") if not fuzzy or fuzzy == ((0,0), (0,0)): equal = hashes[0] == hashes[1] # sometimes images can have different hashes, but pixels can be identical. if not equal: self.logger.info("Image hashes didn't match, checking pixel differences") - max_per_channel, pixels_different = self.get_differences(screenshots) + max_per_channel, pixels_different = self.get_differences(screenshots, urls) equal = pixels_different == 0 and max_per_channel == 0 else: - max_per_channel, pixels_different = self.get_differences(screenshots) + max_per_channel, pixels_different = self.get_differences(screenshots, urls) allowed_per_channel, allowed_different = fuzzy self.logger.info("Allowed %s pixels different, maximum difference per channel %s" % ("-".join(str(item) for item in allowed_different), @@ -379,11 +379,13 @@ class RefTestImplementation(object): allowed_different[0] <= pixels_different <= allowed_different[1])) return equal if relation == "==" else not equal - def get_differences(self, screenshots): + def get_differences(self, screenshots, urls): from PIL import Image, ImageChops, ImageStat lhs = Image.open(io.BytesIO(base64.b64decode(screenshots[0]))).convert("RGB") rhs = Image.open(io.BytesIO(base64.b64decode(screenshots[1]))).convert("RGB") + self.check_if_solid_color(lhs, urls[0]) + self.check_if_solid_color(rhs, urls[1]) diff = ImageChops.difference(lhs, rhs) minimal_diff = diff.crop(diff.getbbox()) mask = minimal_diff.convert("L", dither=None) @@ -394,6 +396,12 @@ class RefTestImplementation(object): (count, per_channel)) return per_channel, count + def check_if_solid_color(self, image, url): + extrema = image.getextrema() + if all(min == max for min, max in extrema): + color = ''.join('%02X' % value for value, _ in extrema) + self.message.append("Screenshot is solid color 0x%s for %s\n" % (color, url)) + def run_test(self, test): viewport_size = test.viewport_size dpi = test.dpi @@ -406,6 +414,7 @@ class RefTestImplementation(object): while stack: hashes = [None, None] screenshots = [None, None] + urls = [None, None] nodes, relation = stack.pop() fuzzy = self.get_fuzzy(test, nodes, relation) @@ -416,8 +425,9 @@ class RefTestImplementation(object): return {"status": data[0], "message": data[1]} hashes[i], screenshots[i] = data + urls[i] = node.url - if self.is_pass(hashes, screenshots, relation, fuzzy): + if self.is_pass(hashes, screenshots, urls, relation, fuzzy): fuzzy = self.get_fuzzy(test, nodes, relation) if nodes[1].references: stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))