diff --git a/tests/wpt/harness/wptrunner/executors/base.py b/tests/wpt/harness/wptrunner/executors/base.py index 1ff09a5f030..614eeee86e6 100644 --- a/tests/wpt/harness/wptrunner/executors/base.py +++ b/tests/wpt/harness/wptrunner/executors/base.py @@ -206,11 +206,12 @@ class RefTestImplementation(object): def logger(self): return self.executor.logger - def get_hash(self, test): + def get_hash(self, test, viewport_size): timeout = test.timeout * self.timeout_multiplier + key = (test.url, viewport_size) - if test.url not in self.screenshot_cache: - success, data = self.executor.screenshot(test) + if key not in self.screenshot_cache: + success, data = self.executor.screenshot(test, viewport_size) if not success: return False, data @@ -218,14 +219,14 @@ class RefTestImplementation(object): screenshot = data hash_value = hashlib.sha1(screenshot).hexdigest() - self.screenshot_cache[test.url] = (hash_value, None) + self.screenshot_cache[key] = (hash_value, None) - rv = True, (hash_value, screenshot) + rv = (hash_value, screenshot) else: - rv = True, self.screenshot_cache[test.url] + rv = self.screenshot_cache[key] - self.message.append("%s %s" % (test.url, rv[1][0])) - return rv + self.message.append("%s %s" % (test.url, rv[0])) + return True, rv def is_pass(self, lhs_hash, rhs_hash, relation): assert relation in ("==", "!=") @@ -234,6 +235,7 @@ class RefTestImplementation(object): (relation == "!=" and lhs_hash != rhs_hash)) def run_test(self, test): + viewport_size = test.viewport_size self.message = [] # Depth-first search of reference tree, with the goal @@ -247,7 +249,7 @@ class RefTestImplementation(object): nodes, relation = stack.pop() for i, node in enumerate(nodes): - success, data = self.get_hash(node) + success, data = self.get_hash(node, viewport_size) if success is False: return {"status": data[0], "message": data[1]} @@ -264,7 +266,7 @@ class RefTestImplementation(object): for i, (node, screenshot) in enumerate(zip(nodes, screenshots)): if screenshot is None: - success, screenshot = self.retake_screenshot(node) + success, screenshot = self.retake_screenshot(node, viewport_size) if success: screenshots[i] = screenshot @@ -275,13 +277,14 @@ class RefTestImplementation(object): "message": "\n".join(self.message), "extra": {"reftest_screenshots": log_data}} - def retake_screenshot(self, node): - success, data = self.executor.screenshot(node) + def retake_screenshot(self, node, viewport_size): + success, data = self.executor.screenshot(node, viewport_size) if not success: return False, data - hash_val, _ = self.screenshot_cache[node.url] - self.screenshot_cache[node.url] = hash_val, data + key = (node.url, viewport_size) + hash_val, _ = self.screenshot_cache[key] + self.screenshot_cache[key] = hash_val, data return True, data class Protocol(object): diff --git a/tests/wpt/harness/wptrunner/executors/executormarionette.py b/tests/wpt/harness/wptrunner/executors/executormarionette.py index 90c76759faf..ecae5ac5e77 100644 --- a/tests/wpt/harness/wptrunner/executors/executormarionette.py +++ b/tests/wpt/harness/wptrunner/executors/executormarionette.py @@ -385,7 +385,10 @@ class MarionetteRefTestExecutor(RefTestExecutor): return self.convert_result(test, result) - def screenshot(self, test): + def screenshot(self, test, viewport_size): + # https://github.com/w3c/wptrunner/issues/166 + assert viewport_size is None + timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None test_url = self.test_url(test) diff --git a/tests/wpt/harness/wptrunner/executors/executorselenium.py b/tests/wpt/harness/wptrunner/executors/executorselenium.py index 6298893503b..5671d476f57 100644 --- a/tests/wpt/harness/wptrunner/executors/executorselenium.py +++ b/tests/wpt/harness/wptrunner/executors/executorselenium.py @@ -247,7 +247,10 @@ class SeleniumRefTestExecutor(RefTestExecutor): return self.convert_result(test, result) - def screenshot(self, test): + def screenshot(self, test, viewport_size): + # https://github.com/w3c/wptrunner/issues/166 + assert viewport_size is None + return SeleniumRun(self._screenshot, self.protocol.webdriver, self.test_url(test), diff --git a/tests/wpt/harness/wptrunner/executors/executorservo.py b/tests/wpt/harness/wptrunner/executors/executorservo.py index b1beea9934a..0f25aaf7b71 100644 --- a/tests/wpt/harness/wptrunner/executors/executorservo.py +++ b/tests/wpt/harness/wptrunner/executors/executorservo.py @@ -196,7 +196,7 @@ class ServoRefTestExecutor(ProcessTestExecutor): os.rmdir(self.tempdir) ProcessTestExecutor.teardown(self) - def screenshot(self, test): + def screenshot(self, test, viewport_size): full_url = self.test_url(test) with TempFilename(self.tempdir) as output_path: @@ -213,6 +213,9 @@ class ServoRefTestExecutor(ProcessTestExecutor): for pref in test.environment.get('prefs', {}): command += ["--pref", pref] + if viewport_size: + command += ["--resolution", viewport_size] + self.command = debug_args + command env = os.environ.copy() diff --git a/tests/wpt/harness/wptrunner/executors/executorservodriver.py b/tests/wpt/harness/wptrunner/executors/executorservodriver.py index 8a914a876eb..0317ca61cb8 100644 --- a/tests/wpt/harness/wptrunner/executors/executorservodriver.py +++ b/tests/wpt/harness/wptrunner/executors/executorservodriver.py @@ -226,7 +226,10 @@ class ServoWebDriverRefTestExecutor(RefTestExecutor): message += traceback.format_exc(e) return test.result_cls("ERROR", message), [] - def screenshot(self, test): + def screenshot(self, test, viewport_size): + # https://github.com/w3c/wptrunner/issues/166 + assert viewport_size is None + timeout = (test.timeout * self.timeout_multiplier + extra_timeout if self.debug_info is None else None) diff --git a/tests/wpt/harness/wptrunner/wpttest.py b/tests/wpt/harness/wptrunner/wpttest.py index 9186b9e96c2..82bcc689b6b 100644 --- a/tests/wpt/harness/wptrunner/wpttest.py +++ b/tests/wpt/harness/wptrunner/wpttest.py @@ -214,7 +214,9 @@ class ReftestTest(Test): result_cls = ReftestResult test_type = "reftest" - def __init__(self, url, inherit_metadata, test_metadata, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"): + def __init__(self, url, inherit_metadata, test_metadata, references, + timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None, + protocol="http"): Test.__init__(self, url, inherit_metadata, test_metadata, timeout, path, protocol) for _, ref_type in references: @@ -222,6 +224,7 @@ class ReftestTest(Test): raise ValueError self.references = references + self.viewport_size = viewport_size @classmethod def from_manifest(cls, @@ -246,6 +249,7 @@ class ReftestTest(Test): [], timeout=timeout, path=manifest_test.path, + viewport_size=manifest_test.viewport_size, protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http") nodes[url] = node diff --git a/tests/wpt/web-platform-tests/tools/manifest/item.py b/tests/wpt/web-platform-tests/tools/manifest/item.py index 58ff22a3b0e..f03ea9d61b5 100644 --- a/tests/wpt/web-platform-tests/tools/manifest/item.py +++ b/tests/wpt/web-platform-tests/tools/manifest/item.py @@ -130,26 +130,29 @@ class RefTest(URLManifestItem): item_type = "reftest" def __init__(self, source_file, url, references, url_base="/", timeout=None, - manifest=None): + viewport_size=None, manifest=None): URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest) for _, ref_type in references: if ref_type not in ["==", "!="]: raise ValueError, "Unrecognised ref_type %s" % ref_type self.references = tuple(references) self.timeout = timeout + self.viewport_size = viewport_size @property def is_reference(self): return self.source_file.name_is_reference def meta_key(self): - return (self.timeout,) + return (self.timeout, self.viewport_size) def to_json(self): rv = URLManifestItem.to_json(self) rv["references"] = self.references if self.timeout is not None: rv["timeout"] = self.timeout + if self.viewport_size is not None: + rv["viewport_size"] = self.viewport_size return rv @classmethod @@ -161,6 +164,7 @@ class RefTest(URLManifestItem): obj["references"], url_base=manifest.url_base, timeout=obj.get("timeout"), + viewport_size=obj.get("viewport_size"), manifest=manifest) diff --git a/tests/wpt/web-platform-tests/tools/manifest/sourcefile.py b/tests/wpt/web-platform-tests/tools/manifest/sourcefile.py index f16ec8a35a9..8950e7b1fbc 100644 --- a/tests/wpt/web-platform-tests/tools/manifest/sourcefile.py +++ b/tests/wpt/web-platform-tests/tools/manifest/sourcefile.py @@ -180,6 +180,23 @@ class SourceFile(object): if timeout_str and timeout_str.lower() == "long": return timeout_str + @cached_property + def viewport_nodes(self): + """List of ElementTree Elements corresponding to nodes in a test that + specify viewport sizes""" + return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='viewport-size']") + + @cached_property + def viewport_size(self): + """The viewport size of a test or reference file""" + if not self.root: + return None + + if not self.viewport_nodes: + return None + + return self.viewport_nodes[0].attrib.get("content", None) + @cached_property def testharness_nodes(self): """List of ElementTree Elements corresponding to nodes representing a @@ -271,7 +288,7 @@ class SourceFile(object): rv.append(TestharnessTest(self, url, timeout=self.timeout)) elif self.content_is_ref_node: - rv = [RefTest(self, self.url, self.references, timeout=self.timeout)] + rv = [RefTest(self, self.url, self.references, timeout=self.timeout, viewport_size=self.viewport_size)] else: # If nothing else it's a helper file, which we don't have a specific type for