Add support for viewport size adjustments in wptrunner.

This commit is contained in:
Ms2ger 2015-12-21 15:42:16 +01:00
parent ee6c5ae2fb
commit 1619a908b3
8 changed files with 62 additions and 22 deletions

View file

@ -206,11 +206,12 @@ class RefTestImplementation(object):
def logger(self):
return self.executor.logger
def get_hash(self, test):
def get_hash(self, test, viewport_size):
timeout = test.timeout * self.timeout_multiplier
key = (test.url, viewport_size)
if test.url not in self.screenshot_cache:
success, data = self.executor.screenshot(test)
if key not in self.screenshot_cache:
success, data = self.executor.screenshot(test, viewport_size)
if not success:
return False, data
@ -218,14 +219,14 @@ class RefTestImplementation(object):
screenshot = data
hash_value = hashlib.sha1(screenshot).hexdigest()
self.screenshot_cache[test.url] = (hash_value, None)
self.screenshot_cache[key] = (hash_value, None)
rv = True, (hash_value, screenshot)
rv = (hash_value, screenshot)
else:
rv = True, self.screenshot_cache[test.url]
rv = self.screenshot_cache[key]
self.message.append("%s %s" % (test.url, rv[1][0]))
return rv
self.message.append("%s %s" % (test.url, rv[0]))
return True, rv
def is_pass(self, lhs_hash, rhs_hash, relation):
assert relation in ("==", "!=")
@ -234,6 +235,7 @@ class RefTestImplementation(object):
(relation == "!=" and lhs_hash != rhs_hash))
def run_test(self, test):
viewport_size = test.viewport_size
self.message = []
# Depth-first search of reference tree, with the goal
@ -247,7 +249,7 @@ class RefTestImplementation(object):
nodes, relation = stack.pop()
for i, node in enumerate(nodes):
success, data = self.get_hash(node)
success, data = self.get_hash(node, viewport_size)
if success is False:
return {"status": data[0], "message": data[1]}
@ -264,7 +266,7 @@ class RefTestImplementation(object):
for i, (node, screenshot) in enumerate(zip(nodes, screenshots)):
if screenshot is None:
success, screenshot = self.retake_screenshot(node)
success, screenshot = self.retake_screenshot(node, viewport_size)
if success:
screenshots[i] = screenshot
@ -275,13 +277,14 @@ class RefTestImplementation(object):
"message": "\n".join(self.message),
"extra": {"reftest_screenshots": log_data}}
def retake_screenshot(self, node):
success, data = self.executor.screenshot(node)
def retake_screenshot(self, node, viewport_size):
success, data = self.executor.screenshot(node, viewport_size)
if not success:
return False, data
hash_val, _ = self.screenshot_cache[node.url]
self.screenshot_cache[node.url] = hash_val, data
key = (node.url, viewport_size)
hash_val, _ = self.screenshot_cache[key]
self.screenshot_cache[key] = hash_val, data
return True, data
class Protocol(object):

View file

@ -385,7 +385,10 @@ class MarionetteRefTestExecutor(RefTestExecutor):
return self.convert_result(test, result)
def screenshot(self, test):
def screenshot(self, test, viewport_size):
# https://github.com/w3c/wptrunner/issues/166
assert viewport_size is None
timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None
test_url = self.test_url(test)

View file

@ -247,7 +247,10 @@ class SeleniumRefTestExecutor(RefTestExecutor):
return self.convert_result(test, result)
def screenshot(self, test):
def screenshot(self, test, viewport_size):
# https://github.com/w3c/wptrunner/issues/166
assert viewport_size is None
return SeleniumRun(self._screenshot,
self.protocol.webdriver,
self.test_url(test),

View file

@ -196,7 +196,7 @@ class ServoRefTestExecutor(ProcessTestExecutor):
os.rmdir(self.tempdir)
ProcessTestExecutor.teardown(self)
def screenshot(self, test):
def screenshot(self, test, viewport_size):
full_url = self.test_url(test)
with TempFilename(self.tempdir) as output_path:
@ -213,6 +213,9 @@ class ServoRefTestExecutor(ProcessTestExecutor):
for pref in test.environment.get('prefs', {}):
command += ["--pref", pref]
if viewport_size:
command += ["--resolution", viewport_size]
self.command = debug_args + command
env = os.environ.copy()

View file

@ -226,7 +226,10 @@ class ServoWebDriverRefTestExecutor(RefTestExecutor):
message += traceback.format_exc(e)
return test.result_cls("ERROR", message), []
def screenshot(self, test):
def screenshot(self, test, viewport_size):
# https://github.com/w3c/wptrunner/issues/166
assert viewport_size is None
timeout = (test.timeout * self.timeout_multiplier + extra_timeout
if self.debug_info is None else None)

View file

@ -214,7 +214,9 @@ class ReftestTest(Test):
result_cls = ReftestResult
test_type = "reftest"
def __init__(self, url, inherit_metadata, test_metadata, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
def __init__(self, url, inherit_metadata, test_metadata, references,
timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None,
protocol="http"):
Test.__init__(self, url, inherit_metadata, test_metadata, timeout, path, protocol)
for _, ref_type in references:
@ -222,6 +224,7 @@ class ReftestTest(Test):
raise ValueError
self.references = references
self.viewport_size = viewport_size
@classmethod
def from_manifest(cls,
@ -246,6 +249,7 @@ class ReftestTest(Test):
[],
timeout=timeout,
path=manifest_test.path,
viewport_size=manifest_test.viewport_size,
protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http")
nodes[url] = node

View file

@ -130,26 +130,29 @@ class RefTest(URLManifestItem):
item_type = "reftest"
def __init__(self, source_file, url, references, url_base="/", timeout=None,
manifest=None):
viewport_size=None, manifest=None):
URLManifestItem.__init__(self, source_file, url, url_base=url_base, manifest=manifest)
for _, ref_type in references:
if ref_type not in ["==", "!="]:
raise ValueError, "Unrecognised ref_type %s" % ref_type
self.references = tuple(references)
self.timeout = timeout
self.viewport_size = viewport_size
@property
def is_reference(self):
return self.source_file.name_is_reference
def meta_key(self):
return (self.timeout,)
return (self.timeout, self.viewport_size)
def to_json(self):
rv = URLManifestItem.to_json(self)
rv["references"] = self.references
if self.timeout is not None:
rv["timeout"] = self.timeout
if self.viewport_size is not None:
rv["viewport_size"] = self.viewport_size
return rv
@classmethod
@ -161,6 +164,7 @@ class RefTest(URLManifestItem):
obj["references"],
url_base=manifest.url_base,
timeout=obj.get("timeout"),
viewport_size=obj.get("viewport_size"),
manifest=manifest)

View file

@ -180,6 +180,23 @@ class SourceFile(object):
if timeout_str and timeout_str.lower() == "long":
return timeout_str
@cached_property
def viewport_nodes(self):
"""List of ElementTree Elements corresponding to nodes in a test that
specify viewport sizes"""
return self.root.findall(".//{http://www.w3.org/1999/xhtml}meta[@name='viewport-size']")
@cached_property
def viewport_size(self):
"""The viewport size of a test or reference file"""
if not self.root:
return None
if not self.viewport_nodes:
return None
return self.viewport_nodes[0].attrib.get("content", None)
@cached_property
def testharness_nodes(self):
"""List of ElementTree Elements corresponding to nodes representing a
@ -271,7 +288,7 @@ class SourceFile(object):
rv.append(TestharnessTest(self, url, timeout=self.timeout))
elif self.content_is_ref_node:
rv = [RefTest(self, self.url, self.references, timeout=self.timeout)]
rv = [RefTest(self, self.url, self.references, timeout=self.timeout, viewport_size=self.viewport_size)]
else:
# If nothing else it's a helper file, which we don't have a specific type for