mirror of
https://github.com/servo/servo.git
synced 2025-08-17 19:35:33 +01:00
Update web-platform-tests to revision e03a9b1341ae9bdb1e4fa03765257b84d26fe2f1
This commit is contained in:
Josh Matthews
parent
7d05c76d18
commit
20a833eb75
5167 changed files with 4696 additions and 297370 deletions
|
|
@ -63,20 +63,21 @@ class HTTPWireProtocol(object):
|
|||
self._timeout = timeout
|
||||
|
||||
def url(self, suffix):
|
||||
return urlparse.urljoin(self.path_prefix, suffix)
|
||||
return urlparse.urljoin(self.url_prefix, suffix)
|
||||
|
||||
def send(self, method, url, body=None, headers=None):
|
||||
def send(self, method, uri, body=None, headers=None):
|
||||
"""Send a command to the remote.
|
||||
|
||||
:param method: "POST" or "GET".
|
||||
:param url: "command part" of the requests URL path
|
||||
:param body: Body of the request. Defaults to an empty dictionary
|
||||
if ``method`` is "POST".
|
||||
:param method: `GET`, `POST`, or `DELETE`.
|
||||
:param uri: Relative endpoint of the requests URL path.
|
||||
:param body: Body of the request. Defaults to an empty
|
||||
dictionary if ``method`` is `POST`.
|
||||
:param headers: Additional headers to include in the request.
|
||||
:return: an instance of wdclient.Response describing the HTTP response
|
||||
received from the remote end.
|
||||
"""
|
||||
|
||||
:return: Instance of ``wdclient.Response`` describing the
|
||||
HTTP response received from the remote end.
|
||||
|
||||
"""
|
||||
if body is None and method == "POST":
|
||||
body = {}
|
||||
|
||||
|
|
@ -89,7 +90,7 @@ class HTTPWireProtocol(object):
|
|||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
url = self.url_prefix + url
|
||||
url = self.url(uri)
|
||||
|
||||
kwargs = {}
|
||||
if self._timeout is not None:
|
||||
|
|
@ -100,8 +101,7 @@ class HTTPWireProtocol(object):
|
|||
conn.request(method, url, body, headers)
|
||||
|
||||
try:
|
||||
response = Response.from_http_response(conn.getresponse())
|
||||
response = conn.getresponse()
|
||||
return Response.from_http_response(response)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -70,9 +70,9 @@ class Firefox(Browser):
|
|||
raise ValueError("Unable to construct a valid Firefox package name for current platform")
|
||||
|
||||
if platform == "linux":
|
||||
bits = "-%s" % uname[-1]
|
||||
bits = "-%s" % uname[4]
|
||||
elif platform == "win":
|
||||
bits = "64" if uname[-1] == "x86_64" else "32"
|
||||
bits = "64" if uname[4] == "x86_64" else "32"
|
||||
else:
|
||||
bits = ""
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ class Firefox(Browser):
|
|||
raise ValueError("Unable to construct a valid Geckodriver package name for current platform")
|
||||
|
||||
if platform in ("linux", "win"):
|
||||
bits = "64" if uname[-1] == "x86_64" else "32"
|
||||
bits = "64" if uname[4] == "x86_64" else "32"
|
||||
else:
|
||||
bits = ""
|
||||
|
||||
|
|
@ -227,7 +227,7 @@ class Chrome(Browser):
|
|||
raise ValueError("Unable to construct a valid Chrome package name for current platform")
|
||||
|
||||
if platform == "linux":
|
||||
bits = "64" if uname[-1] == "x86_64" else "32"
|
||||
bits = "64" if uname[4] == "x86_64" else "32"
|
||||
elif platform == "mac":
|
||||
bits = "64"
|
||||
elif platform == "win":
|
||||
|
|
|
|||
|
|
@ -146,8 +146,6 @@ An example of an expectation file is::
|
|||
example_default_key: example_value
|
||||
|
||||
[filename.html]
|
||||
type: testharness
|
||||
|
||||
[subtest1]
|
||||
expected: FAIL
|
||||
|
||||
|
|
@ -158,7 +156,6 @@ An example of an expectation file is::
|
|||
FAIL
|
||||
|
||||
[filename.html?query=something]
|
||||
type: testharness
|
||||
disabled: bug12345
|
||||
|
||||
The file consists of two elements, key-value pairs and
|
||||
|
|
@ -230,9 +227,6 @@ The web-platform-test harness knows about several keys:
|
|||
`disabled`
|
||||
Any value indicates that the test is disabled.
|
||||
|
||||
`type`
|
||||
The test type e.g. `testharness`, `reftest`, or `wdspec`.
|
||||
|
||||
`reftype`
|
||||
The type of comparison for reftests; either `==` or `!=`.
|
||||
|
||||
|
|
|
|||
|
|
@ -190,7 +190,8 @@ class FirefoxBrowser(Browser):
|
|||
"network.dns.localDomains": ",".join(hostnames),
|
||||
"network.proxy.type": 0,
|
||||
"places.history.enabled": False,
|
||||
"dom.send_after_paint_to_content": True})
|
||||
"dom.send_after_paint_to_content": True,
|
||||
"network.preload": True})
|
||||
if self.e10s:
|
||||
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
|
||||
|
||||
|
|
|
|||
|
|
@ -150,6 +150,7 @@ class SauceConnect():
|
|||
"--api-key=%s" % self.sauce_key,
|
||||
"--no-remove-colliding-tunnels",
|
||||
"--tunnel-identifier=%s" % self.sauce_tunnel_id,
|
||||
"--metrics-address=0.0.0.0:9876",
|
||||
"--readyfile=./sauce_is_ready",
|
||||
"--tunnel-domains",
|
||||
"web-platform.test",
|
||||
|
|
|
|||
|
|
@ -62,12 +62,12 @@ class MarionetteProtocol(Protocol):
|
|||
self.marionette = marionette.Marionette(host='localhost',
|
||||
port=self.marionette_port,
|
||||
socket_timeout=None,
|
||||
startup_timeout=startup_timeout)
|
||||
startup_timeout=None)
|
||||
|
||||
# XXX Move this timeout somewhere
|
||||
self.logger.debug("Waiting for Marionette connection")
|
||||
while True:
|
||||
success = self.marionette.wait_for_port(60 * self.timeout_multiplier)
|
||||
success = self.marionette.wait_for_port(startup_timeout)
|
||||
#When running in a debugger wait indefinitely for firefox to start
|
||||
if success or self.executor.debug_info is None:
|
||||
break
|
||||
|
|
|
|||
|
|
@ -29,8 +29,11 @@ set of results and conditionals. The AST of the underlying parsed manifest
|
|||
is updated with the changes, and the result is serialised to a file.
|
||||
"""
|
||||
|
||||
|
||||
class ConditionError(Exception):
|
||||
pass
|
||||
def __init__(self, cond=None):
|
||||
self.cond = cond
|
||||
|
||||
|
||||
Result = namedtuple("Result", ["run_info", "status"])
|
||||
|
||||
|
|
@ -102,6 +105,7 @@ class ExpectedManifest(ManifestItem):
|
|||
return urlparse.urljoin(self.url_base,
|
||||
"/".join(self.test_path.split(os.path.sep)))
|
||||
|
||||
|
||||
class TestNode(ManifestItem):
|
||||
def __init__(self, node):
|
||||
"""Tree node associated with a particular test in a manifest
|
||||
|
|
@ -111,12 +115,13 @@ class TestNode(ManifestItem):
|
|||
ManifestItem.__init__(self, node)
|
||||
self.updated_expected = []
|
||||
self.new_expected = []
|
||||
self.new_disabled = False
|
||||
self.subtests = {}
|
||||
self.default_status = None
|
||||
self._from_file = True
|
||||
|
||||
@classmethod
|
||||
def create(cls, test_type, test_id):
|
||||
def create(cls, test_id):
|
||||
"""Create a TestNode corresponding to a given test
|
||||
|
||||
:param test_type: The type of the test
|
||||
|
|
@ -127,14 +132,13 @@ class TestNode(ManifestItem):
|
|||
node = DataNode(name)
|
||||
self = cls(node)
|
||||
|
||||
self.set("type", test_type)
|
||||
self._from_file = False
|
||||
return self
|
||||
|
||||
@property
|
||||
def is_empty(self):
|
||||
required_keys = set(["type"])
|
||||
if set(self._data.keys()) != required_keys:
|
||||
ignore_keys = set(["type"])
|
||||
if set(self._data.keys()) - ignore_keys:
|
||||
return False
|
||||
return all(child.is_empty for child in self.children)
|
||||
|
||||
|
|
@ -182,7 +186,7 @@ class TestNode(ManifestItem):
|
|||
self.new_expected.append(Result(run_info, result.status))
|
||||
self.root.modified = True
|
||||
|
||||
def coalesce_expected(self):
|
||||
def coalesce_expected(self, stability=None):
|
||||
"""Update the underlying manifest AST for this test based on all the
|
||||
added results.
|
||||
|
||||
|
|
@ -191,9 +195,11 @@ class TestNode(ManifestItem):
|
|||
that get more than one different result in the updated run, and add new
|
||||
conditionals for anything that doesn't match an existing conditional.
|
||||
|
||||
Conditionals not matched by any added result are not changed."""
|
||||
Conditionals not matched by any added result are not changed.
|
||||
|
||||
final_conditionals = []
|
||||
When `stability` is not None, disable any test that shows multiple
|
||||
unexpected results for the same set of parameters.
|
||||
"""
|
||||
|
||||
try:
|
||||
unconditional_status = self.get("expected")
|
||||
|
|
@ -203,7 +209,7 @@ class TestNode(ManifestItem):
|
|||
for conditional_value, results in self.updated_expected:
|
||||
if not results:
|
||||
# The conditional didn't match anything in these runs so leave it alone
|
||||
final_conditionals.append(conditional_value)
|
||||
pass
|
||||
elif all(results[0].status == result.status for result in results):
|
||||
# All the new values for this conditional matched, so update the node
|
||||
result = results[0]
|
||||
|
|
@ -213,7 +219,6 @@ class TestNode(ManifestItem):
|
|||
self.remove_value("expected", conditional_value)
|
||||
else:
|
||||
conditional_value.value = result.status
|
||||
final_conditionals.append(conditional_value)
|
||||
elif conditional_value.condition_node is not None:
|
||||
# Blow away the existing condition and rebuild from scratch
|
||||
# This isn't sure to work if we have a conditional later that matches
|
||||
|
|
@ -234,20 +239,22 @@ class TestNode(ManifestItem):
|
|||
status = self.new_expected[0].status
|
||||
if status != self.default_status:
|
||||
self.set("expected", status, condition=None)
|
||||
final_conditionals.append(self._data["expected"][-1])
|
||||
else:
|
||||
try:
|
||||
conditionals = group_conditionals(
|
||||
self.new_expected,
|
||||
property_order=self.root.property_order,
|
||||
boolean_properties=self.root.boolean_properties)
|
||||
except ConditionError:
|
||||
print "Conflicting test results for %s, cannot update" % self.root.test_path
|
||||
except ConditionError as e:
|
||||
if stability is not None:
|
||||
self.set("disabled", stability or "unstable", e.cond.children[0])
|
||||
self.new_disabled = True
|
||||
else:
|
||||
print "Conflicting test results for %s, cannot update" % self.root.test_path
|
||||
return
|
||||
for conditional_node, status in conditionals:
|
||||
if status != unconditional_status:
|
||||
self.set("expected", status, condition=conditional_node.children[0])
|
||||
final_conditionals.append(self._data["expected"][-1])
|
||||
|
||||
if ("expected" in self._data and
|
||||
len(self._data["expected"]) > 0 and
|
||||
|
|
@ -368,6 +375,9 @@ def group_conditionals(values, property_order=None, boolean_properties=None):
|
|||
for run_info, status in values:
|
||||
prop_set = tuple((prop, run_info[prop]) for prop in include_props)
|
||||
if prop_set in conditions:
|
||||
if conditions[prop_set][1] != status:
|
||||
# A prop_set contains contradictory results
|
||||
raise ConditionError(make_expr(prop_set, status, boolean_properties))
|
||||
continue
|
||||
|
||||
expr = make_expr(prop_set, status, boolean_properties=boolean_properties)
|
||||
|
|
|
|||
|
|
@ -29,9 +29,13 @@ def load_test_manifests(serve_root, test_paths):
|
|||
|
||||
def update_expected(test_paths, serve_root, log_file_names,
|
||||
rev_old=None, rev_new="HEAD", ignore_existing=False,
|
||||
sync_root=None, property_order=None, boolean_properties=None):
|
||||
sync_root=None, property_order=None, boolean_properties=None,
|
||||
stability=None):
|
||||
"""Update the metadata files for web-platform-tests based on
|
||||
the results obtained in a previous run"""
|
||||
the results obtained in a previous run or runs
|
||||
|
||||
If stability is not None, assume log_file_names refers to logs from repeated
|
||||
test jobs, disable tests that don't behave as expected on all runs"""
|
||||
|
||||
manifests = load_test_manifests(serve_root, test_paths)
|
||||
|
||||
|
|
@ -45,17 +49,25 @@ def update_expected(test_paths, serve_root, log_file_names,
|
|||
if rev_old is not None:
|
||||
change_data = load_change_data(rev_old, rev_new, repo=sync_root)
|
||||
|
||||
|
||||
expected_map_by_manifest = update_from_logs(manifests,
|
||||
*log_file_names,
|
||||
ignore_existing=ignore_existing,
|
||||
property_order=property_order,
|
||||
boolean_properties=boolean_properties)
|
||||
boolean_properties=boolean_properties,
|
||||
stability=stability)
|
||||
|
||||
for test_manifest, expected_map in expected_map_by_manifest.iteritems():
|
||||
url_base = manifests[test_manifest]["url_base"]
|
||||
metadata_path = test_paths[url_base]["metadata_path"]
|
||||
write_changes(metadata_path, expected_map)
|
||||
if stability is not None:
|
||||
for tree in expected_map.itervalues():
|
||||
for test in tree.iterchildren():
|
||||
for subtest in test.iterchildren():
|
||||
if subtest.new_disabled:
|
||||
print os.path.dirname(subtest.root.test_path) + "/" + subtest.name
|
||||
if test.new_disabled:
|
||||
print test.root.test_path
|
||||
|
||||
results_changed = [item.test_path for item in expected_map.itervalues() if item.modified]
|
||||
|
||||
|
|
@ -121,7 +133,9 @@ def unexpected_changes(manifests, change_data, files_changed):
|
|||
# For each test in the list of tests:
|
||||
# for each conditional:
|
||||
# If all the new values match (or there aren't any) retain that conditional
|
||||
# If any new values mismatch mark the test as needing human attention
|
||||
# If any new values mismatch:
|
||||
# If stability and any repeated values don't match, disable the test
|
||||
# else mark the test as needing human attention
|
||||
# Check if all the RHS values are the same; if so collapse the conditionals
|
||||
|
||||
|
||||
|
|
@ -129,6 +143,7 @@ def update_from_logs(manifests, *log_filenames, **kwargs):
|
|||
ignore_existing = kwargs.get("ignore_existing", False)
|
||||
property_order = kwargs.get("property_order")
|
||||
boolean_properties = kwargs.get("boolean_properties")
|
||||
stability = kwargs.get("stability")
|
||||
|
||||
expected_map = {}
|
||||
id_test_map = {}
|
||||
|
|
@ -152,8 +167,8 @@ def update_from_logs(manifests, *log_filenames, **kwargs):
|
|||
for tree in manifest_expected.itervalues():
|
||||
for test in tree.iterchildren():
|
||||
for subtest in test.iterchildren():
|
||||
subtest.coalesce_expected()
|
||||
test.coalesce_expected()
|
||||
subtest.coalesce_expected(stability=stability)
|
||||
test.coalesce_expected(stability=stability)
|
||||
|
||||
return expected_map
|
||||
|
||||
|
|
@ -231,14 +246,16 @@ class ExpectedUpdater(object):
|
|||
def suite_start(self, data):
|
||||
self.run_info = data["run_info"]
|
||||
|
||||
def test_id(self, id):
|
||||
if type(id) in types.StringTypes:
|
||||
return id
|
||||
else:
|
||||
return tuple(id)
|
||||
def test_type(self, path):
|
||||
for manifest in self.test_manifests.iterkeys():
|
||||
tests = list(manifest.iterpath(path))
|
||||
if len(tests):
|
||||
assert all(test.item_type == tests[0].item_type for test in tests)
|
||||
return tests[0].item_type
|
||||
assert False
|
||||
|
||||
def test_start(self, data):
|
||||
test_id = self.test_id(data["test"])
|
||||
test_id = data["test"]
|
||||
try:
|
||||
test_manifest, test = self.id_path_map[test_id]
|
||||
expected_node = self.expected_tree[test_manifest][test].get_test(test_id)
|
||||
|
|
@ -253,11 +270,10 @@ class ExpectedUpdater(object):
|
|||
self.tests_visited[test_id] = set()
|
||||
|
||||
def test_status(self, data):
|
||||
test_id = self.test_id(data["test"])
|
||||
test = self.test_cache.get(test_id)
|
||||
test = self.test_cache.get(data["test"])
|
||||
if test is None:
|
||||
return
|
||||
test_cls = wpttest.manifest_test_cls[test.test_type]
|
||||
test_cls = wpttest.manifest_test_cls[self.test_type(test.root.test_path)]
|
||||
|
||||
subtest = test.get_subtest(data["subtest"])
|
||||
|
||||
|
|
@ -271,11 +287,11 @@ class ExpectedUpdater(object):
|
|||
subtest.set_result(self.run_info, result)
|
||||
|
||||
def test_end(self, data):
|
||||
test_id = self.test_id(data["test"])
|
||||
test_id = data["test"]
|
||||
test = self.test_cache.get(test_id)
|
||||
if test is None:
|
||||
return
|
||||
test_cls = wpttest.manifest_test_cls[test.test_type]
|
||||
test_cls = wpttest.manifest_test_cls[self.test_type(test.root.test_path)]
|
||||
|
||||
if data["status"] == "SKIP":
|
||||
return
|
||||
|
|
@ -283,7 +299,6 @@ class ExpectedUpdater(object):
|
|||
result = test_cls.result_cls(
|
||||
data["status"],
|
||||
data.get("message"))
|
||||
|
||||
test.set_result(self.run_info, result)
|
||||
del self.test_cache[test_id]
|
||||
|
||||
|
|
@ -322,7 +337,7 @@ def create_expected(test_manifest, test_path, tests, property_order=None,
|
|||
property_order=property_order,
|
||||
boolean_properties=boolean_properties)
|
||||
for test in tests:
|
||||
expected.append(manifestupdate.TestNode.create(test.item_type, test.id))
|
||||
expected.append(manifestupdate.TestNode.create(test.id))
|
||||
return expected
|
||||
|
||||
|
||||
|
|
@ -346,6 +361,6 @@ def load_expected(test_manifest, metadata_path, test_path, tests, property_order
|
|||
# Add tests that don't have expected data
|
||||
for test in tests:
|
||||
if not expected_manifest.has_test(test.id):
|
||||
expected_manifest.append(manifestupdate.TestNode.create(test.item_type, test.id))
|
||||
expected_manifest.append(manifestupdate.TestNode.create(test.id))
|
||||
|
||||
return expected_manifest
|
||||
|
|
|
|||
|
|
@ -33,7 +33,8 @@ class UpdateExpected(Step):
|
|||
ignore_existing=state.ignore_existing,
|
||||
sync_root=sync_root,
|
||||
property_order=state.property_order,
|
||||
boolean_properties=state.boolean_properties)
|
||||
boolean_properties=state.boolean_properties,
|
||||
stability=state.stability)
|
||||
|
||||
|
||||
class CreateMetadataPatch(Step):
|
||||
|
|
|
|||
|
|
@ -90,6 +90,7 @@ class UpdateMetadata(Step):
|
|||
with state.push(["local_tree", "sync_tree", "paths", "serve_root"]):
|
||||
state.run_log = kwargs["run_log"]
|
||||
state.ignore_existing = kwargs["ignore_existing"]
|
||||
state.stability = kwargs["stability"]
|
||||
state.patch = kwargs["patch"]
|
||||
state.suite_name = kwargs["suite_name"]
|
||||
state.product = kwargs["product"]
|
||||
|
|
|
|||
|
|
@ -463,6 +463,9 @@ def create_parser_update(product_choices=None):
|
|||
parser.add_argument("--sync", dest="sync", action="store_true", default=False,
|
||||
help="Sync the tests with the latest from upstream (implies --patch)")
|
||||
parser.add_argument("--ignore-existing", action="store_true", help="When updating test results only consider results from the logfiles provided, not existing expectations.")
|
||||
parser.add_argument("--stability", nargs="?", action="store", const="unstable", default=None,
|
||||
help=("Reason for disabling tests. When updating test results, disable tests that have "
|
||||
"inconsistent results across many runs with the given reason."))
|
||||
parser.add_argument("--continue", action="store_true", help="Continue a previously started run of the update script")
|
||||
parser.add_argument("--abort", action="store_true", help="Clear state from a previous incomplete run of the update script")
|
||||
parser.add_argument("--exclude", action="store", nargs="*",
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ class ManifestSerializer(NodeVisitor):
|
|||
def serialize(self, root):
|
||||
self.indent = 2
|
||||
rv = "\n".join(self.visit(root))
|
||||
if not rv:
|
||||
return rv
|
||||
if rv[-1] != "\n":
|
||||
rv = rv + "\n"
|
||||
return rv
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue