mirror of
https://github.com/servo/servo.git
synced 2025-08-03 12:40:06 +01:00
Update CSS tests to revision 31d63cc79bd4c929ed582229e936d7b389f3e6ab
This commit is contained in:
James Graham
parent
1a81b18b9f
commit
2c9faf5363
91915 changed files with 5979820 additions and 0 deletions
310
tests/wpt/css-tests/tools/runner/report.py
Normal file
310
tests/wpt/css-tests/tools/runner/report.py
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from cgi import escape
|
||||
from collections import defaultdict
|
||||
|
||||
import types
|
||||
|
||||
|
||||
def html_escape(item, escape_quote=False):
|
||||
if isinstance(item, types.StringTypes):
|
||||
rv = escape(item)
|
||||
if escape_quote:
|
||||
rv = rv.replace('"', """)
|
||||
return rv
|
||||
else:
|
||||
return item
|
||||
|
||||
|
||||
class Raw(object):
|
||||
"""Simple wrapper around a string to stop it being escaped by html_escape"""
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __unicode__(self):
|
||||
return unicode(self.value)
|
||||
|
||||
|
||||
class Node(object):
|
||||
"""Node structure used when building HTML"""
|
||||
def __init__(self, name, attrs, children):
|
||||
#Need list of void elements
|
||||
self.name = name
|
||||
self.attrs = attrs
|
||||
self.children = children
|
||||
|
||||
def __unicode__(self):
|
||||
if self.attrs:
|
||||
#Need to escape
|
||||
attrs_unicode = " " + " ".join("%s=\"%s\"" % (html_escape(key),
|
||||
html_escape(value,
|
||||
escape_quote=True))
|
||||
for key, value in self.attrs.iteritems())
|
||||
else:
|
||||
attrs_unicode = ""
|
||||
return "<%s%s>%s</%s>\n" % (self.name,
|
||||
attrs_unicode,
|
||||
"".join(unicode(html_escape(item))
|
||||
for item in self.children),
|
||||
self.name)
|
||||
|
||||
def __str__(self):
|
||||
return unicode(self).encode("utf8")
|
||||
|
||||
|
||||
class RootNode(object):
|
||||
"""Special Node representing the document root"""
|
||||
def __init__(self, *children):
|
||||
self.children = ["<!DOCTYPE html>"] + list(children)
|
||||
|
||||
def __unicode__(self):
|
||||
return "".join(unicode(item) for item in self.children)
|
||||
|
||||
def __str__(self):
|
||||
return unicode(self).encode("utf8")
|
||||
|
||||
|
||||
def flatten(iterable):
|
||||
"""Flatten a list of lists by one level so that
|
||||
[1,["abc"], "def",[2, [3]]]
|
||||
becomes
|
||||
[1, "abc", "def", 2, [3]]"""
|
||||
rv = []
|
||||
for item in iterable:
|
||||
if hasattr(item, "__iter__") and not isinstance(item, types.StringTypes):
|
||||
rv.extend(item)
|
||||
else:
|
||||
rv.append(item)
|
||||
return rv
|
||||
|
||||
|
||||
class HTML(object):
|
||||
"""Simple HTML templating system. An instance of this class can create
|
||||
element nodes by calling methods with the same name as the element,
|
||||
passing in children as positional arguments or as a list, and attributes
|
||||
as keyword arguments, with _ replacing - and trailing _ for python keywords
|
||||
|
||||
e.g.
|
||||
|
||||
h = HTML()
|
||||
print h.html(
|
||||
html.head(),
|
||||
html.body([html.h1("Hello World!")], class_="body-class")
|
||||
)
|
||||
Would give
|
||||
<!DOCTYPE html><html><head></head><body class="body-class"><h1>Hello World!</h1></body></html>"""
|
||||
def __getattr__(self, name):
|
||||
def make_html(self, *content, **attrs):
|
||||
for attr_name in attrs.keys():
|
||||
if "_" in attr_name:
|
||||
new_name = attr_name.replace("_", "-")
|
||||
if new_name.endswith("-"):
|
||||
new_name = new_name[:-1]
|
||||
attrs[new_name] = attrs.pop(attr_name)
|
||||
return Node(name, attrs, flatten(content))
|
||||
|
||||
method = types.MethodType(make_html, self, HTML)
|
||||
setattr(self, name, method)
|
||||
return method
|
||||
|
||||
def __call__(self, *children):
|
||||
return RootNode(*flatten(children))
|
||||
|
||||
|
||||
h = HTML()
|
||||
|
||||
|
||||
class TestResult(object):
|
||||
"""Simple holder for the results of a single test in a single UA"""
|
||||
def __init__(self, test):
|
||||
self.test = test
|
||||
self.results = {}
|
||||
|
||||
def __cmp__(self, other):
|
||||
return self.test == other.test
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.test)
|
||||
|
||||
|
||||
def load_data(args):
|
||||
"""Load data treating args as a list of UA name, filename pairs"""
|
||||
pairs = []
|
||||
for i in xrange(0, len(args), 2):
|
||||
pairs.append(args[i:i+2])
|
||||
|
||||
rv = {}
|
||||
for UA, filename in pairs:
|
||||
with open(filename) as f:
|
||||
rv[UA] = json.load(f)
|
||||
|
||||
return rv
|
||||
|
||||
|
||||
def test_id(id):
|
||||
"""Convert a test id in JSON into an immutable object that
|
||||
can be used as a dictionary key"""
|
||||
if isinstance(id, list):
|
||||
return tuple(id)
|
||||
else:
|
||||
return id
|
||||
|
||||
|
||||
def all_tests(data):
|
||||
tests = defaultdict(set)
|
||||
for UA, results in data.iteritems():
|
||||
for result in results["results"]:
|
||||
id = test_id(result["test"])
|
||||
tests[id] |= set(subtest["name"] for subtest in result["subtests"])
|
||||
return tests
|
||||
|
||||
|
||||
def group_results(data):
|
||||
"""Produce a list of UAs and a dictionary mapping specific tests to their
|
||||
status in all UAs e.g.
|
||||
["UA1", "UA2"], {"test_id":{"harness":{"UA1": (status1, message1),
|
||||
"UA2": (status2, message2)},
|
||||
"subtests":{"subtest1": "UA1": (status1-1, message1-1),
|
||||
"UA2": (status2-1, message2-1)}}}
|
||||
Status and message are None if the test didn't run in a particular UA.
|
||||
Message is None if the test didn't produce a message"""
|
||||
tests = all_tests(data)
|
||||
|
||||
UAs = data.keys()
|
||||
|
||||
def result():
|
||||
return {
|
||||
"harness": dict((UA, (None, None)) for UA in UAs),
|
||||
"subtests": None # init this later
|
||||
}
|
||||
|
||||
results_by_test = defaultdict(result)
|
||||
|
||||
for UA, results in data.iteritems():
|
||||
for test_data in results["results"]:
|
||||
id = test_id(test_data["test"])
|
||||
result = results_by_test[id]
|
||||
|
||||
if result["subtests"] is None:
|
||||
result["subtests"] = dict(
|
||||
(name, dict((UA, (None, None)) for UA in UAs)) for name in tests[id]
|
||||
)
|
||||
|
||||
result["harness"][UA] = (test_data["status"], test_data["message"])
|
||||
for subtest in test_data["subtests"]:
|
||||
result["subtests"][subtest["name"]][UA] = (subtest["status"],
|
||||
subtest["message"])
|
||||
|
||||
return UAs, results_by_test
|
||||
|
||||
|
||||
def status_cell(status, message=None):
|
||||
"""Produce a table cell showing the status of a test"""
|
||||
status = status if status is not None else "NONE"
|
||||
kwargs = {}
|
||||
if message:
|
||||
kwargs["title"] = message
|
||||
status_text = status.title()
|
||||
return h.td(status_text, class_="status " + status,
|
||||
**kwargs)
|
||||
|
||||
|
||||
def test_link(test_id, subtest=None):
|
||||
"""Produce an <a> element linking to a test"""
|
||||
if isinstance(test_id, types.StringTypes):
|
||||
rv = [h.a(test_id, href=test_id)]
|
||||
else:
|
||||
rv = [h.a(test_id[0], href=test_id[0]),
|
||||
" %s " % test_id[1],
|
||||
h.a(test_id[2], href=test_id[2])]
|
||||
if subtest is not None:
|
||||
rv.append(" [%s]" % subtest)
|
||||
return rv
|
||||
|
||||
|
||||
def summary(UAs, results_by_test):
|
||||
"""Render the implementation report summary"""
|
||||
not_passing = []
|
||||
for test, results in results_by_test.iteritems():
|
||||
if not any(item[0] in ("PASS", "OK") for item in results["harness"].values()):
|
||||
not_passing.append((test, None))
|
||||
for subtest_name, subtest_results in results["subtests"].iteritems():
|
||||
if not any(item[0] == "PASS" for item in subtest_results.values()):
|
||||
not_passing.append((test, subtest_name))
|
||||
if not_passing:
|
||||
rv = [
|
||||
h.p("The following tests failed to pass in all UAs:"),
|
||||
h.ul([h.li(test_link(test, subtest))
|
||||
for test, subtest in not_passing])
|
||||
]
|
||||
else:
|
||||
rv = "All tests passed in at least one UA"
|
||||
return rv
|
||||
|
||||
|
||||
def result_rows(UAs, test, result):
|
||||
"""Render the results for each test run"""
|
||||
yield h.tr(
|
||||
h.td(
|
||||
test_link(test),
|
||||
rowspan=(1 + len(result["subtests"]))
|
||||
),
|
||||
h.td(),
|
||||
[status_cell(status, message)
|
||||
for UA, (status, message) in sorted(result["harness"].items())],
|
||||
class_="test"
|
||||
)
|
||||
|
||||
for name, subtest_result in sorted(result["subtests"].iteritems()):
|
||||
yield h.tr(
|
||||
h.td(name),
|
||||
[status_cell(status, message)
|
||||
for UA, (status, message) in sorted(subtest_result.items())],
|
||||
class_="subtest"
|
||||
)
|
||||
|
||||
|
||||
def result_bodies(UAs, results_by_test):
|
||||
return [h.tbody(result_rows(UAs, test, result))
|
||||
for test, result in sorted(results_by_test.iteritems())]
|
||||
|
||||
|
||||
def generate_html(UAs, results_by_test):
|
||||
"""Generate all the HTML output"""
|
||||
doc = h(h.html([
|
||||
h.head(h.meta(charset="utf8"),
|
||||
h.title("Implementation Report"),
|
||||
h.link(href="report.css", rel="stylesheet")),
|
||||
h.body(h.h1("Implementation Report"),
|
||||
h.h2("Summary"),
|
||||
summary(UAs, results_by_test),
|
||||
h.h2("Full Results"),
|
||||
h.table(
|
||||
h.thead(
|
||||
h.tr(
|
||||
h.th("Test"),
|
||||
h.th("Subtest"),
|
||||
[h.th(UA) for UA in sorted(UAs)]
|
||||
)
|
||||
),
|
||||
result_bodies(UAs, results_by_test)
|
||||
)
|
||||
)
|
||||
]))
|
||||
|
||||
return doc
|
||||
|
||||
|
||||
def main(filenames):
|
||||
data = load_data(filenames)
|
||||
UAs, results_by_test = group_results(data)
|
||||
return generate_html(UAs, results_by_test)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if not sys.argv[1:]:
|
||||
print """Please supply a list of UA name, filename pairs e.g.
|
||||
|
||||
python report.py Firefox firefox.json Chrome chrome.json IE internet_explorer.json"""
|
||||
print main(sys.argv[1:])
|
||||
Loading…
Add table
Add a link
Reference in a new issue