Make WPT results output more useful

Before when a subtest failed, the text of the failed assertion was not
printed. This changes makes sure that it is printed in both the console
and the aggregated test output.

Also fix a couple typing errors.
This commit is contained in:
Martin Robinson 2023-04-25 13:13:27 +02:00
parent 7114b31cf7
commit 379b3d03c8
2 changed files with 13 additions and 9 deletions

View file

@ -21,7 +21,7 @@ import sys
import textwrap
import xml.etree.ElementTree as ElementTree
from typing import Optional
from typing import List, Optional
SUBTEST_RESULT_TRUNCATION = 10
@ -34,14 +34,14 @@ class Item:
self.children = children
@classmethod
def from_result(cls, result: dict, title_key: str = "path", title_prefix: str = "", print_stack=True):
def from_result(cls, result: dict, title: Optional[str] = None, print_stack=True):
expected = result["expected"]
actual = result["actual"]
title = result[title_key]
title = title if title else result["path"]
if expected != actual:
title = f"{actual} [expected {expected}] {title_prefix}{title}"
title = f"{actual} [expected {expected}] {title}"
else:
title = f"{actual} {title_prefix}{title}"
title = f"{actual} {title}"
issue_url = "http://github.com/servo/servo/issues/"
if "issues" in result and result["issues"]:
@ -54,7 +54,10 @@ class Item:
subtest_results = result.get("unexpected_subtest_results", [])
children = [
cls.from_result(subtest_result, "subtest", "subtest: ", False)
cls.from_result(
subtest_result,
f"subtest: {subtest_result['subtest']} {subtest_result.get('message', '')}",
False)
for subtest_result in subtest_results
]
return cls(title, body, children)
@ -115,13 +118,13 @@ def get_results() -> Optional[Item]:
def is_stable_and_unexpected(result):
return not is_flaky(result) and not result["issues"]
def add_children(children, results, filter_func, text):
def add_children(children: List[Item], results: List[dict], filter_func, text):
filtered = [Item.from_result(result) for result in
filter(filter_func, results)]
if filtered:
children.append(Item(f"{text} ({len(filtered)})", "", filtered))
children = []
children: List[Item] = []
add_children(children, unexpected, is_flaky, "Flaky unexpected result")
add_children(children, unexpected, is_stable_and_known,
"Stable unexpected results that are known to be intermittent")

View file

@ -93,7 +93,8 @@ class UnexpectedResult():
# Test names sometimes contain control characters, which we want
# to be printed in their raw form, and not their interpreted form.
first_line += f" {result.path.encode('unicode-escape').decode('utf-8')}"
title = result.subtest if isinstance(result, UnexpectedSubtestResult) else result.path
first_line += f" {title.encode('unicode-escape').decode('utf-8')}"
if isinstance(result, UnexpectedResult) and result.issues:
first_line += f" ({', '.join([f'#{bug}' for bug in result.issues])})"