Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-08-05 21:50:18 +01:00
Code Issues Projects Releases Packages Wiki Activity

Make WPT results output more useful

Browse source 
Before when a subtest failed, the text of the failed assertion was not
printed. This changes makes sure that it is printed in both the console
and the aggregated test output.

Also fix a couple typing errors.
This commit is contained in:
Martin Robinson 2023-04-25 13:13:27 +02:00
parent 7114b31cf7
commit 379b3d03c8
2 changed files with 13 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

19
etc/ci/report_aggregated_expected_results.py
View file

@ -21,7 +21,7 @@ import sys
import textwrap import textwrap
import xml.etree.ElementTree as ElementTree import xml.etree.ElementTree as ElementTree
from typing import Optional from typing import List, Optional
SUBTEST_RESULT_TRUNCATION = 10 SUBTEST_RESULT_TRUNCATION = 10
@ -34,14 +34,14 @@ class Item:
self.children = children self.children = children
@classmethod @classmethod
def from_result(cls, result: dict, title_key: str = "path", title_prefix: str = "", print_stack=True): def from_result(cls, result: dict, title: Optional[str] = None, print_stack=True):
expected = result["expected"] expected = result["expected"]
actual = result["actual"] actual = result["actual"]
title = result[title_key] title = title if title else result["path"]
if expected != actual: if expected != actual:
title = f"{actual} [expected {expected}] {title_prefix}{title}" title = f"{actual} [expected {expected}] {title}"
else: else:
title = f"{actual} {title_prefix}{title}" title = f"{actual} {title}"
issue_url = "http://github.com/servo/servo/issues/" issue_url = "http://github.com/servo/servo/issues/"
if "issues" in result and result["issues"]: if "issues" in result and result["issues"]:
@ -54,7 +54,10 @@ class Item:
subtest_results = result.get("unexpected_subtest_results", []) subtest_results = result.get("unexpected_subtest_results", [])
children = [ children = [
cls.from_result(subtest_result, "subtest", "subtest: ", False) cls.from_result(
subtest_result,
f"subtest: {subtest_result['subtest']} {subtest_result.get('message', '')}",
False)
for subtest_result in subtest_results for subtest_result in subtest_results
] ]
return cls(title, body, children) return cls(title, body, children)
@ -115,13 +118,13 @@ def get_results() -> Optional[Item]:
def is_stable_and_unexpected(result): def is_stable_and_unexpected(result):
return not is_flaky(result) and not result["issues"] return not is_flaky(result) and not result["issues"]
def add_children(children, results, filter_func, text): def add_children(children: List[Item], results: List[dict], filter_func, text):
filtered = [Item.from_result(result) for result in filtered = [Item.from_result(result) for result in
filter(filter_func, results)] filter(filter_func, results)]
if filtered: if filtered:
children.append(Item(f"{text} ({len(filtered)})", "", filtered)) children.append(Item(f"{text} ({len(filtered)})", "", filtered))
children = [] children: List[Item] = []
add_children(children, unexpected, is_flaky, "Flaky unexpected result") add_children(children, unexpected, is_flaky, "Flaky unexpected result")
add_children(children, unexpected, is_stable_and_known, add_children(children, unexpected, is_stable_and_known,
"Stable unexpected results that are known to be intermittent") "Stable unexpected results that are known to be intermittent")

3
python/wpt/grouping_formatter.py
View file

@ -93,7 +93,8 @@ class UnexpectedResult():
# Test names sometimes contain control characters, which we want # Test names sometimes contain control characters, which we want
# to be printed in their raw form, and not their interpreted form. # to be printed in their raw form, and not their interpreted form.
first_line += f" {result.path.encode('unicode-escape').decode('utf-8')}" title = result.subtest if isinstance(result, UnexpectedSubtestResult) else result.path
first_line += f" {title.encode('unicode-escape').decode('utf-8')}"
if isinstance(result, UnexpectedResult) and result.issues: if isinstance(result, UnexpectedResult) and result.issues:
first_line += f" ({', '.join([f'#{bug}' for bug in result.issues])})" first_line += f" ({', '.join([f'#{bug}' for bug in result.issues])})"