diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2aeed661b3d..c464b67154a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -146,7 +146,6 @@ jobs: # --release --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \ # --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ # --log-raw test-wpt.${{ matrix.chunk_id }}.log \ - # --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \ # --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log # - name: Archive logs # uses: actions/upload-artifact@v3 @@ -155,7 +154,6 @@ jobs: # name: wpt${{ matrix.chunk_id }}-logs-macos # path: | # test-wpt.${{ matrix.chunk_id }}.log - # wpt-jsonsummary.${{ matrix.chunk_id }}.log # filtered-wpt-summary.${{ matrix.chunk_id }}.log build-linux: @@ -218,8 +216,14 @@ jobs: --release --processes $(nproc) --timeout-multiplier 2 \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \ - --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \ - --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log + --filter-intermittents=filtered-wpt-results.${{ matrix.chunk_id }}.json + - name: Archive filtered results + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: wpt-filtered-results-linux + path: | + filtered-wpt-results.${{ matrix.chunk_id }}.json - name: Archive logs uses: actions/upload-artifact@v3 if: ${{ failure() }} @@ -227,15 +231,33 @@ jobs: name: wpt-logs-linux path: | test-wpt.${{ matrix.chunk_id }}.log - wpt-jsonsummary.${{ matrix.chunk_id }}.log - filtered-wpt-summary.${{ matrix.chunk_id }}.log + filtered-wpt-results.${{ matrix.chunk_id }}.json + + report_test_results: + name: Reporting test results + runs-on: ubuntu-latest + if: always() + needs: + - "linux-wpt" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/download-artifact@v3 + with: + name: wpt-filtered-results-linux + path: wpt-filtered-results-linux + - name: Comment on PR with results + run: etc/ci/report_aggregated_expected_results.py wpt-filtered-results-linux/* + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} build_result: name: homu build finished runs-on: ubuntu-latest needs: - "build-win" - - "build-linux" - "build-mac" - "linux-wpt" # - "mac-wpt" diff --git a/etc/ci/report_aggregated_expected_results.py b/etc/ci/report_aggregated_expected_results.py new file mode 100755 index 00000000000..bfab949d2f3 --- /dev/null +++ b/etc/ci/report_aggregated_expected_results.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python + +# Copyright 2023 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +# This allows using types that are defined later in the file. +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import textwrap +import xml.etree.ElementTree as ElementTree + +from typing import Optional + + +SUBTEST_RESULT_TRUNCATION = 10 + + +class Item: + def __init__(self, title: str, body: str, children: list[Item]): + self.title = title + self.body = body + self.children = children + + @classmethod + def from_result(cls, result: dict, title_key: str = "path", title_prefix: str = "", print_stack=True): + expected = result["expected"] + actual = result["actual"] + title = result[title_key] + if expected != actual: + title = f"{actual} [expected {expected}] {title_prefix}{title}" + else: + title = f"{actual} {title_prefix}{title}" + stack = result["stack"] if result["stack"] and print_stack else "" + body = f"{result['message']}\n{stack}".strip() + + subtest_results = result.get("unexpected_subtest_results", []) + children = [ + cls.from_result(subtest_result, "subtest", "subtest: ", False) + for subtest_result in subtest_results + ] + return cls(title, body, children) + + def to_string(self, bullet: str = "", indent: str = ""): + output = f"{indent}{bullet}{self.title}\n" + if self.body: + output += textwrap.indent(f"{self.body}\n", " " * len(indent + bullet)) + output += "\n".join([child.to_string("• ", indent + " ") + for child in self.children]) + return output.rstrip() + + def to_html(self, level: int = 0) -> ElementTree.Element: + if level == 0: + title = result = ElementTree.Element("div") + elif level == 1: + result = ElementTree.Element("details") + title = ElementTree.SubElement(result, "summary") + else: + result = ElementTree.Element("li") + title = ElementTree.SubElement(result, "span") + title.text = self.title + + if self.children: + # Some tests have dozens of failing tests, which overwhelm the + # output. Limit the output for subtests in GitHub comment output. + max_children = len(self.children) if level < 2 else SUBTEST_RESULT_TRUNCATION + if len(self.children) > max_children: + children = self.children[:max_children] + children.append(Item( + f"And {len(self.children) - max_children} more unexpected results...", + "", [])) + else: + children = self.children + container = ElementTree.SubElement(result, "div" if not level else "ul") + for child in children: + container.append(child.to_html(level + 1)) + + return result + + +def get_results() -> Optional[Item]: + unexpected = [] + known_intermittents = [] + for filename in sys.argv[1:]: + with open(filename, encoding="utf-8") as file: + data = json.load(file) + unexpected += data["unexpected"] + known_intermittents += data["known_intermittents"] + + children = [] + if unexpected: + children.append( + Item(f"Tests producing unexpected results ({len(unexpected)})", "", + [Item.from_result(result) for result in unexpected]), + ) + if known_intermittents: + children.append( + Item("Unexpected results that are known to be intermittent " + f"({len(known_intermittents)})", "", + [Item.from_result(result) for result in known_intermittents]) + ) + return Item("Results from try job:", "", children) if children else None + + +def get_pr_number() -> Optional[str]: + github_context = json.loads(os.environ.get("GITHUB_CONTEXT", "{}")) + if "event" not in github_context: + return None + if "head_commit" not in github_context["event"]: + return None + commit_title = github_context["event"]["head_commit"]["message"] + match = re.match(r"^Auto merge of #(\d+)", commit_title) + return match.group(1) if match else None + + +def main(): + results = get_results() + if not results: + print("Did not find any unexpected results.") + return + + print(results.to_string()) + + pr_number = get_pr_number() + if pr_number: + html_string = ElementTree.tostring(results.to_html(), encoding="unicode") + process = subprocess.Popen(['gh', 'pr', 'comment', pr_number, '-F', '-'], stdin=subprocess.PIPE) + process.communicate(input=html_string.encode("utf-8"))[0] + else: + print("Could not find PR number in environment. Not making GitHub comment.") + + +if __name__ == "__main__": + main() diff --git a/tests/wpt/servowpt.py b/tests/wpt/servowpt.py index 40e238e0406..e66e359877f 100644 --- a/tests/wpt/servowpt.py +++ b/tests/wpt/servowpt.py @@ -2,6 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import dataclasses import grouping_formatter import json import os @@ -218,30 +219,34 @@ def filter_intermittents( else: filter = TrackerFilter() - intermittents = [] - actually_unexpected = [] + known_intermittents: List[UnexpectedResult] = [] + unexpected: List[UnexpectedResult] = [] for i, result in enumerate(unexpected_results): print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r") if filter.is_failure_intermittent(result.path): - intermittents.append(result) + known_intermittents.append(result) else: - actually_unexpected.append(result) - - output = "\n".join([ - f"{len(intermittents)} known-intermittent unexpected result", - *[str(result) for result in intermittents], - "", - f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents", - *[str(result) for result in actually_unexpected], - ]) + unexpected.append(result) if output_file: with open(output_file, "w", encoding="utf-8") as file: - file.write(output) + file.write(json.dumps({ + "known_intermittents": + [dataclasses.asdict(result) for result in known_intermittents], + "unexpected": + [dataclasses.asdict(result) for result in unexpected], + })) + output = "\n".join([ + f"{len(known_intermittents)} known-intermittent unexpected result", + *[str(result) for result in known_intermittents], + "", + f"{len(unexpected)} unexpected results that are NOT known-intermittents", + *[str(result) for result in unexpected], + ]) print(output) print(80 * "=") - return not actually_unexpected + return not unexpected def main():