Auto merge of #29315 - mrobinson:isolate-intermittents-2, r=delan

Output test results as GitHub comments

After filtering intermittents, output the results as JSON. Update the
GitHub workflow to aggregate this JSON data into an artifact and use the
aggregated data to generate a GitHub comment with details about the try
run. The idea here is that this comment will make it easier to track
intermittent tests and notice when a change affects a test marked as
intermittent -- either causing it to permanently fail or fixing it.

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes do not require tests because they modify the CI infrastructure.

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

.github/workflows/main.yml

@@ -146,7 +146,6 @@ jobs:
   #           --release --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \
   #           --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
   #           --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-  #           --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \
   #           --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log
   #     - name: Archive logs
   #       uses: actions/upload-artifact@v3
@@ -155,7 +154,6 @@ jobs:
   #         name: wpt${{ matrix.chunk_id }}-logs-macos
   #         path: |
   #           test-wpt.${{ matrix.chunk_id }}.log
-  #           wpt-jsonsummary.${{ matrix.chunk_id }}.log
   #           filtered-wpt-summary.${{ matrix.chunk_id }}.log
 
   build-linux:
@@ -218,8 +216,14 @@ jobs:
             --release --processes $(nproc) --timeout-multiplier 2 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-            --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \
-            --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log
+            --filter-intermittents=filtered-wpt-results.${{ matrix.chunk_id }}.json
+      - name: Archive filtered results
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: wpt-filtered-results-linux
+          path: |
+            filtered-wpt-results.${{ matrix.chunk_id }}.json
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: ${{ failure() }}
@@ -227,15 +231,33 @@ jobs:
           name: wpt-logs-linux
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
-            wpt-jsonsummary.${{ matrix.chunk_id }}.log
-            filtered-wpt-summary.${{ matrix.chunk_id }}.log
+            filtered-wpt-results.${{ matrix.chunk_id }}.json
+
+  report_test_results:
+    name: Reporting test results
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - "linux-wpt"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/download-artifact@v3
+        with:
+          name: wpt-filtered-results-linux
+          path: wpt-filtered-results-linux
+      - name: Comment on PR with results
+        run: etc/ci/report_aggregated_expected_results.py wpt-filtered-results-linux/*
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   build_result:
     name: homu build finished
     runs-on: ubuntu-latest
     needs:
       - "build-win"
-      - "build-linux"
       - "build-mac"
       - "linux-wpt"
       # - "mac-wpt"

etc/ci/report_aggregated_expected_results.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+
+# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This allows using types that are defined later in the file.
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+import textwrap
+import xml.etree.ElementTree as ElementTree
+
+from typing import Optional
+
+
+SUBTEST_RESULT_TRUNCATION = 10
+
+
+class Item:
+    def __init__(self, title: str, body: str, children: list[Item]):
+        self.title = title
+        self.body = body
+        self.children = children
+
+    @classmethod
+    def from_result(cls, result: dict, title_key: str = "path", title_prefix: str = "", print_stack=True):
+        expected = result["expected"]
+        actual = result["actual"]
+        title = result[title_key]
+        if expected != actual:
+            title = f"{actual} [expected {expected}] {title_prefix}{title}"
+        else:
+            title = f"{actual} {title_prefix}{title}"
+        stack = result["stack"] if result["stack"] and print_stack else ""
+        body = f"{result['message']}\n{stack}".strip()
+
+        subtest_results = result.get("unexpected_subtest_results", [])
+        children = [
+            cls.from_result(subtest_result, "subtest", "subtest: ", False)
+            for subtest_result in subtest_results
+        ]
+        return cls(title, body, children)
+
+    def to_string(self, bullet: str = "", indent: str = ""):
+        output = f"{indent}{bullet}{self.title}\n"
+        if self.body:
+            output += textwrap.indent(f"{self.body}\n", " " * len(indent + bullet))
+        output += "\n".join([child.to_string("• ", indent + "  ")
+                             for child in self.children])
+        return output.rstrip()
+
+    def to_html(self, level: int = 0) -> ElementTree.Element:
+        if level == 0:
+            title = result = ElementTree.Element("div")
+        elif level == 1:
+            result = ElementTree.Element("details")
+            title = ElementTree.SubElement(result, "summary")
+        else:
+            result = ElementTree.Element("li")
+            title = ElementTree.SubElement(result, "span")
+        title.text = self.title
+
+        if self.children:
+            # Some tests have dozens of failing tests, which overwhelm the
+            # output. Limit the output for subtests in GitHub comment output.
+            max_children = len(self.children) if level < 2 else SUBTEST_RESULT_TRUNCATION
+            if len(self.children) > max_children:
+                children = self.children[:max_children]
+                children.append(Item(
+                    f"And {len(self.children) - max_children} more unexpected results...",
+                    "", []))
+            else:
+                children = self.children
+            container = ElementTree.SubElement(result, "div" if not level else "ul")
+            for child in children:
+                container.append(child.to_html(level + 1))
+
+        return result
+
+
+def get_results() -> Optional[Item]:
+    unexpected = []
+    known_intermittents = []
+    for filename in sys.argv[1:]:
+        with open(filename, encoding="utf-8") as file:
+            data = json.load(file)
+            unexpected += data["unexpected"]
+            known_intermittents += data["known_intermittents"]
+
+    children = []
+    if unexpected:
+        children.append(
+            Item(f"Tests producing unexpected results ({len(unexpected)})", "",
+                 [Item.from_result(result) for result in unexpected]),
+        )
+    if known_intermittents:
+        children.append(
+            Item("Unexpected results that are known to be intermittent "
+                 f"({len(known_intermittents)})", "",
+                 [Item.from_result(result) for result in known_intermittents])
+        )
+    return Item("Results from try job:", "", children) if children else None
+
+
+def get_pr_number() -> Optional[str]:
+    github_context = json.loads(os.environ.get("GITHUB_CONTEXT", "{}"))
+    if "event" not in github_context:
+        return None
+    if "head_commit" not in github_context["event"]:
+        return None
+    commit_title = github_context["event"]["head_commit"]["message"]
+    match = re.match(r"^Auto merge of #(\d+)", commit_title)
+    return match.group(1) if match else None
+
+
+def main():
+    results = get_results()
+    if not results:
+        print("Did not find any unexpected results.")
+        return
+
+    print(results.to_string())
+
+    pr_number = get_pr_number()
+    if pr_number:
+        html_string = ElementTree.tostring(results.to_html(), encoding="unicode")
+        process = subprocess.Popen(['gh', 'pr', 'comment', pr_number, '-F', '-'], stdin=subprocess.PIPE)
+        process.communicate(input=html_string.encode("utf-8"))[0]
+    else:
+        print("Could not find PR number in environment. Not making GitHub comment.")
+
+
+if __name__ == "__main__":
+    main()

tests/wpt/servowpt.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+import dataclasses
 import grouping_formatter
 import json
 import os
@@ -218,30 +219,34 @@ def filter_intermittents(
     else:
         filter = TrackerFilter()
 
-    intermittents = []
-    actually_unexpected = []
+    known_intermittents: List[UnexpectedResult] = []
+    unexpected: List[UnexpectedResult] = []
     for i, result in enumerate(unexpected_results):
         print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
         if filter.is_failure_intermittent(result.path):
-            intermittents.append(result)
+            known_intermittents.append(result)
         else:
-            actually_unexpected.append(result)
-
-    output = "\n".join([
-        f"{len(intermittents)} known-intermittent unexpected result",
-        *[str(result) for result in intermittents],
-        "",
-        f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents",
-        *[str(result) for result in actually_unexpected],
-    ])
+            unexpected.append(result)
 
     if output_file:
         with open(output_file, "w", encoding="utf-8") as file:
-            file.write(output)
+            file.write(json.dumps({
+                "known_intermittents":
+                    [dataclasses.asdict(result) for result in known_intermittents],
+                "unexpected":
+                    [dataclasses.asdict(result) for result in unexpected],
+            }))
 
+    output = "\n".join([
+        f"{len(known_intermittents)} known-intermittent unexpected result",
+        *[str(result) for result in known_intermittents],
+        "",
+        f"{len(unexpected)} unexpected results that are NOT known-intermittents",
+        *[str(result) for result in unexpected],
+    ])
     print(output)
     print(80 * "=")
-    return not actually_unexpected
+    return not unexpected
 
 
 def main():