Output test results as a GitHub comment

After filtering intermittents, output the results as JSON. Update the
GitHub workflow to aggregate this JSON data into an artifact and use the
aggregated data to generate a GitHub comment with details about the try
run. The idea here is that this comment will make it easier to track
intermittent tests and notice when a change affects a test marked as
intermittent -- either causing it to permanently fail or fixing it.

.github/workflows/main.yml

@@ -146,7 +146,6 @@ jobs:
   #           --release --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \
   #           --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
   #           --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-  #           --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \
   #           --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log
   #     - name: Archive logs
   #       uses: actions/upload-artifact@v3
@@ -155,7 +154,6 @@ jobs:
   #         name: wpt${{ matrix.chunk_id }}-logs-macos
   #         path: |
   #           test-wpt.${{ matrix.chunk_id }}.log
-  #           wpt-jsonsummary.${{ matrix.chunk_id }}.log
   #           filtered-wpt-summary.${{ matrix.chunk_id }}.log
 
   build-linux:
@@ -218,8 +216,14 @@ jobs:
             --release --processes $(nproc) --timeout-multiplier 2 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-            --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \
+            --filter-intermittents=filtered-wpt-results.${{ matrix.chunk_id }}.json
-            --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log
+      - name: Archive filtered results
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: wpt-filtered-results-linux
+          path: |
+            filtered-wpt-results.${{ matrix.chunk_id }}.json
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: ${{ failure() }}
@@ -227,15 +231,33 @@ jobs:
           name: wpt-logs-linux
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
-            wpt-jsonsummary.${{ matrix.chunk_id }}.log
+            filtered-wpt-results.${{ matrix.chunk_id }}.json
-            filtered-wpt-summary.${{ matrix.chunk_id }}.log
+
+  report_test_results:
+    name: Reporting test results
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - "linux-wpt"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/download-artifact@v3
+        with:
+          name: wpt-filtered-results-linux
+          path: wpt-filtered-results-linux
+      - name: Comment on PR with results
+        run: etc/ci/report_aggregated_expected_results.py wpt-filtered-results-linux/*
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   build_result:
     name: homu build finished
     runs-on: ubuntu-latest
     needs:
       - "build-win"
-      - "build-linux"
       - "build-mac"
       - "linux-wpt"
       # - "mac-wpt"

etc/ci/report_aggregated_expected_results.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+
+# Copyright 2023 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This allows using types that are defined later in the file.
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+import textwrap
+import xml.etree.ElementTree as ElementTree
+
+from typing import Optional
+
+
+SUBTEST_RESULT_TRUNCATION = 10
+
+
+class Item:
+    def __init__(self, title: str, body: str, children: list[Item]):
+        self.title = title
+        self.body = body
+        self.children = children
+
+    @classmethod
+    def from_result(cls, result: dict, title_key: str = "path", title_prefix: str = "", print_stack=True):
+        expected = result["expected"]
+        actual = result["actual"]
+        title = result[title_key]
+        if expected != actual:
+            title = f"{actual} [expected {expected}] {title_prefix}{title}"
+        else:
+            title = f"{actual} {title_prefix}{title}"
+        stack = result["stack"] if result["stack"] and print_stack else ""
+        body = f"{result['message']}\n{stack}".strip()
+
+        subtest_results = result.get("unexpected_subtest_results", [])
+        children = [
+            cls.from_result(subtest_result, "subtest", "subtest: ", False)
+            for subtest_result in subtest_results
+        ]
+        return cls(title, body, children)
+
+    def to_string(self, bullet: str = "", indent: str = ""):
+        output = f"{indent}{bullet}{self.title}\n"
+        if self.body:
+            output += textwrap.indent(f"{self.body}\n", " " * len(indent + bullet))
+        output += "\n".join([child.to_string("• ", indent + "  ")
+                             for child in self.children])
+        return output.rstrip()
+
+    def to_html(self, level: int = 0) -> ElementTree.Element:
+        if level == 0:
+            title = result = ElementTree.Element("div")
+        elif level == 1:
+            result = ElementTree.Element("details")
+            title = ElementTree.SubElement(result, "summary")
+        else:
+            result = ElementTree.Element("li")
+            title = ElementTree.SubElement(result, "span")
+        title.text = self.title
+
+        if self.children:
+            # Some tests have dozens of failing tests, which overwhelm the
+            # output. Limit the output for subtests in GitHub comment output.
+            max_children = len(self.children) if level < 2 else SUBTEST_RESULT_TRUNCATION
+            if len(self.children) > max_children:
+                children = self.children[:max_children]
+                children.append(Item(
+                    f"And {len(self.children) - max_children} more unexpected results...",
+                    "", []))
+            else:
+                children = self.children
+            container = ElementTree.SubElement(result, "div" if not level else "ul")
+            for child in children:
+                container.append(child.to_html(level + 1))
+
+        return result
+
+
+def get_results() -> Optional[Item]:
+    unexpected = []
+    known_intermittents = []
+    for filename in sys.argv[1:]:
+        with open(filename, encoding="utf-8") as file:
+            data = json.load(file)
+            unexpected += data["unexpected"]
+            known_intermittents += data["known_intermittents"]
+
+    children = []
+    if unexpected:
+        children.append(
+            Item(f"Tests producing unexpected results ({len(unexpected)})", "",
+                 [Item.from_result(result) for result in unexpected]),
+        )
+    if known_intermittents:
+        children.append(
+            Item("Unexpected results that are known to be intermittent "
+                 f"({len(known_intermittents)})", "",
+                 [Item.from_result(result) for result in known_intermittents])
+        )
+    return Item("Results from try job:", "", children) if children else None
+
+
+def get_pr_number() -> Optional[str]:
+    github_context = json.loads(os.environ.get("GITHUB_CONTEXT", "{}"))
+    if "event" not in github_context:
+        return None
+    if "head_commit" not in github_context["event"]:
+        return None
+    commit_title = github_context["event"]["head_commit"]["message"]
+    match = re.match(r"^Auto merge of #(\d+)", commit_title)
+    return match.group(1) if match else None
+
+
+def main():
+    results = get_results()
+    if not results:
+        print("Did not find any unexpected results.")
+        return
+
+    print(results.to_string())
+
+    pr_number = get_pr_number()
+    if pr_number:
+        html_string = ElementTree.tostring(results.to_html(), encoding="unicode")
+        process = subprocess.Popen(['gh', 'pr', 'comment', pr_number, '-F', '-'], stdin=subprocess.PIPE)
+        process.communicate(input=html_string.encode("utf-8"))[0]
+    else:
+        print("Could not find PR number in environment. Not making GitHub comment.")
+
+
+if __name__ == "__main__":
+    main()

tests/wpt/servowpt.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+import dataclasses
 import grouping_formatter
 import json
 import os
@@ -218,30 +219,34 @@ def filter_intermittents(
     else:
         filter = TrackerFilter()
 
-    intermittents = []
+    known_intermittents: List[UnexpectedResult] = []
-    actually_unexpected = []
+    unexpected: List[UnexpectedResult] = []
     for i, result in enumerate(unexpected_results):
         print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
         if filter.is_failure_intermittent(result.path):
-            intermittents.append(result)
+            known_intermittents.append(result)
         else:
-            actually_unexpected.append(result)
+            unexpected.append(result)
-
-    output = "\n".join([
-        f"{len(intermittents)} known-intermittent unexpected result",
-        *[str(result) for result in intermittents],
-        "",
-        f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents",
-        *[str(result) for result in actually_unexpected],
-    ])
 
     if output_file:
         with open(output_file, "w", encoding="utf-8") as file:
-            file.write(output)
+            file.write(json.dumps({
+                "known_intermittents":
+                    [dataclasses.asdict(result) for result in known_intermittents],
+                "unexpected":
+                    [dataclasses.asdict(result) for result in unexpected],
+            }))
 
+    output = "\n".join([
+        f"{len(known_intermittents)} known-intermittent unexpected result",
+        *[str(result) for result in known_intermittents],
+        "",
+        f"{len(unexpected)} unexpected results that are NOT known-intermittents",
+        *[str(result) for result in unexpected],
+    ])
     print(output)
     print(80 * "=")
-    return not actually_unexpected
+    return not unexpected
 
 
 def main():