Combine all try workflows (#30096)

There are currently two ways to run try. One is to push to the `try` or
`try-*` branches and the other is to trigger a workflow via GitHub
comment. This change combines these methods into one workflow. In
addition, WPT results are reported together rather than separately and
filtered results for all WPT tests are bundled together in the same
artifact.

.github/workflows/linux-wpt.yml

@@ -65,12 +65,13 @@ jobs:
       - name: Run tests
         if: ${{ inputs.wpt != 'sync' }}
         run: |
+          mkdir linux-${{ inputs.layout }}
           python3 ./mach test-wpt --with-${{ inputs.layout }} \
             --release --processes $(nproc) --timeout-multiplier 2 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-            --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \
-            --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json
+            --log-raw-unexpected linux-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log \
+            --filter-intermittents linux-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
           INTERMITTENT_TRACKER_DASHBOARD_SECRET: ${{ secrets.INTERMITTENT_TRACKER_DASHBOARD_SECRET }}
@@ -86,10 +87,12 @@ jobs:
         uses: actions/upload-artifact@v3
         if: ${{ always() && inputs.wpt != 'sync' }}
         with:
-          name: wpt-filtered-results-linux-${{ inputs.layout }}
+          name: wpt-filtered-results
+          # The wildcard here ensures that the relative path is preserved in the archive:
+          # See https://github.com/actions/upload-artifact/issues/174
           path: |
-            filtered-test-wpt.${{ matrix.chunk_id }}.json
-            unexpected-test-wpt.${{ matrix.chunk_id }}.log
+            ./*linux-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
+            ./*linux-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: ${{ failure() && inputs.wpt != 'sync' }}
@@ -105,34 +108,3 @@ jobs:
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
             wpt-jsonsummary.${{ matrix.chunk_id }}.log
-
-  report-test-results:
-    name: Report WPT Results
-    runs-on: ubuntu-latest
-    if: ${{ always() && !cancelled() && (github.ref_name == 'try-wpt' || github.ref_name == 'try-wpt-2020' || inputs.wpt == 'test') }}
-    needs:
-      - "linux-wpt"
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      - uses: actions/download-artifact@v3
-        with:
-          name: wpt-filtered-results-linux-${{ inputs.layout }}
-          path: wpt-filtered-results-linux
-      - name: Create aggregated unexpected results
-        run: |
-          cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.layout }}.log
-      - name: Archive aggregate results
-        uses: actions/upload-artifact@v3
-        with:
-          name: wpt-filtered-results-linux
-          path: |
-            unexpected-test-wpt-${{ inputs.layout }}.log
-      - name: Comment on PR with results
-        run: |
-          etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.layout }}" wpt-filtered-results-linux/*.json
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          RESULTS: ${{ toJson(needs.*.result) }}

.github/workflows/linux.yml

@@ -5,7 +5,7 @@ on:
       wpt:
         required: false
         type: string
-      layout:
+      wpt-layout:
         required: false
         type: string
       unit-tests:
@@ -26,7 +26,7 @@ on:
         required: false
         type: choice
         options: ["test", "sync"]
-      layout:
+      wpt-layout:
         required: false
         type: choice
         options: ["none", "2013", "2020", "all"]
@@ -38,8 +38,6 @@ on:
         required: false
         default: false
         type: boolean
-  push:
-    branches: ["try-linux", "try-wpt", "try-wpt-2020"]
 
 env:
   RUST_BACKTRACE: 1
@@ -84,7 +82,7 @@ jobs:
       - name: Script tests
         run: ./mach test-scripts
       - name: Unit tests
-        if: ${{ inputs.unit-tests || github.ref_name == 'try-linux' }}
+        if: ${{ inputs.unit-tests }}
         run: python3 ./mach test-unit --release
       - name: Rename build timing
         run: cp -r target/cargo-timings target/cargo-timings-linux
@@ -122,7 +120,7 @@ jobs:
           path: target.tar.gz
 
   wpt-2020:
-    if: ${{ github.ref_name == 'try-wpt-2020' || inputs.layout == '2020' || inputs.layout == 'all' }}
+    if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
     name: Linux WPT Tests 2020
     needs: ["build"]
     uses: ./.github/workflows/linux-wpt.yml
@@ -131,7 +129,7 @@ jobs:
       layout: "layout-2020"
 
   wpt-2013:
-    if: ${{ github.ref_name == 'try-wpt' || inputs.layout == '2013' || inputs.layout == 'all' }}
+    if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
     name: Linux WPT Tests 2013
     needs: ["build"]
     uses: ./.github/workflows/linux-wpt.yml

.github/workflows/mac-wpt.yml

@@ -45,20 +45,23 @@ jobs:
         run: python3 ./mach smoketest --release
       - name: Run tests
         run: |
+          mkdir macos-${{ inputs.layout }}
           python3 ./mach test-wpt --with-${{ inputs.layout }} \
             --release --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-            --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \
-            --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json
+            --log-raw-unexpected macos-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log \
+            --filter-intermittents macos-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
       - name: Archive filtered results
         uses: actions/upload-artifact@v3
         if: always()
         with:
           name: wpt-filtered-results-mac-${{ inputs.layout }}
+          # The wildcard here ensures that the relative path is preserved in the archive:
+          # See https://github.com/actions/upload-artifact/issues/174
           path: |
-            filtered-test-wpt.${{ matrix.chunk_id }}.json
-            unexpected-test-wpt.${{ matrix.chunk_id }}.log
+            ./*macos-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
+            ./*macos-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: failure()
@@ -67,32 +70,3 @@ jobs:
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
             filtered-wpt-results.${{ matrix.chunk_id }}.json
-
-  report-test-results:
-    name: Reporting test results
-    runs-on: ubuntu-latest
-    if: ${{ always() && !cancelled() }}
-    needs: [ mac-wpt ]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      - uses: actions/download-artifact@v3
-        with:
-          name: wpt-filtered-results-mac-${{ inputs.layout }}
-          path: wpt-filtered-results-mac
-      - name: Create aggregated unexpected results
-        run: cat wpt-filtered-results-mac/*.log > unexpected-test-wpt.log
-      - name: Archive aggregate results
-        uses: actions/upload-artifact@v3
-        with:
-          name: wpt-filtered-results-mac-${{ inputs.layout }}
-          path: |
-            unexpected-test-wpt.log
-      - name: Comment on PR with results
-        run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.layout }}"
-          wpt-filtered-results-mac/*.json
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          RESULTS: ${{ toJson(needs.*.result) }}

.github/workflows/mac.yml

@@ -31,8 +31,6 @@ on:
         required: false
         default: false
         type: boolean
-  push:
-    branches: ["try-mac", "try-wpt-mac", "try-wpt-mac-2020"]
 
 env:
   RUST_BACKTRACE: 1
@@ -73,7 +71,7 @@ jobs:
       - name: Script tests
         run: ./mach test-scripts
       - name: Unit tests
-        if: ${{ inputs.unit-tests || github.ref_name == 'try-mac' }}
+        if: ${{ inputs.unit-tests }}
         run: python3 ./mach test-unit --release
       - name: Package
         run: python3 ./mach package --release
@@ -111,7 +109,7 @@ jobs:
           path: target.tar.gz
 
   wpt-2020:
-    if: ${{ github.ref_name == 'try-wpt-mac-2020' || inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
+    if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
     name: Mac WPT Tests 2020
     needs: ["build"]
     uses: ./.github/workflows/mac-wpt.yml
@@ -119,7 +117,7 @@ jobs:
       layout: "layout-2020"
 
   wpt-2013:
-    if: ${{ github.ref_name == 'try-wpt-mac' || inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
+    if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
     name: Mac WPT Tests 2013
     needs: ["build"]
     uses: ./.github/workflows/mac-wpt.yml

.github/workflows/main.yml

@@ -5,7 +5,7 @@ on:
     # Run the entire pipeline for 'master' even though the merge queue already runs checks
     # for every change. This just offers an extra layer of testing and covers the case of
     # random force pushes.
-    branches: ["master", "try"]
+    branches: ["master"]
   pull_request:
     types: ['opened', 'synchronize']
     branches: ["**"]
@@ -16,7 +16,10 @@ on:
       platform:
         required: true
         type: string
-      layout:
+      linux-wpt-layout:
+        required: true
+        type: string
+      mac-wpt-layout:
         required: true
         type: string
       unit-tests:
@@ -28,7 +31,11 @@ on:
         required: false
         type: choice
         options: ["none", "linux", "windows", "macos", "all", "sync"]
-      layout:
+      linux-wpt-layout:
+        required: false
+        type: choice
+        options: ["none", "2013", "2020", "all"]
+      mac-wpt-layout:
         required: false
         type: choice
         options: ["none", "2013", "2020", "all"]
@@ -48,11 +55,17 @@ jobs:
         uses: actions/github-script@v6
         with:
           script: |
-            // Never skip workflow runs for pull requests, merge groups, or manually triggered
-            // workfows / try jobs, which might need to actually run / retry WPT tests.
-            if (!['issue_comment', 'merge_group', 'pull_request', 'workflow_run', 'workflow_call'].includes(context.eventName)) {
-              // Skip the run if an identical run already exists. This helps to avoid running
-              // the workflow over and over again for the same commit hash.
+            // Skip this workflow if an identical run already exists. Note that we don't
+            // always want to skip workflows. In particular we don't skip duplicate
+            // workflows if:
+            //   1. This is a try job (triggered via an `issue_comment` event or a push to
+            //      a non-master branch)
+            //   2. This is a merge queue event (`merge_queue`)
+            //   3. This event was triggered by a pull request update (`pull_request`)
+            //   4. This event was triggered manually `workflow_run` / `workflow_call`
+            let isPushToMaster = context.eventName == "push" && process.env.GITHUB_REF_NAME == "master";
+            let isTryRun = context.eventName == 'issue_comment' || (context.eventName == 'push' && !isPushToMaster);
+            if (!isTryRun && !['merge_group', 'pull_request', 'workflow_run', 'workflow_call'].includes(context.eventName)) {
               if ((await github.rest.actions.listWorkflowRuns({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
@@ -68,20 +81,23 @@ jobs:
             // We need to pick defaults if the inputs are not provided. Unprovided inputs
             // are empty strings in this template.
             let platform = "${{ inputs.platform }}" || "linux";
-            let layout = "${{ inputs.layout }}" || "none";
             let unit_tests = Boolean(${{ inputs.unit-tests }})
+            let linux_wpt_layout = "${{ inputs.linux-wpt-layout }}" || "none";
+            let mac_wpt_layout = "${{ inputs.mac-wpt-layout }}" || "none";
 
             // Merge queue runs and pushes to master should always trigger a full build and test.
-            if (["push", "merge_group"].includes(context.eventName)) {
+            if (isPushToMaster || context.eventName == "merge_group") {
               platform = "all";
-              layout = "all";
               unit_tests = true;
+              linux_wpt_layout = "all";
+              mac_wpt_layout = "none";
             }
 
             let returnValue =  {
               platform,
-              layout,
               unit_tests,
+              linux_wpt_layout,
+              mac_wpt_layout,
             };
             console.log("Using configuration: " + JSON.stringify(returnValue));
             return returnValue;
@@ -100,6 +116,7 @@ jobs:
     if: ${{ contains(fromJson('["macos", "all"]'), fromJson(needs.decision.outputs.configuration).platform) }}
     uses: ./.github/workflows/mac.yml
     with:
+      wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).mac_wpt_layout }}
       unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
 
   build-linux:
@@ -109,9 +126,53 @@ jobs:
     uses: ./.github/workflows/linux.yml
     with:
       wpt: 'test'
-      layout: ${{ fromJson(needs.decision.outputs.configuration).layout }}
+      wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).linux_wpt_layout }}
       unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
 
+  report-test-results:
+    name: Report WPT Results
+    runs-on: ubuntu-latest
+    needs:
+      - "decision"
+      - "build-win"
+      - "build-mac"
+      - "build-linux"
+    if: ${{
+      always() && !cancelled() &&
+      fromJson(needs.decision.outputs.configuration).platform != 'none' &&
+      (
+          fromJson(needs.decision.outputs.configuration).linux_wpt_layout != 'none' ||
+          fromJson(needs.decision.outputs.configuration).mac_wpt_layout != 'none'
+      )
+      }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/download-artifact@v3
+      - name: Create aggregated unexpected results
+        run: |
+          mkdir -p wpt-filtered-results
+          cd wpt-filtered-results
+          for file in *; do \
+            if [ -d "$file" ]; then \
+              cat $file/*.log > "unexpected-test-wpt-$file.log"; \
+            fi \
+          done
+      - name: Archive aggregate results
+        uses: actions/upload-artifact@v3
+        with:
+          name: wpt-filtered-results
+          path: |
+            unexpected-test-wpt-*.log
+      - name: Comment on PR with results
+        run: |
+          etc/ci/report_aggregated_expected_results.py wpt-filtered-results
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RESULTS: ${{ toJson(needs.*.result) }}
+
   build-result:
     name: Result
     runs-on: ubuntu-latest

.github/workflows/try.yml

@@ -3,11 +3,13 @@ name: Try
 on:
   issue_comment:
     types: [created]
+  push:
+    branches: ["try", "try-*"]
 
 jobs:
   parse-comment:
-    name: Process Comment
-    if: ${{ github.event.issue.pull_request }}
+    name: Process Comment or Branch Name
+    if: ${{ github.event_name == 'push' || (github.event_name == 'issue_comment' && github.event.issue.pull_request) }}
     runs-on: ubuntu-latest
     outputs:
       configuration: ${{ steps.configuration.outputs.result }}
@@ -18,41 +20,55 @@ jobs:
           script: |
             function makeComment(body) {
                 console.log(body);
-                github.rest.issues.createComment({
-                  issue_number: context.issue.number,
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  body
-                })
+                if (context.eventName == "issue_comment") {
+                  github.rest.issues.createComment({
+                    issue_number: context.issue.number,
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    body
+                  });
+                }
             }
 
-            let tokens = context.payload.comment.body.split(/\s+/);
-            let tagIndex = tokens.indexOf("@bors-servo");
-            if (tagIndex == -1 || tagIndex + 1 >= tokens.length) {
-              return { try: false };
+            let tryString = "";
+            if (context.eventName == "push") {
+              // Replace the first instance of "-" in the branch name to make it similar
+              // to the try syntax string expected below.
+              tryString = process.env.GITHUB_REF_NAME.replace("-", "=");
+            } else {
+              // This is a pull request comment event.
+              let tokens = context.payload.comment.body.split(/\s+/);
+              let tagIndex = tokens.indexOf("@bors-servo");
+              if (tagIndex == -1 || tagIndex + 1 >= tokens.length) {
+                return { try: false };
+              }
+              tryString = tokens[tagIndex + 1];
             }
 
-            let tryString = tokens[tagIndex + 1];
             console.log("Found try string: '" + tryString + "'");
             let returnValue = { try: false };
             if (tryString == "try") {
-              returnValue = { try: true, platform: 'all', layout: 'all', unit_tests: true, };
-            } else if (tryString == "try=wpt") {
-              returnValue = { try: true, platform: 'linux', layout: '2013', unit_tests: false };
-            } else if (tryString == "try=wpt-2020") {
-              returnValue = { try: true, platform: 'linux', layout: '2020', unit_tests: false };
+              returnValue = { try: true, platform: 'all', linux_wpt_layout: 'all', mac_wpt_layout: 'none', unit_tests: true, };
             } else if (tryString == "try=linux") {
-              returnValue = { try: true, platform: 'linux', layout: 'none', unit_tests: true };
+              returnValue = { try: true, platform: 'linux', linux_wpt_layout: 'none', mac_wpt_layout: 'none', unit_tests: true };
             } else if (tryString == "try=mac") {
-              returnValue = { try: true, platform: 'macos', layout: 'none', unit_tests: true };
+              returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: 'none', unit_tests: true };
             } else if (tryString == "try=windows") {
-              returnValue = { try: true, platform: 'windows', layout: 'none', unit_tests: true };
+              returnValue = { try: true, platform: 'windows', linux_wpt_layout: 'none', unit_tests: true };
+            } else if (tryString == "try=wpt") {
+              returnValue = { try: true, platform: 'linux', linux_wpt_layout: '2013', mac_wpt_layout: 'none', unit_tests: false };
+            } else if (tryString == "try=wpt-2020") {
+              returnValue = { try: true, platform: 'linux', linux_wpt_layout: '2020', mac_wpt_layout: 'none', unit_tests: false };
+            } else if (tryString == "try=mac-wpt") {
+              returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: '2013', unit_tests: false };
+            } else if (tryString == "try=mac-wpt-2020") {
+              returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: '2020', unit_tests: false };
             } else {
               makeComment("🤔 Unknown try string '" + tryString + "'");
               return returnValue;
             }
 
-            if (returnValue.try) {
+            if (returnValue.try && context.eventName == "issue_comment") {
               let username = context.payload.comment.user.login;
               let result = await github.rest.repos.getCollaboratorPermissionLevel({
                 owner: context.repo.owner,
@@ -80,7 +96,8 @@ jobs:
     uses: ./.github/workflows/main.yml
     with:
       platform: ${{ fromJson(needs.parse-comment.outputs.configuration).platform }}
-      layout: ${{ fromJson(needs.parse-comment.outputs.configuration).layout }}
+      linux-wpt-layout: ${{ fromJson(needs.parse-comment.outputs.configuration).linux_wpt_layout }}
+      mac-wpt-layout: ${{ fromJson(needs.parse-comment.outputs.configuration).mac_wpt_layout }}
       unit-tests: ${{ fromJson(needs.parse-comment.outputs.configuration).unit_tests }}
 
   results:
@@ -89,37 +106,26 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ always() && fromJson(needs.parse-comment.outputs.configuration).try}}
     steps:
-      - name: Success
-        if: ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
+      - name: Result Comment
         uses: actions/github-script@v6
         with:
           script: |
+              let success = ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }};
+
               const url = context.serverUrl +
                 "/" + context.repo.owner +
                 "/" + context.repo.repo +
                 "/actions/runs/" + context.runId;
               const formattedURL = "[#" + context.runId + "](" + url + ")";
-              github.rest.issues.createComment({
-                issue_number: context.issue.number,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: "✨ Try run (" + formattedURL + ") " + "succeeded.",
-              });
-      - name: Failure
-        if: ${{ contains(needs.*.result, 'failure') }}
-        uses: actions/github-script@v6
-        with:
-          script: |
-              const url = context.serverUrl +
-                "/" + context.repo.owner +
-                "/" + context.repo.repo +
-                "/actions/runs/" + context.runId;
-              const formattedURL = "[#" + context.runId + "](" + url + ")";
-              github.rest.issues.createComment({
-                issue_number: context.issue.number,
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                body: "⚠️ Try run (" + formattedURL + ") " + "failed.",
-              });
-
 
+              let body = success ?
+                "✨ Try run (" + formattedURL + ") " + "succeeded." :
+                "⚠️ Try run (" + formattedURL + ") " + "failed.";
+              if (context.eventName == "issue_comment") {
+                github.rest.issues.createComment({
+                  issue_number: context.issue.number,
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  body
+                });
+              }

.github/workflows/windows.yml

@@ -24,8 +24,6 @@ on:
         required: false
         default: false
         type: boolean
-  push:
-    branches: ["try-windows"]
 
 env:
   RUST_BACKTRACE: 1
@@ -69,7 +67,7 @@ jobs:
       - name: Smoketest
         run: python mach smoketest --angle --release
       - name: Unit tests
-        if: ${{ inputs.unit-tests || github.ref_name == 'try-windows' }}
+        if: ${{ inputs.unit-tests }}
         run: python mach test-unit --release
       - name: Rename build timing
         run: cp C:\a\servo\servo\target\cargo-timings C:\a\servo\servo\target\cargo-timings-windows -Recurse

etc/ci/report_aggregated_expected_results.py

@@ -12,12 +12,13 @@
 # This allows using types that are defined later in the file.
 from __future__ import annotations
 from datetime import datetime
+import glob
 
 import json
 import os
 import re
 import subprocess
-import argparse
+import sys
 import textwrap
 import xml.etree.ElementTree as ElementTree
 
@@ -73,7 +74,8 @@ class Item:
 
     def to_html(self, level: int = 0) -> ElementTree.Element:
         if level == 0:
-            title = result = ElementTree.Element("span")
+            result = ElementTree.Element("span")
+            title = ElementTree.SubElement(result, "h4")
         elif level == 1:
             result = ElementTree.Element("details")
             title = ElementTree.SubElement(result, "summary")
@@ -176,7 +178,7 @@ def get_pr_number() -> Optional[str]:
     return None
 
 
-def create_check_run(body: str, tag: str = ""):
+def create_check_run(body: str):
     # This process is based on the documentation here:
     # https://docs.github.com/en/rest/checks/runs?apiVersion=2022-11-28#create-a-check-runs
     results = json.loads(os.environ.get("RESULTS", "{}"))
@@ -197,14 +199,14 @@ def create_check_run(body: str, tag: str = ""):
         return None
     repo = github_context["repository"]
     data = {
-        'name': tag,
+        'name': 'wpt-results-report',
         'head_sha': github_context["sha"],
         'status': 'completed',
         'started_at': datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
         'conclusion': conclusion,
         'completed_at': datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
         'output': {
-            'title': f'Aggregated {tag} report',
+            'title': 'WPT Results Report',
             'summary': body,
             'images': [{'alt': 'WPT logo', 'image_url': 'https://avatars.githubusercontent.com/u/37226233'}]
         },
@@ -222,23 +224,34 @@ def create_check_run(body: str, tag: str = ""):
 
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--tag", default="wpt", action="store",
-                        help="A string tag used to distinguish the results.")
-    args, filenames = parser.parse_known_args()
-    results = get_results(filenames, args.tag)
+    if len(sys.argv) < 2:
+        print("Must pass the directory containing filtered WPT results as an argument.")
+        sys.exit(1)
+
+    results = []
+    for dir in os.listdir(sys.argv[1]):
+        if not os.path.isdir(dir):
+            continue
+        new_results = get_results(
+            glob.glob(os.path.join(dir, "*.json")),
+            os.path.basename(dir))
+        if new_results:
+            results.append(new_results)
+
     if not results:
         print("Did not find any unexpected results.")
-        create_check_run("Did not find any unexpected results.", args.tag)
+        create_check_run("Did not find any unexpected results.")
         return
 
-    print(results.to_string())
+    for result in results:
+        print(result.to_string() + "\n")
+
+    html_string = "\n".join(
+        [ElementTree.tostring(result.to_html(), encoding="unicode") for result in results])
+    print(html_string)
+    create_check_run(html_string)
 
     pr_number = get_pr_number()
-    html_string = ElementTree.tostring(
-        results.to_html(), encoding="unicode")
-    create_check_run(html_string, args.tag)
-
     if pr_number:
         process = subprocess.Popen(
             ['gh', 'pr', 'comment', pr_number, '-F', '-'], stdin=subprocess.PIPE)