Revert "Combine all try workflows (#30096)" (#30149)

This reverts commit fed3491f23.

.github/workflows/linux-wpt.yml

@@ -65,13 +65,12 @@ jobs:
       - name: Run tests
         if: ${{ inputs.wpt != 'sync' }}
         run: |
-          mkdir linux-${{ inputs.layout }}
           python3 ./mach test-wpt --with-${{ inputs.layout }} \
             --release --processes $(nproc) --timeout-multiplier 2 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-            --log-raw-unexpected linux-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log \
+            --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \
-            --filter-intermittents linux-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
+            --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
           INTERMITTENT_TRACKER_DASHBOARD_SECRET: ${{ secrets.INTERMITTENT_TRACKER_DASHBOARD_SECRET }}
@@ -87,12 +86,10 @@ jobs:
         uses: actions/upload-artifact@v3
         if: ${{ always() && inputs.wpt != 'sync' }}
         with:
-          name: wpt-filtered-results
+          name: wpt-filtered-results-linux-${{ inputs.layout }}
-          # The wildcard here ensures that the relative path is preserved in the archive:
-          # See https://github.com/actions/upload-artifact/issues/174
           path: |
-            ./*linux-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
+            filtered-test-wpt.${{ matrix.chunk_id }}.json
-            ./*linux-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log
+            unexpected-test-wpt.${{ matrix.chunk_id }}.log
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: ${{ failure() && inputs.wpt != 'sync' }}
@@ -108,3 +105,34 @@ jobs:
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
             wpt-jsonsummary.${{ matrix.chunk_id }}.log
+
+  report-test-results:
+    name: Report WPT Results
+    runs-on: ubuntu-latest
+    if: ${{ always() && !cancelled() && (github.ref_name == 'try-wpt' || github.ref_name == 'try-wpt-2020' || inputs.wpt == 'test') }}
+    needs:
+      - "linux-wpt"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/download-artifact@v3
+        with:
+          name: wpt-filtered-results-linux-${{ inputs.layout }}
+          path: wpt-filtered-results-linux
+      - name: Create aggregated unexpected results
+        run: |
+          cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.layout }}.log
+      - name: Archive aggregate results
+        uses: actions/upload-artifact@v3
+        with:
+          name: wpt-filtered-results-linux
+          path: |
+            unexpected-test-wpt-${{ inputs.layout }}.log
+      - name: Comment on PR with results
+        run: |
+          etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.layout }}" wpt-filtered-results-linux/*.json
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RESULTS: ${{ toJson(needs.*.result) }}

.github/workflows/linux.yml

@@ -5,7 +5,7 @@ on:
       wpt:
         required: false
         type: string
-      wpt-layout:
+      layout:
         required: false
         type: string
       unit-tests:
@@ -26,7 +26,7 @@ on:
         required: false
         type: choice
         options: ["test", "sync"]
-      wpt-layout:
+      layout:
         required: false
         type: choice
         options: ["none", "2013", "2020", "all"]
@@ -38,6 +38,8 @@ on:
         required: false
         default: false
         type: boolean
+  push:
+    branches: ["try-linux", "try-wpt", "try-wpt-2020"]
 
 env:
   RUST_BACKTRACE: 1
@@ -82,7 +84,7 @@ jobs:
       - name: Script tests
         run: ./mach test-scripts
       - name: Unit tests
-        if: ${{ inputs.unit-tests }}
+        if: ${{ inputs.unit-tests || github.ref_name == 'try-linux' }}
         run: python3 ./mach test-unit --release
       - name: Rename build timing
         run: cp -r target/cargo-timings target/cargo-timings-linux
@@ -120,7 +122,7 @@ jobs:
           path: target.tar.gz
 
   wpt-2020:
-    if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
+    if: ${{ github.ref_name == 'try-wpt-2020' || inputs.layout == '2020' || inputs.layout == 'all' }}
     name: Linux WPT Tests 2020
     needs: ["build"]
     uses: ./.github/workflows/linux-wpt.yml
@@ -129,7 +131,7 @@ jobs:
       layout: "layout-2020"
 
   wpt-2013:
-    if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
+    if: ${{ github.ref_name == 'try-wpt' || inputs.layout == '2013' || inputs.layout == 'all' }}
     name: Linux WPT Tests 2013
     needs: ["build"]
     uses: ./.github/workflows/linux-wpt.yml

.github/workflows/mac-wpt.yml

@@ -45,23 +45,20 @@ jobs:
         run: python3 ./mach smoketest --release
       - name: Run tests
         run: |
-          mkdir macos-${{ inputs.layout }}
           python3 ./mach test-wpt --with-${{ inputs.layout }} \
             --release --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-            --log-raw-unexpected macos-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log \
+            --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \
-            --filter-intermittents macos-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
+            --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json
       - name: Archive filtered results
         uses: actions/upload-artifact@v3
         if: always()
         with:
           name: wpt-filtered-results-mac-${{ inputs.layout }}
-          # The wildcard here ensures that the relative path is preserved in the archive:
-          # See https://github.com/actions/upload-artifact/issues/174
           path: |
-            ./*macos-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json
+            filtered-test-wpt.${{ matrix.chunk_id }}.json
-            ./*macos-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log
+            unexpected-test-wpt.${{ matrix.chunk_id }}.log
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: failure()
@@ -70,3 +67,32 @@ jobs:
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
             filtered-wpt-results.${{ matrix.chunk_id }}.json
+
+  report-test-results:
+    name: Reporting test results
+    runs-on: ubuntu-latest
+    if: ${{ always() && !cancelled() }}
+    needs: [ mac-wpt ]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/download-artifact@v3
+        with:
+          name: wpt-filtered-results-mac-${{ inputs.layout }}
+          path: wpt-filtered-results-mac
+      - name: Create aggregated unexpected results
+        run: cat wpt-filtered-results-mac/*.log > unexpected-test-wpt.log
+      - name: Archive aggregate results
+        uses: actions/upload-artifact@v3
+        with:
+          name: wpt-filtered-results-mac-${{ inputs.layout }}
+          path: |
+            unexpected-test-wpt.log
+      - name: Comment on PR with results
+        run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.layout }}"
+          wpt-filtered-results-mac/*.json
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RESULTS: ${{ toJson(needs.*.result) }}

.github/workflows/mac.yml

@@ -31,6 +31,8 @@ on:
         required: false
         default: false
         type: boolean
+  push:
+    branches: ["try-mac", "try-wpt-mac", "try-wpt-mac-2020"]
 
 env:
   RUST_BACKTRACE: 1
@@ -71,7 +73,7 @@ jobs:
       - name: Script tests
         run: ./mach test-scripts
       - name: Unit tests
-        if: ${{ inputs.unit-tests }}
+        if: ${{ inputs.unit-tests || github.ref_name == 'try-mac' }}
         run: python3 ./mach test-unit --release
       - name: Package
         run: python3 ./mach package --release
@@ -109,7 +111,7 @@ jobs:
           path: target.tar.gz
 
   wpt-2020:
-    if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
+    if: ${{ github.ref_name == 'try-wpt-mac-2020' || inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
     name: Mac WPT Tests 2020
     needs: ["build"]
     uses: ./.github/workflows/mac-wpt.yml
@@ -117,7 +119,7 @@ jobs:
       layout: "layout-2020"
 
   wpt-2013:
-    if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
+    if: ${{ github.ref_name == 'try-wpt-mac' || inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
     name: Mac WPT Tests 2013
     needs: ["build"]
     uses: ./.github/workflows/mac-wpt.yml

.github/workflows/main.yml

@@ -5,7 +5,7 @@ on:
     # Run the entire pipeline for 'master' even though the merge queue already runs checks
     # for every change. This just offers an extra layer of testing and covers the case of
     # random force pushes.
-    branches: ["master"]
+    branches: ["master", "try"]
   pull_request:
     types: ['opened', 'synchronize']
     branches: ["**"]
@@ -16,10 +16,7 @@ on:
       platform:
         required: true
         type: string
-      linux-wpt-layout:
+      layout:
-        required: true
-        type: string
-      mac-wpt-layout:
         required: true
         type: string
       unit-tests:
@@ -31,11 +28,7 @@ on:
         required: false
         type: choice
         options: ["none", "linux", "windows", "macos", "all", "sync"]
-      linux-wpt-layout:
+      layout:
-        required: false
-        type: choice
-        options: ["none", "2013", "2020", "all"]
-      mac-wpt-layout:
         required: false
         type: choice
         options: ["none", "2013", "2020", "all"]
@@ -55,17 +48,11 @@ jobs:
         uses: actions/github-script@v6
         with:
           script: |
-            // Skip this workflow if an identical run already exists. Note that we don't
+            // Never skip workflow runs for pull requests, merge groups, or manually triggered
-            // always want to skip workflows. In particular we don't skip duplicate
+            // workfows / try jobs, which might need to actually run / retry WPT tests.
-            // workflows if:
+            if (!['issue_comment', 'merge_group', 'pull_request', 'workflow_run', 'workflow_call'].includes(context.eventName)) {
-            //   1. This is a try job (triggered via an `issue_comment` event or a push to
+              // Skip the run if an identical run already exists. This helps to avoid running
-            //      a non-master branch)
+              // the workflow over and over again for the same commit hash.
-            //   2. This is a merge queue event (`merge_queue`)
-            //   3. This event was triggered by a pull request update (`pull_request`)
-            //   4. This event was triggered manually `workflow_run` / `workflow_call`
-            let isPushToMaster = context.eventName == "push" && process.env.GITHUB_REF_NAME == "master";
-            let isTryRun = context.eventName == 'issue_comment' || (context.eventName == 'push' && !isPushToMaster);
-            if (!isTryRun && !['merge_group', 'pull_request', 'workflow_run', 'workflow_call'].includes(context.eventName)) {
               if ((await github.rest.actions.listWorkflowRuns({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
@@ -81,23 +68,20 @@ jobs:
             // We need to pick defaults if the inputs are not provided. Unprovided inputs
             // are empty strings in this template.
             let platform = "${{ inputs.platform }}" || "linux";
+            let layout = "${{ inputs.layout }}" || "none";
             let unit_tests = Boolean(${{ inputs.unit-tests }})
-            let linux_wpt_layout = "${{ inputs.linux-wpt-layout }}" || "none";
-            let mac_wpt_layout = "${{ inputs.mac-wpt-layout }}" || "none";
 
             // Merge queue runs and pushes to master should always trigger a full build and test.
-            if (isPushToMaster || context.eventName == "merge_group") {
+            if (["push", "merge_group"].includes(context.eventName)) {
               platform = "all";
+              layout = "all";
               unit_tests = true;
-              linux_wpt_layout = "all";
-              mac_wpt_layout = "none";
             }
 
             let returnValue =  {
               platform,
+              layout,
               unit_tests,
-              linux_wpt_layout,
-              mac_wpt_layout,
             };
             console.log("Using configuration: " + JSON.stringify(returnValue));
             return returnValue;
@@ -116,7 +100,6 @@ jobs:
     if: ${{ contains(fromJson('["macos", "all"]'), fromJson(needs.decision.outputs.configuration).platform) }}
     uses: ./.github/workflows/mac.yml
     with:
-      wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).mac_wpt_layout }}
       unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
 
   build-linux:
@@ -126,53 +109,9 @@ jobs:
     uses: ./.github/workflows/linux.yml
     with:
       wpt: 'test'
-      wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).linux_wpt_layout }}
+      layout: ${{ fromJson(needs.decision.outputs.configuration).layout }}
       unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
 
-  report-test-results:
-    name: Report WPT Results
-    runs-on: ubuntu-latest
-    needs:
-      - "decision"
-      - "build-win"
-      - "build-mac"
-      - "build-linux"
-    if: ${{
-      always() && !cancelled() &&
-      fromJson(needs.decision.outputs.configuration).platform != 'none' &&
-      (
-          fromJson(needs.decision.outputs.configuration).linux_wpt_layout != 'none' ||
-          fromJson(needs.decision.outputs.configuration).mac_wpt_layout != 'none'
-      )
-      }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 2
-      - uses: actions/download-artifact@v3
-      - name: Create aggregated unexpected results
-        run: |
-          mkdir -p wpt-filtered-results
-          cd wpt-filtered-results
-          for file in *; do \
-            if [ -d "$file" ]; then \
-              cat $file/*.log > "unexpected-test-wpt-$file.log"; \
-            fi \
-          done
-      - name: Archive aggregate results
-        uses: actions/upload-artifact@v3
-        with:
-          name: wpt-filtered-results
-          path: |
-            unexpected-test-wpt-*.log
-      - name: Comment on PR with results
-        run: |
-          etc/ci/report_aggregated_expected_results.py wpt-filtered-results
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          RESULTS: ${{ toJson(needs.*.result) }}
-
   build-result:
     name: Result
     runs-on: ubuntu-latest

.github/workflows/try.yml

@@ -3,13 +3,11 @@ name: Try
 on:
   issue_comment:
     types: [created]
-  push:
-    branches: ["try", "try-*"]
 
 jobs:
   parse-comment:
-    name: Process Comment or Branch Name
+    name: Process Comment
-    if: ${{ github.event_name == 'push' || (github.event_name == 'issue_comment' && github.event.issue.pull_request) }}
+    if: ${{ github.event.issue.pull_request }}
     runs-on: ubuntu-latest
     outputs:
       configuration: ${{ steps.configuration.outputs.result }}
@@ -20,55 +18,41 @@ jobs:
           script: |
             function makeComment(body) {
                 console.log(body);
-                if (context.eventName == "issue_comment") {
                 github.rest.issues.createComment({
                   issue_number: context.issue.number,
                   owner: context.repo.owner,
                   repo: context.repo.repo,
                   body
-                  });
+                })
-                }
             }
 
-            let tryString = "";
-            if (context.eventName == "push") {
-              // Replace the first instance of "-" in the branch name to make it similar
-              // to the try syntax string expected below.
-              tryString = process.env.GITHUB_REF_NAME.replace("-", "=");
-            } else {
-              // This is a pull request comment event.
             let tokens = context.payload.comment.body.split(/\s+/);
             let tagIndex = tokens.indexOf("@bors-servo");
             if (tagIndex == -1 || tagIndex + 1 >= tokens.length) {
               return { try: false };
             }
-              tryString = tokens[tagIndex + 1];
-            }
 
+            let tryString = tokens[tagIndex + 1];
             console.log("Found try string: '" + tryString + "'");
             let returnValue = { try: false };
             if (tryString == "try") {
-              returnValue = { try: true, platform: 'all', linux_wpt_layout: 'all', mac_wpt_layout: 'none', unit_tests: true, };
+              returnValue = { try: true, platform: 'all', layout: 'all', unit_tests: true, };
-            } else if (tryString == "try=linux") {
-              returnValue = { try: true, platform: 'linux', linux_wpt_layout: 'none', mac_wpt_layout: 'none', unit_tests: true };
-            } else if (tryString == "try=mac") {
-              returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: 'none', unit_tests: true };
-            } else if (tryString == "try=windows") {
-              returnValue = { try: true, platform: 'windows', linux_wpt_layout: 'none', unit_tests: true };
             } else if (tryString == "try=wpt") {
-              returnValue = { try: true, platform: 'linux', linux_wpt_layout: '2013', mac_wpt_layout: 'none', unit_tests: false };
+              returnValue = { try: true, platform: 'linux', layout: '2013', unit_tests: false };
             } else if (tryString == "try=wpt-2020") {
-              returnValue = { try: true, platform: 'linux', linux_wpt_layout: '2020', mac_wpt_layout: 'none', unit_tests: false };
+              returnValue = { try: true, platform: 'linux', layout: '2020', unit_tests: false };
-            } else if (tryString == "try=mac-wpt") {
+            } else if (tryString == "try=linux") {
-              returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: '2013', unit_tests: false };
+              returnValue = { try: true, platform: 'linux', layout: 'none', unit_tests: true };
-            } else if (tryString == "try=mac-wpt-2020") {
+            } else if (tryString == "try=mac") {
-              returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: '2020', unit_tests: false };
+              returnValue = { try: true, platform: 'macos', layout: 'none', unit_tests: true };
+            } else if (tryString == "try=windows") {
+              returnValue = { try: true, platform: 'windows', layout: 'none', unit_tests: true };
             } else {
               makeComment("🤔 Unknown try string '" + tryString + "'");
               return returnValue;
             }
 
-            if (returnValue.try && context.eventName == "issue_comment") {
+            if (returnValue.try) {
               let username = context.payload.comment.user.login;
               let result = await github.rest.repos.getCollaboratorPermissionLevel({
                 owner: context.repo.owner,
@@ -96,8 +80,7 @@ jobs:
     uses: ./.github/workflows/main.yml
     with:
       platform: ${{ fromJson(needs.parse-comment.outputs.configuration).platform }}
-      linux-wpt-layout: ${{ fromJson(needs.parse-comment.outputs.configuration).linux_wpt_layout }}
+      layout: ${{ fromJson(needs.parse-comment.outputs.configuration).layout }}
-      mac-wpt-layout: ${{ fromJson(needs.parse-comment.outputs.configuration).mac_wpt_layout }}
       unit-tests: ${{ fromJson(needs.parse-comment.outputs.configuration).unit_tests }}
 
   results:
@@ -106,26 +89,37 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ always() && fromJson(needs.parse-comment.outputs.configuration).try}}
     steps:
-      - name: Result Comment
+      - name: Success
+        if: ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
         uses: actions/github-script@v6
         with:
           script: |
-              let success = ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }};
-
               const url = context.serverUrl +
                 "/" + context.repo.owner +
                 "/" + context.repo.repo +
                 "/actions/runs/" + context.runId;
               const formattedURL = "[#" + context.runId + "](" + url + ")";
-
-              let body = success ?
-                "✨ Try run (" + formattedURL + ") " + "succeeded." :
-                "⚠️ Try run (" + formattedURL + ") " + "failed.";
-              if (context.eventName == "issue_comment") {
               github.rest.issues.createComment({
                 issue_number: context.issue.number,
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                  body
+                body: "✨ Try run (" + formattedURL + ") " + "succeeded.",
               });
-              }
+      - name: Failure
+        if: ${{ contains(needs.*.result, 'failure') }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+              const url = context.serverUrl +
+                "/" + context.repo.owner +
+                "/" + context.repo.repo +
+                "/actions/runs/" + context.runId;
+              const formattedURL = "[#" + context.runId + "](" + url + ")";
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: "⚠️ Try run (" + formattedURL + ") " + "failed.",
+              });
+
+

.github/workflows/windows.yml

@@ -24,6 +24,8 @@ on:
         required: false
         default: false
         type: boolean
+  push:
+    branches: ["try-windows"]
 
 env:
   RUST_BACKTRACE: 1
@@ -69,7 +71,7 @@ jobs:
       - name: Smoketest
         run: python mach smoketest --angle --release
       - name: Unit tests
-        if: ${{ inputs.unit-tests }}
+        if: ${{ inputs.unit-tests || github.ref_name == 'try-windows' }}
         run: python mach test-unit --release
       - name: Rename build timing
         run: cp C:\a\servo\servo\target\cargo-timings C:\a\servo\servo\target\cargo-timings-windows -Recurse

etc/ci/report_aggregated_expected_results.py

@@ -12,13 +12,12 @@
 # This allows using types that are defined later in the file.
 from __future__ import annotations
 from datetime import datetime
-import glob
 
 import json
 import os
 import re
 import subprocess
-import sys
+import argparse
 import textwrap
 import xml.etree.ElementTree as ElementTree
 
@@ -74,8 +73,7 @@ class Item:
 
     def to_html(self, level: int = 0) -> ElementTree.Element:
         if level == 0:
-            result = ElementTree.Element("span")
+            title = result = ElementTree.Element("span")
-            title = ElementTree.SubElement(result, "h4")
         elif level == 1:
             result = ElementTree.Element("details")
             title = ElementTree.SubElement(result, "summary")
@@ -178,7 +176,7 @@ def get_pr_number() -> Optional[str]:
     return None
 
 
-def create_check_run(body: str):
+def create_check_run(body: str, tag: str = ""):
     # This process is based on the documentation here:
     # https://docs.github.com/en/rest/checks/runs?apiVersion=2022-11-28#create-a-check-runs
     results = json.loads(os.environ.get("RESULTS", "{}"))
@@ -199,14 +197,14 @@ def create_check_run(body: str):
         return None
     repo = github_context["repository"]
     data = {
-        'name': 'wpt-results-report',
+        'name': tag,
         'head_sha': github_context["sha"],
         'status': 'completed',
         'started_at': datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
         'conclusion': conclusion,
         'completed_at': datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
         'output': {
-            'title': 'WPT Results Report',
+            'title': f'Aggregated {tag} report',
             'summary': body,
             'images': [{'alt': 'WPT logo', 'image_url': 'https://avatars.githubusercontent.com/u/37226233'}]
         },
@@ -224,34 +222,23 @@ def create_check_run(body: str):
 
 
 def main():
-    if len(sys.argv) < 2:
+    parser = argparse.ArgumentParser()
-        print("Must pass the directory containing filtered WPT results as an argument.")
+    parser.add_argument("--tag", default="wpt", action="store",
-        sys.exit(1)
+                        help="A string tag used to distinguish the results.")
-
+    args, filenames = parser.parse_known_args()
-    results = []
+    results = get_results(filenames, args.tag)
-    for dir in os.listdir(sys.argv[1]):
-        if not os.path.isdir(dir):
-            continue
-        new_results = get_results(
-            glob.glob(os.path.join(dir, "*.json")),
-            os.path.basename(dir))
-        if new_results:
-            results.append(new_results)
-
     if not results:
         print("Did not find any unexpected results.")
-        create_check_run("Did not find any unexpected results.")
+        create_check_run("Did not find any unexpected results.", args.tag)
         return
 
-    for result in results:
+    print(results.to_string())
-        print(result.to_string() + "\n")
-
-    html_string = "\n".join(
-        [ElementTree.tostring(result.to_html(), encoding="unicode") for result in results])
-    print(html_string)
-    create_check_run(html_string)
 
     pr_number = get_pr_number()
+    html_string = ElementTree.tostring(
+        results.to_html(), encoding="unicode")
+    create_check_run(html_string, args.tag)
+
     if pr_number:
         process = subprocess.Popen(
             ['gh', 'pr', 'comment', pr_number, '-F', '-'], stdin=subprocess.PIPE)