From d3203976dc12c06978d4957a889b0ff23f2a8fe2 Mon Sep 17 00:00:00 2001 From: Martin Robinson Date: Mon, 21 Aug 2023 19:00:14 +0200 Subject: [PATCH] Revert "Combine all try workflows (#30096)" (#30149) This reverts commit fed3491f23f8b7f3064ce297776deda76e485289. --- .github/workflows/linux-wpt.yml | 46 ++++++-- .github/workflows/linux.yml | 12 ++- .github/workflows/mac-wpt.yml | 42 ++++++-- .github/workflows/mac.yml | 8 +- .github/workflows/main.yml | 87 +++------------- .github/workflows/try.yml | 104 +++++++++---------- .github/workflows/windows.yml | 4 +- etc/ci/report_aggregated_expected_results.py | 45 +++----- 8 files changed, 164 insertions(+), 184 deletions(-) diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index ae8247d1579..39110d4161d 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -65,13 +65,12 @@ jobs: - name: Run tests if: ${{ inputs.wpt != 'sync' }} run: | - mkdir linux-${{ inputs.layout }} python3 ./mach test-wpt --with-${{ inputs.layout }} \ --release --processes $(nproc) --timeout-multiplier 2 \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \ - --log-raw-unexpected linux-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log \ - --filter-intermittents linux-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json + --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \ + --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json env: GITHUB_CONTEXT: ${{ toJson(github) }} INTERMITTENT_TRACKER_DASHBOARD_SECRET: ${{ secrets.INTERMITTENT_TRACKER_DASHBOARD_SECRET }} @@ -87,12 +86,10 @@ jobs: uses: actions/upload-artifact@v3 if: ${{ always() && inputs.wpt != 'sync' }} with: - name: wpt-filtered-results - # The wildcard here ensures that the relative path is preserved in the archive: - # See https://github.com/actions/upload-artifact/issues/174 + name: wpt-filtered-results-linux-${{ inputs.layout }} path: | - ./*linux-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json - ./*linux-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log + filtered-test-wpt.${{ matrix.chunk_id }}.json + unexpected-test-wpt.${{ matrix.chunk_id }}.log - name: Archive logs uses: actions/upload-artifact@v3 if: ${{ failure() && inputs.wpt != 'sync' }} @@ -107,4 +104,35 @@ jobs: name: wpt-logs-linux-${{ inputs.layout }} path: | test-wpt.${{ matrix.chunk_id }}.log - wpt-jsonsummary.${{ matrix.chunk_id }}.log \ No newline at end of file + wpt-jsonsummary.${{ matrix.chunk_id }}.log + + report-test-results: + name: Report WPT Results + runs-on: ubuntu-latest + if: ${{ always() && !cancelled() && (github.ref_name == 'try-wpt' || github.ref_name == 'try-wpt-2020' || inputs.wpt == 'test') }} + needs: + - "linux-wpt" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/download-artifact@v3 + with: + name: wpt-filtered-results-linux-${{ inputs.layout }} + path: wpt-filtered-results-linux + - name: Create aggregated unexpected results + run: | + cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.layout }}.log + - name: Archive aggregate results + uses: actions/upload-artifact@v3 + with: + name: wpt-filtered-results-linux + path: | + unexpected-test-wpt-${{ inputs.layout }}.log + - name: Comment on PR with results + run: | + etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.layout }}" wpt-filtered-results-linux/*.json + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RESULTS: ${{ toJson(needs.*.result) }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b818a9505ef..3ef755554d6 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -5,7 +5,7 @@ on: wpt: required: false type: string - wpt-layout: + layout: required: false type: string unit-tests: @@ -26,7 +26,7 @@ on: required: false type: choice options: ["test", "sync"] - wpt-layout: + layout: required: false type: choice options: ["none", "2013", "2020", "all"] @@ -38,6 +38,8 @@ on: required: false default: false type: boolean + push: + branches: ["try-linux", "try-wpt", "try-wpt-2020"] env: RUST_BACKTRACE: 1 @@ -82,7 +84,7 @@ jobs: - name: Script tests run: ./mach test-scripts - name: Unit tests - if: ${{ inputs.unit-tests }} + if: ${{ inputs.unit-tests || github.ref_name == 'try-linux' }} run: python3 ./mach test-unit --release - name: Rename build timing run: cp -r target/cargo-timings target/cargo-timings-linux @@ -120,7 +122,7 @@ jobs: path: target.tar.gz wpt-2020: - if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }} + if: ${{ github.ref_name == 'try-wpt-2020' || inputs.layout == '2020' || inputs.layout == 'all' }} name: Linux WPT Tests 2020 needs: ["build"] uses: ./.github/workflows/linux-wpt.yml @@ -129,7 +131,7 @@ jobs: layout: "layout-2020" wpt-2013: - if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }} + if: ${{ github.ref_name == 'try-wpt' || inputs.layout == '2013' || inputs.layout == 'all' }} name: Linux WPT Tests 2013 needs: ["build"] uses: ./.github/workflows/linux-wpt.yml diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml index 7d067ecda29..da608fa9533 100644 --- a/.github/workflows/mac-wpt.yml +++ b/.github/workflows/mac-wpt.yml @@ -45,23 +45,20 @@ jobs: run: python3 ./mach smoketest --release - name: Run tests run: | - mkdir macos-${{ inputs.layout }} python3 ./mach test-wpt --with-${{ inputs.layout }} \ --release --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \ - --log-raw-unexpected macos-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log \ - --filter-intermittents macos-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json + --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \ + --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json - name: Archive filtered results uses: actions/upload-artifact@v3 if: always() with: name: wpt-filtered-results-mac-${{ inputs.layout }} - # The wildcard here ensures that the relative path is preserved in the archive: - # See https://github.com/actions/upload-artifact/issues/174 path: | - ./*macos-${{ inputs.layout }}/filtered-test-wpt.${{ matrix.chunk_id }}.json - ./*macos-${{ inputs.layout }}/unexpected-test-wpt.${{ matrix.chunk_id }}.log + filtered-test-wpt.${{ matrix.chunk_id }}.json + unexpected-test-wpt.${{ matrix.chunk_id }}.log - name: Archive logs uses: actions/upload-artifact@v3 if: failure() @@ -69,4 +66,33 @@ jobs: name: wpt-logs-mac-${{ inputs.layout }} path: | test-wpt.${{ matrix.chunk_id }}.log - filtered-wpt-results.${{ matrix.chunk_id }}.json \ No newline at end of file + filtered-wpt-results.${{ matrix.chunk_id }}.json + + report-test-results: + name: Reporting test results + runs-on: ubuntu-latest + if: ${{ always() && !cancelled() }} + needs: [ mac-wpt ] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/download-artifact@v3 + with: + name: wpt-filtered-results-mac-${{ inputs.layout }} + path: wpt-filtered-results-mac + - name: Create aggregated unexpected results + run: cat wpt-filtered-results-mac/*.log > unexpected-test-wpt.log + - name: Archive aggregate results + uses: actions/upload-artifact@v3 + with: + name: wpt-filtered-results-mac-${{ inputs.layout }} + path: | + unexpected-test-wpt.log + - name: Comment on PR with results + run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.layout }}" + wpt-filtered-results-mac/*.json + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RESULTS: ${{ toJson(needs.*.result) }} diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 844eadaa2d0..db7b9518886 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -31,6 +31,8 @@ on: required: false default: false type: boolean + push: + branches: ["try-mac", "try-wpt-mac", "try-wpt-mac-2020"] env: RUST_BACKTRACE: 1 @@ -71,7 +73,7 @@ jobs: - name: Script tests run: ./mach test-scripts - name: Unit tests - if: ${{ inputs.unit-tests }} + if: ${{ inputs.unit-tests || github.ref_name == 'try-mac' }} run: python3 ./mach test-unit --release - name: Package run: python3 ./mach package --release @@ -109,7 +111,7 @@ jobs: path: target.tar.gz wpt-2020: - if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }} + if: ${{ github.ref_name == 'try-wpt-mac-2020' || inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }} name: Mac WPT Tests 2020 needs: ["build"] uses: ./.github/workflows/mac-wpt.yml @@ -117,7 +119,7 @@ jobs: layout: "layout-2020" wpt-2013: - if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }} + if: ${{ github.ref_name == 'try-wpt-mac' || inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }} name: Mac WPT Tests 2013 needs: ["build"] uses: ./.github/workflows/mac-wpt.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b6c7a3d18f..22218897d19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,7 +5,7 @@ on: # Run the entire pipeline for 'master' even though the merge queue already runs checks # for every change. This just offers an extra layer of testing and covers the case of # random force pushes. - branches: ["master"] + branches: ["master", "try"] pull_request: types: ['opened', 'synchronize'] branches: ["**"] @@ -16,10 +16,7 @@ on: platform: required: true type: string - linux-wpt-layout: - required: true - type: string - mac-wpt-layout: + layout: required: true type: string unit-tests: @@ -31,11 +28,7 @@ on: required: false type: choice options: ["none", "linux", "windows", "macos", "all", "sync"] - linux-wpt-layout: - required: false - type: choice - options: ["none", "2013", "2020", "all"] - mac-wpt-layout: + layout: required: false type: choice options: ["none", "2013", "2020", "all"] @@ -55,17 +48,11 @@ jobs: uses: actions/github-script@v6 with: script: | - // Skip this workflow if an identical run already exists. Note that we don't - // always want to skip workflows. In particular we don't skip duplicate - // workflows if: - // 1. This is a try job (triggered via an `issue_comment` event or a push to - // a non-master branch) - // 2. This is a merge queue event (`merge_queue`) - // 3. This event was triggered by a pull request update (`pull_request`) - // 4. This event was triggered manually `workflow_run` / `workflow_call` - let isPushToMaster = context.eventName == "push" && process.env.GITHUB_REF_NAME == "master"; - let isTryRun = context.eventName == 'issue_comment' || (context.eventName == 'push' && !isPushToMaster); - if (!isTryRun && !['merge_group', 'pull_request', 'workflow_run', 'workflow_call'].includes(context.eventName)) { + // Never skip workflow runs for pull requests, merge groups, or manually triggered + // workfows / try jobs, which might need to actually run / retry WPT tests. + if (!['issue_comment', 'merge_group', 'pull_request', 'workflow_run', 'workflow_call'].includes(context.eventName)) { + // Skip the run if an identical run already exists. This helps to avoid running + // the workflow over and over again for the same commit hash. if ((await github.rest.actions.listWorkflowRuns({ owner: context.repo.owner, repo: context.repo.repo, @@ -81,23 +68,20 @@ jobs: // We need to pick defaults if the inputs are not provided. Unprovided inputs // are empty strings in this template. let platform = "${{ inputs.platform }}" || "linux"; + let layout = "${{ inputs.layout }}" || "none"; let unit_tests = Boolean(${{ inputs.unit-tests }}) - let linux_wpt_layout = "${{ inputs.linux-wpt-layout }}" || "none"; - let mac_wpt_layout = "${{ inputs.mac-wpt-layout }}" || "none"; // Merge queue runs and pushes to master should always trigger a full build and test. - if (isPushToMaster || context.eventName == "merge_group") { + if (["push", "merge_group"].includes(context.eventName)) { platform = "all"; + layout = "all"; unit_tests = true; - linux_wpt_layout = "all"; - mac_wpt_layout = "none"; } let returnValue = { platform, + layout, unit_tests, - linux_wpt_layout, - mac_wpt_layout, }; console.log("Using configuration: " + JSON.stringify(returnValue)); return returnValue; @@ -116,7 +100,6 @@ jobs: if: ${{ contains(fromJson('["macos", "all"]'), fromJson(needs.decision.outputs.configuration).platform) }} uses: ./.github/workflows/mac.yml with: - wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).mac_wpt_layout }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} build-linux: @@ -126,53 +109,9 @@ jobs: uses: ./.github/workflows/linux.yml with: wpt: 'test' - wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).linux_wpt_layout }} + layout: ${{ fromJson(needs.decision.outputs.configuration).layout }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} - report-test-results: - name: Report WPT Results - runs-on: ubuntu-latest - needs: - - "decision" - - "build-win" - - "build-mac" - - "build-linux" - if: ${{ - always() && !cancelled() && - fromJson(needs.decision.outputs.configuration).platform != 'none' && - ( - fromJson(needs.decision.outputs.configuration).linux_wpt_layout != 'none' || - fromJson(needs.decision.outputs.configuration).mac_wpt_layout != 'none' - ) - }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - uses: actions/download-artifact@v3 - - name: Create aggregated unexpected results - run: | - mkdir -p wpt-filtered-results - cd wpt-filtered-results - for file in *; do \ - if [ -d "$file" ]; then \ - cat $file/*.log > "unexpected-test-wpt-$file.log"; \ - fi \ - done - - name: Archive aggregate results - uses: actions/upload-artifact@v3 - with: - name: wpt-filtered-results - path: | - unexpected-test-wpt-*.log - - name: Comment on PR with results - run: | - etc/ci/report_aggregated_expected_results.py wpt-filtered-results - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - RESULTS: ${{ toJson(needs.*.result) }} - build-result: name: Result runs-on: ubuntu-latest diff --git a/.github/workflows/try.yml b/.github/workflows/try.yml index e624f474edf..88936b8f59b 100644 --- a/.github/workflows/try.yml +++ b/.github/workflows/try.yml @@ -3,13 +3,11 @@ name: Try on: issue_comment: types: [created] - push: - branches: ["try", "try-*"] jobs: parse-comment: - name: Process Comment or Branch Name - if: ${{ github.event_name == 'push' || (github.event_name == 'issue_comment' && github.event.issue.pull_request) }} + name: Process Comment + if: ${{ github.event.issue.pull_request }} runs-on: ubuntu-latest outputs: configuration: ${{ steps.configuration.outputs.result }} @@ -20,55 +18,41 @@ jobs: script: | function makeComment(body) { console.log(body); - if (context.eventName == "issue_comment") { - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body - }); - } + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body + }) } - let tryString = ""; - if (context.eventName == "push") { - // Replace the first instance of "-" in the branch name to make it similar - // to the try syntax string expected below. - tryString = process.env.GITHUB_REF_NAME.replace("-", "="); - } else { - // This is a pull request comment event. - let tokens = context.payload.comment.body.split(/\s+/); - let tagIndex = tokens.indexOf("@bors-servo"); - if (tagIndex == -1 || tagIndex + 1 >= tokens.length) { - return { try: false }; - } - tryString = tokens[tagIndex + 1]; + let tokens = context.payload.comment.body.split(/\s+/); + let tagIndex = tokens.indexOf("@bors-servo"); + if (tagIndex == -1 || tagIndex + 1 >= tokens.length) { + return { try: false }; } + let tryString = tokens[tagIndex + 1]; console.log("Found try string: '" + tryString + "'"); let returnValue = { try: false }; if (tryString == "try") { - returnValue = { try: true, platform: 'all', linux_wpt_layout: 'all', mac_wpt_layout: 'none', unit_tests: true, }; - } else if (tryString == "try=linux") { - returnValue = { try: true, platform: 'linux', linux_wpt_layout: 'none', mac_wpt_layout: 'none', unit_tests: true }; - } else if (tryString == "try=mac") { - returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: 'none', unit_tests: true }; - } else if (tryString == "try=windows") { - returnValue = { try: true, platform: 'windows', linux_wpt_layout: 'none', unit_tests: true }; + returnValue = { try: true, platform: 'all', layout: 'all', unit_tests: true, }; } else if (tryString == "try=wpt") { - returnValue = { try: true, platform: 'linux', linux_wpt_layout: '2013', mac_wpt_layout: 'none', unit_tests: false }; + returnValue = { try: true, platform: 'linux', layout: '2013', unit_tests: false }; } else if (tryString == "try=wpt-2020") { - returnValue = { try: true, platform: 'linux', linux_wpt_layout: '2020', mac_wpt_layout: 'none', unit_tests: false }; - } else if (tryString == "try=mac-wpt") { - returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: '2013', unit_tests: false }; - } else if (tryString == "try=mac-wpt-2020") { - returnValue = { try: true, platform: 'macos', linux_wpt_layout: 'none', mac_wpt_layout: '2020', unit_tests: false }; + returnValue = { try: true, platform: 'linux', layout: '2020', unit_tests: false }; + } else if (tryString == "try=linux") { + returnValue = { try: true, platform: 'linux', layout: 'none', unit_tests: true }; + } else if (tryString == "try=mac") { + returnValue = { try: true, platform: 'macos', layout: 'none', unit_tests: true }; + } else if (tryString == "try=windows") { + returnValue = { try: true, platform: 'windows', layout: 'none', unit_tests: true }; } else { makeComment("🤔 Unknown try string '" + tryString + "'"); return returnValue; } - if (returnValue.try && context.eventName == "issue_comment") { + if (returnValue.try) { let username = context.payload.comment.user.login; let result = await github.rest.repos.getCollaboratorPermissionLevel({ owner: context.repo.owner, @@ -96,8 +80,7 @@ jobs: uses: ./.github/workflows/main.yml with: platform: ${{ fromJson(needs.parse-comment.outputs.configuration).platform }} - linux-wpt-layout: ${{ fromJson(needs.parse-comment.outputs.configuration).linux_wpt_layout }} - mac-wpt-layout: ${{ fromJson(needs.parse-comment.outputs.configuration).mac_wpt_layout }} + layout: ${{ fromJson(needs.parse-comment.outputs.configuration).layout }} unit-tests: ${{ fromJson(needs.parse-comment.outputs.configuration).unit_tests }} results: @@ -106,26 +89,37 @@ jobs: runs-on: ubuntu-latest if: ${{ always() && fromJson(needs.parse-comment.outputs.configuration).try}} steps: - - name: Result Comment + - name: Success + if: ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} uses: actions/github-script@v6 with: script: | - let success = ${{ !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}; - const url = context.serverUrl + "/" + context.repo.owner + "/" + context.repo.repo + "/actions/runs/" + context.runId; const formattedURL = "[#" + context.runId + "](" + url + ")"; + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: "✨ Try run (" + formattedURL + ") " + "succeeded.", + }); + - name: Failure + if: ${{ contains(needs.*.result, 'failure') }} + uses: actions/github-script@v6 + with: + script: | + const url = context.serverUrl + + "/" + context.repo.owner + + "/" + context.repo.repo + + "/actions/runs/" + context.runId; + const formattedURL = "[#" + context.runId + "](" + url + ")"; + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: "⚠️ Try run (" + formattedURL + ") " + "failed.", + }); + - let body = success ? - "✨ Try run (" + formattedURL + ") " + "succeeded." : - "⚠️ Try run (" + formattedURL + ") " + "failed."; - if (context.eventName == "issue_comment") { - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body - }); - } diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b7a4eedc1c1..38eb0e9b3eb 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -24,6 +24,8 @@ on: required: false default: false type: boolean + push: + branches: ["try-windows"] env: RUST_BACKTRACE: 1 @@ -69,7 +71,7 @@ jobs: - name: Smoketest run: python mach smoketest --angle --release - name: Unit tests - if: ${{ inputs.unit-tests }} + if: ${{ inputs.unit-tests || github.ref_name == 'try-windows' }} run: python mach test-unit --release - name: Rename build timing run: cp C:\a\servo\servo\target\cargo-timings C:\a\servo\servo\target\cargo-timings-windows -Recurse diff --git a/etc/ci/report_aggregated_expected_results.py b/etc/ci/report_aggregated_expected_results.py index 2929f245ccc..f0046456c27 100755 --- a/etc/ci/report_aggregated_expected_results.py +++ b/etc/ci/report_aggregated_expected_results.py @@ -12,13 +12,12 @@ # This allows using types that are defined later in the file. from __future__ import annotations from datetime import datetime -import glob import json import os import re import subprocess -import sys +import argparse import textwrap import xml.etree.ElementTree as ElementTree @@ -74,8 +73,7 @@ class Item: def to_html(self, level: int = 0) -> ElementTree.Element: if level == 0: - result = ElementTree.Element("span") - title = ElementTree.SubElement(result, "h4") + title = result = ElementTree.Element("span") elif level == 1: result = ElementTree.Element("details") title = ElementTree.SubElement(result, "summary") @@ -178,7 +176,7 @@ def get_pr_number() -> Optional[str]: return None -def create_check_run(body: str): +def create_check_run(body: str, tag: str = ""): # This process is based on the documentation here: # https://docs.github.com/en/rest/checks/runs?apiVersion=2022-11-28#create-a-check-runs results = json.loads(os.environ.get("RESULTS", "{}")) @@ -199,14 +197,14 @@ def create_check_run(body: str): return None repo = github_context["repository"] data = { - 'name': 'wpt-results-report', + 'name': tag, 'head_sha': github_context["sha"], 'status': 'completed', 'started_at': datetime.utcnow().replace(microsecond=0).isoformat() + "Z", 'conclusion': conclusion, 'completed_at': datetime.utcnow().replace(microsecond=0).isoformat() + "Z", 'output': { - 'title': 'WPT Results Report', + 'title': f'Aggregated {tag} report', 'summary': body, 'images': [{'alt': 'WPT logo', 'image_url': 'https://avatars.githubusercontent.com/u/37226233'}] }, @@ -224,34 +222,23 @@ def create_check_run(body: str): def main(): - if len(sys.argv) < 2: - print("Must pass the directory containing filtered WPT results as an argument.") - sys.exit(1) - - results = [] - for dir in os.listdir(sys.argv[1]): - if not os.path.isdir(dir): - continue - new_results = get_results( - glob.glob(os.path.join(dir, "*.json")), - os.path.basename(dir)) - if new_results: - results.append(new_results) - + parser = argparse.ArgumentParser() + parser.add_argument("--tag", default="wpt", action="store", + help="A string tag used to distinguish the results.") + args, filenames = parser.parse_known_args() + results = get_results(filenames, args.tag) if not results: print("Did not find any unexpected results.") - create_check_run("Did not find any unexpected results.") + create_check_run("Did not find any unexpected results.", args.tag) return - for result in results: - print(result.to_string() + "\n") - - html_string = "\n".join( - [ElementTree.tostring(result.to_html(), encoding="unicode") for result in results]) - print(html_string) - create_check_run(html_string) + print(results.to_string()) pr_number = get_pr_number() + html_string = ElementTree.tostring( + results.to_html(), encoding="unicode") + create_check_run(html_string, args.tag) + if pr_number: process = subprocess.Popen( ['gh', 'pr', 'comment', pr_number, '-F', '-'], stdin=subprocess.PIPE)