From 1f0f50b22bb43e965febf31f5a436113f2958725 Mon Sep 17 00:00:00 2001 From: Samson <16504129+sagudev@users.noreply.github.com> Date: Fri, 22 Dec 2023 13:49:39 +0100 Subject: [PATCH] Customizable wpt options and profile in workflows (#30912) * wpt -> wpt-sync * Add profile configuration option * Add wpt (for custom tests selection) to CI * Renaming of wpt-* options * fixup! do renames also in scripts * fixup! profile * Fix try labels --- .github/workflows/linux-wpt.yml | 42 ++++++++++++---------- .github/workflows/linux.yml | 36 ++++++++++++------- .github/workflows/mac-wpt.yml | 22 +++++++----- .github/workflows/mac.yml | 14 ++++++-- .github/workflows/main.yml | 34 ++++++++++++------ .github/workflows/scheduled-wpt-import.yml | 4 +-- .github/workflows/try.yml | 14 ++++---- 7 files changed, 105 insertions(+), 61 deletions(-) diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index 7ac08bfb861..c9b674090ac 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -5,21 +5,25 @@ on: profile: required: true type: string - wpt: + wpt-tests-to-run: + default: "" required: false type: string - layout: + wpt-sync-from-upstream: + required: false + type: boolean + wpt-layout: required: true type: string env: RUST_BACKTRACE: 1 SHELL: /bin/bash - WPT_COMMAND_LINE_ARG: "${{ inputs.layout == 'layout-2013' && '--legacy-layout' || '' }}" + WPT_COMMAND_LINE_ARG: "${{ inputs.wpt-layout == 'layout-2013' && '--legacy-layout' || '' }}" jobs: linux-wpt: - name: WPT ${{ inputs.layout }} + name: WPT ${{ inputs.wpt-layout }} runs-on: ubuntu-22.04 env: max_chunk_id: 20 @@ -63,13 +67,13 @@ jobs: sudo apt install ./libffi6_3.2.1-8_amd64.deb python3 ./mach bootstrap-gstreamer - name: Sync from upstream WPT - if: ${{ inputs.wpt == 'sync' }} + if: ${{ inputs.wpt-sync-from-upstream == 'true' }} run: | ./mach update-wpt --sync --patch - name: Run tests - if: ${{ inputs.wpt != 'sync' }} + if: ${{ inputs.wpt-sync-from-upstream != 'true' }} run: | - python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ + python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG ${{ inputs.wpt-tests-to-run }} \ --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \ @@ -79,7 +83,7 @@ jobs: GITHUB_CONTEXT: ${{ toJson(github) }} INTERMITTENT_TRACKER_DASHBOARD_SECRET: ${{ secrets.INTERMITTENT_TRACKER_DASHBOARD_SECRET }} - name: Run tests (sync) - if: ${{ inputs.wpt == 'sync' }} + if: ${{ inputs.wpt-sync-from-upstream == 'true' }} run: | python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \ @@ -88,24 +92,24 @@ jobs: --always-succeed - name: Archive filtered results uses: actions/upload-artifact@v3 - if: ${{ always() && inputs.wpt != 'sync' }} + if: ${{ always() && inputs.wpt-sync-from-upstream != 'true' }} with: - name: wpt-filtered-results-linux-${{ inputs.layout }} + name: wpt-filtered-results-linux-${{ inputs.wpt-layout }} path: | filtered-test-wpt.${{ matrix.chunk_id }}.json unexpected-test-wpt.${{ matrix.chunk_id }}.log - name: Archive logs uses: actions/upload-artifact@v3 - if: ${{ failure() && inputs.wpt != 'sync' }} + if: ${{ failure() && inputs.wpt-sync-from-upstream != 'true' }} with: - name: wpt-logs-linux-${{ inputs.layout }} + name: wpt-logs-linux-${{ inputs.wpt-layout }} path: | test-wpt.${{ matrix.chunk_id }}.log - name: Archive logs uses: actions/upload-artifact@v3 - if: ${{ inputs.wpt == 'sync' }} + if: ${{ inputs.wpt-sync-from-upstream == 'true' }} with: - name: wpt-logs-linux-${{ inputs.layout }} + name: wpt-logs-linux-${{ inputs.wpt-layout }} path: | test-wpt.${{ matrix.chunk_id }}.log wpt-jsonsummary.${{ matrix.chunk_id }}.log @@ -113,7 +117,7 @@ jobs: report-test-results: name: Report WPT Results runs-on: ubuntu-latest - if: ${{ always() && !cancelled() && inputs.wpt == 'test' }} + if: ${{ always() && !cancelled() && inputs.wpt-sync-from-upstream != 'true' }} needs: - "linux-wpt" steps: @@ -122,20 +126,20 @@ jobs: fetch-depth: 2 - uses: actions/download-artifact@v3 with: - name: wpt-filtered-results-linux-${{ inputs.layout }} + name: wpt-filtered-results-linux-${{ inputs.wpt-layout }} path: wpt-filtered-results-linux - name: Create aggregated unexpected results run: | - cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.layout }}.log + cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.wpt-layout }}.log - name: Archive aggregate results uses: actions/upload-artifact@v3 with: name: wpt-filtered-results-linux path: | - unexpected-test-wpt-${{ inputs.layout }}.log + unexpected-test-wpt-${{ inputs.wpt-layout }}.log - name: Comment on PR with results run: | - etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.layout }}" wpt-filtered-results-linux/*.json + etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.wpt-layout }}" wpt-filtered-results-linux/*.json env: GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f89d791d535..0a3f412a049 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -6,10 +6,15 @@ on: required: false default: "release" type: string - wpt: + wpt-tests-to-run: + default: "" required: false type: string - layout: + wpt-sync-from-upstream: + required: false + default: false + type: boolean + wpt-layout: required: false type: string unit-tests: @@ -30,12 +35,15 @@ on: default: "release" type: choice options: ["release", "debug", "production"] - wpt: - default: "test" + wpt-tests-to-run: + default: "" required: false - type: choice - options: ["test", "sync"] - layout: + type: string + wpt-sync-from-upstream: + default: false + required: false + type: boolean + wpt-layout: required: false type: choice options: ["none", "2013", "2020", "all"] @@ -140,25 +148,27 @@ jobs: path: target.tar.gz wpt-2020: - if: ${{ inputs.layout == '2020' || inputs.layout == 'all' }} + if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }} name: Linux WPT Tests 2020 needs: ["build"] uses: ./.github/workflows/linux-wpt.yml with: + wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }} profile: ${{ inputs.profile }} - wpt: ${{ inputs.wpt }} - layout: "layout-2020" + wpt-sync-from-upstream: ${{ inputs.wpt-sync-from-upstream }} + wpt-layout: "layout-2020" secrets: inherit wpt-2013: - if: ${{ inputs.layout == '2013' || inputs.layout == 'all' }} + if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }} name: Linux WPT Tests 2013 needs: ["build"] uses: ./.github/workflows/linux-wpt.yml with: + wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }} profile: ${{ inputs.profile }} - wpt: ${{ inputs.wpt }} - layout: "layout-2013" + wpt-sync-from-upstream: ${{ inputs.wpt-sync-from-upstream }} + wpt-layout: "layout-2013" secrets: inherit result: diff --git a/.github/workflows/mac-wpt.yml b/.github/workflows/mac-wpt.yml index ca19f60a3be..76d21604e1b 100644 --- a/.github/workflows/mac-wpt.yml +++ b/.github/workflows/mac-wpt.yml @@ -3,21 +3,25 @@ name: Mac WPT Tests on: workflow_call: inputs: + wpt-tests-to-run: + default: "" + required: false + type: string profile: required: true type: string - layout: + wpt-layout: required: true type: string env: RUST_BACKTRACE: 1 SHELL: /bin/bash - WPT_COMMAND_LINE_ARG: "${{ inputs.layout == 'layout-2013' && '--legacy-layout' || '' }}" + WPT_COMMAND_LINE_ARG: "${{ inputs.wpt-layout == 'layout-2013' && '--legacy-layout' || '' }}" jobs: mac-wpt: - name: WPT ${{ inputs.layout }} + name: WPT ${{ inputs.wpt-layout }} runs-on: macos-13 env: max_chunk_id: 5 @@ -49,7 +53,7 @@ jobs: run: python3 ./mach smoketest --${{ inputs.profile }} - name: Run tests run: | - python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ + python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG ${{ inputs.wpt-tests-to-run }} \ --${{ inputs.profile }} --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \ @@ -59,7 +63,7 @@ jobs: uses: actions/upload-artifact@v3 if: always() with: - name: wpt-filtered-results-mac-${{ inputs.layout }} + name: wpt-filtered-results-mac-${{ inputs.wpt-layout }} path: | filtered-test-wpt.${{ matrix.chunk_id }}.json unexpected-test-wpt.${{ matrix.chunk_id }}.log @@ -67,7 +71,7 @@ jobs: uses: actions/upload-artifact@v3 if: failure() with: - name: wpt-logs-mac-${{ inputs.layout }} + name: wpt-logs-mac-${{ inputs.wpt-layout }} path: | test-wpt.${{ matrix.chunk_id }}.log filtered-wpt-results.${{ matrix.chunk_id }}.json @@ -83,18 +87,18 @@ jobs: fetch-depth: 2 - uses: actions/download-artifact@v3 with: - name: wpt-filtered-results-mac-${{ inputs.layout }} + name: wpt-filtered-results-mac-${{ inputs.wpt-layout }} path: wpt-filtered-results-mac - name: Create aggregated unexpected results run: cat wpt-filtered-results-mac/*.log > unexpected-test-wpt.log - name: Archive aggregate results uses: actions/upload-artifact@v3 with: - name: wpt-filtered-results-mac-${{ inputs.layout }} + name: wpt-filtered-results-mac-${{ inputs.wpt-layout }} path: | unexpected-test-wpt.log - name: Comment on PR with results - run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.layout }}" + run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.wpt-layout }}" wpt-filtered-results-mac/*.json env: GITHUB_CONTEXT: ${{ toJson(github) }} diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 5e2b6948853..c416a53888c 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -7,6 +7,10 @@ on: required: false default: "release" type: string + wpt-tests-to-run: + default: "" + required: false + type: string wpt-layout: required: false type: string @@ -28,6 +32,10 @@ on: default: "release" type: choice options: ["release", "debug", "production"] + wpt-tests-to-run: + default: "" + required: false + type: string wpt-layout: required: false type: choice @@ -147,7 +155,8 @@ jobs: uses: ./.github/workflows/mac-wpt.yml with: profile: ${{ inputs.profile }} - layout: "layout-2020" + wpt-layout: "layout-2020" + wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }} secrets: inherit wpt-2013: @@ -157,7 +166,8 @@ jobs: uses: ./.github/workflows/mac-wpt.yml with: profile: ${{ inputs.profile }} - layout: "layout-2013" + wpt-layout: "layout-2013" + wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }} secrets: inherit result: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d841afd5095..82defcc2f6c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,11 +18,20 @@ on: type: string workflow_dispatch: inputs: + profile: + required: false + default: "release" + type: choice + options: ["release", "debug", "production"] + wpt-tests-to-run: + default: "" + required: false + type: string platform: required: false type: choice options: ["linux", "windows", "macos", "all"] - layout: + wpt-layout: required: false type: choice options: ["none", "2013", "2020", "all"] @@ -53,13 +62,15 @@ jobs: // We need to pick defaults if the inputs are not provided. Unprovided inputs // are empty strings in this template. let platform = "${{ inputs.platform }}" || "linux"; - let layout = "${{ inputs.layout }}" || "none"; + let profile = "${{ inputs.profile }}" || "release"; + let wpt_layout = "${{ inputs.wpt-layout }}" || "none"; + let wpt_tests_to_run = "${{ inputs.wpt-tests-to-run }}" || ""; let unit_tests = Boolean(${{ inputs.unit-tests }}) // Merge queue runs and pushes to `main` should always trigger a full build and test. if (["push", "merge_group"].includes(context.eventName)) { platform = "all"; - layout = "all"; + wpt_layout = "all"; unit_tests = true; } @@ -70,10 +81,12 @@ jobs: platforms = [ platform ]; } - let returnValue = { + let returnValue = { platforms, - layout, + wpt_layout, unit_tests, + profile, + wpt_tests_to_run, }; console.log("Using configuration: " + JSON.stringify(returnValue)); @@ -85,7 +98,7 @@ jobs: if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'windows') }} uses: ./.github/workflows/windows.yml with: - profile: "release" + profile: ${{ fromJson(needs.decision.outputs.configuration).profile }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} secrets: inherit @@ -95,7 +108,8 @@ jobs: if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'macos') }} uses: ./.github/workflows/mac.yml with: - profile: "release" + wpt-tests-to-run: ${{ fromJson(needs.decision.outputs.configuration).wpt_tests_to_run }} + profile: ${{ fromJson(needs.decision.outputs.configuration).profile }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} secrets: inherit @@ -105,9 +119,9 @@ jobs: if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'linux') }} uses: ./.github/workflows/linux.yml with: - profile: "release" - wpt: 'test' - layout: ${{ fromJson(needs.decision.outputs.configuration).layout }} + wpt-tests-to-run: ${{ fromJson(needs.decision.outputs.configuration).wpt_tests_to_run }} + profile: ${{ fromJson(needs.decision.outputs.configuration).profile }} + wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).wpt_layout }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} secrets: inherit diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml index bc4166b05b0..5cf9d957732 100644 --- a/.github/workflows/scheduled-wpt-import.yml +++ b/.github/workflows/scheduled-wpt-import.yml @@ -15,8 +15,8 @@ jobs: name: Linux uses: ./.github/workflows/linux.yml with: - wpt: 'sync' - layout: 'all' + wpt-sync-from-upstream: true + wpt-layout: 'all' unit-tests: false sync: diff --git a/.github/workflows/try.yml b/.github/workflows/try.yml index e6d4a97773d..5ae34e97f01 100644 --- a/.github/workflows/try.yml +++ b/.github/workflows/try.yml @@ -22,7 +22,7 @@ jobs: function makeComment(body) { console.log(body); - if (github.event_name != 'pull_request_target') + if (context.eventName != 'pull_request_target') return; github.rest.issues.createComment({ @@ -61,7 +61,7 @@ jobs: if (tryString.includes("full")) { configuration.platforms = ["linux", "macos", "windows"]; configuration.unit_tests = true; - configuration.layout = "all"; + configuration.wpt_layout = "all"; return configuration; } @@ -79,20 +79,22 @@ jobs: if (tryString.includes("wpt")) { addPlatformToConfiguration("linux", configuration); if (tryString.includes("2020")) { - configuration.layout = combineWPTLayoutOptions(configuration.layout, "2020"); + configuration.wpt_layout = combineWPTLayoutOptions(configuration.layout, "2020"); } else { - configuration.layout = combineWPTLayoutOptions(configuration.layout, "2013"); + configuration.wpt_layout = combineWPTLayoutOptions(configuration.layout, "2013"); } } } let configuration = { platforms: [], - layout: "none", + wpt_layout: "none", unit_tests: false, + profile: "release", + wpt_tests_to_run: "", }; - if (github.event_name == 'pull_request_target') { + if (context.eventName == 'pull_request_target') { for (const label of context.payload.pull_request.labels) { if (!label.name.startsWith("T-")) { continue;