Customizable wpt options and profile in workflows (#30912)

* wpt -> wpt-sync

* Add profile configuration option

* Add wpt (for custom tests selection) to CI

* Renaming of wpt-* options

* fixup! do renames also in scripts

* fixup! profile

* Fix try labels

.github/workflows/linux-wpt.yml

@@ -5,21 +5,25 @@ on:
       profile:
         required: true
         type: string
-      wpt:
+      wpt-tests-to-run:
+        default: ""
         required: false
         type: string
-      layout:
+      wpt-sync-from-upstream:
+        required: false
+        type: boolean
+      wpt-layout:
         required: true
         type: string
 
 env:
   RUST_BACKTRACE: 1
   SHELL: /bin/bash
-  WPT_COMMAND_LINE_ARG: "${{ inputs.layout == 'layout-2013' && '--legacy-layout' || '' }}"
+  WPT_COMMAND_LINE_ARG: "${{ inputs.wpt-layout == 'layout-2013' && '--legacy-layout' || '' }}"
 
 jobs:
   linux-wpt:
-    name: WPT ${{ inputs.layout }}
+    name: WPT ${{ inputs.wpt-layout }}
     runs-on: ubuntu-22.04
     env:
       max_chunk_id: 20
@@ -63,13 +67,13 @@ jobs:
           sudo apt install ./libffi6_3.2.1-8_amd64.deb
           python3 ./mach bootstrap-gstreamer
       - name: Sync from upstream WPT
-        if: ${{ inputs.wpt == 'sync' }}
+        if: ${{ inputs.wpt-sync-from-upstream == 'true' }}
         run: |
           ./mach update-wpt --sync --patch
       - name: Run tests
-        if: ${{ inputs.wpt != 'sync' }}
+        if: ${{ inputs.wpt-sync-from-upstream != 'true' }}
         run: |
-          python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \
+          python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG ${{ inputs.wpt-tests-to-run }} \
             --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
@@ -79,7 +83,7 @@ jobs:
           GITHUB_CONTEXT: ${{ toJson(github) }}
           INTERMITTENT_TRACKER_DASHBOARD_SECRET: ${{ secrets.INTERMITTENT_TRACKER_DASHBOARD_SECRET }}
       - name: Run tests (sync)
-        if: ${{ inputs.wpt == 'sync' }}
+        if: ${{ inputs.wpt-sync-from-upstream == 'true' }}
         run: |
           python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \
             --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \
@@ -88,24 +92,24 @@ jobs:
             --always-succeed
       - name: Archive filtered results
         uses: actions/upload-artifact@v3
-        if: ${{ always() && inputs.wpt != 'sync' }}
+        if: ${{ always() && inputs.wpt-sync-from-upstream != 'true' }}
         with:
-          name: wpt-filtered-results-linux-${{ inputs.layout }}
+          name: wpt-filtered-results-linux-${{ inputs.wpt-layout }}
           path: |
             filtered-test-wpt.${{ matrix.chunk_id }}.json
             unexpected-test-wpt.${{ matrix.chunk_id }}.log
       - name: Archive logs
         uses: actions/upload-artifact@v3
-        if: ${{ failure() && inputs.wpt != 'sync' }}
+        if: ${{ failure() && inputs.wpt-sync-from-upstream != 'true' }}
         with:
-          name: wpt-logs-linux-${{ inputs.layout }}
+          name: wpt-logs-linux-${{ inputs.wpt-layout }}
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
       - name: Archive logs
         uses: actions/upload-artifact@v3
-        if: ${{ inputs.wpt == 'sync' }}
+        if: ${{ inputs.wpt-sync-from-upstream == 'true' }}
         with:
-          name: wpt-logs-linux-${{ inputs.layout }}
+          name: wpt-logs-linux-${{ inputs.wpt-layout }}
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
             wpt-jsonsummary.${{ matrix.chunk_id }}.log
@@ -113,7 +117,7 @@ jobs:
   report-test-results:
     name: Report WPT Results
     runs-on: ubuntu-latest
-    if: ${{ always() && !cancelled() && inputs.wpt == 'test' }}
+    if: ${{ always() && !cancelled() && inputs.wpt-sync-from-upstream != 'true' }}
     needs:
       - "linux-wpt"
     steps:
@@ -122,20 +126,20 @@ jobs:
           fetch-depth: 2
       - uses: actions/download-artifact@v3
         with:
-          name: wpt-filtered-results-linux-${{ inputs.layout }}
+          name: wpt-filtered-results-linux-${{ inputs.wpt-layout }}
           path: wpt-filtered-results-linux
       - name: Create aggregated unexpected results
         run: |
-          cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.layout }}.log
+          cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.wpt-layout }}.log
       - name: Archive aggregate results
         uses: actions/upload-artifact@v3
         with:
           name: wpt-filtered-results-linux
           path: |
-            unexpected-test-wpt-${{ inputs.layout }}.log
+            unexpected-test-wpt-${{ inputs.wpt-layout }}.log
       - name: Comment on PR with results
         run: |
-          etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.layout }}" wpt-filtered-results-linux/*.json
+          etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.wpt-layout }}" wpt-filtered-results-linux/*.json
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/linux.yml

@@ -6,10 +6,15 @@ on:
         required: false
         default: "release"
         type: string
-      wpt:
+      wpt-tests-to-run:
+        default: ""
         required: false
         type: string
-      layout:
+      wpt-sync-from-upstream:
+        required: false
+        default: false
+        type: boolean
+      wpt-layout:
         required: false
         type: string
       unit-tests:
@@ -30,12 +35,15 @@ on:
         default: "release"
         type: choice
         options: ["release", "debug", "production"]
-      wpt:
-        default: "test"
+      wpt-tests-to-run:
+        default: ""
         required: false
-        type: choice
-        options: ["test", "sync"]
-      layout:
+        type: string
+      wpt-sync-from-upstream:
+        default: false
+        required: false
+        type: boolean
+      wpt-layout:
         required: false
         type: choice
         options: ["none", "2013", "2020", "all"]
@@ -140,25 +148,27 @@ jobs:
           path: target.tar.gz
 
   wpt-2020:
-    if: ${{ inputs.layout == '2020' || inputs.layout == 'all' }}
+    if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
     name: Linux WPT Tests 2020
     needs: ["build"]
     uses: ./.github/workflows/linux-wpt.yml
     with:
+      wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
       profile: ${{ inputs.profile }}
-      wpt: ${{ inputs.wpt }}
-      layout: "layout-2020"
+      wpt-sync-from-upstream: ${{ inputs.wpt-sync-from-upstream }}
+      wpt-layout: "layout-2020"
     secrets: inherit
 
   wpt-2013:
-    if: ${{ inputs.layout == '2013' || inputs.layout == 'all' }}
+    if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
     name: Linux WPT Tests 2013
     needs: ["build"]
     uses: ./.github/workflows/linux-wpt.yml
     with:
+      wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
       profile: ${{ inputs.profile }}
-      wpt: ${{ inputs.wpt }}
-      layout: "layout-2013"
+      wpt-sync-from-upstream: ${{ inputs.wpt-sync-from-upstream }}
+      wpt-layout: "layout-2013"
     secrets: inherit
 
   result:

.github/workflows/mac-wpt.yml

@@ -3,21 +3,25 @@ name: Mac WPT Tests
 on:
   workflow_call:
     inputs:
+      wpt-tests-to-run:
+        default: ""
+        required: false
+        type: string
       profile:
         required: true
         type: string
-      layout:
+      wpt-layout:
         required: true
         type: string
 
 env:
   RUST_BACKTRACE: 1
   SHELL: /bin/bash
-  WPT_COMMAND_LINE_ARG: "${{ inputs.layout == 'layout-2013' && '--legacy-layout' || '' }}"
+  WPT_COMMAND_LINE_ARG: "${{ inputs.wpt-layout == 'layout-2013' && '--legacy-layout' || '' }}"
 
 jobs:
   mac-wpt:
-    name: WPT ${{ inputs.layout }}
+    name: WPT ${{ inputs.wpt-layout }}
     runs-on: macos-13
     env:
       max_chunk_id: 5
@@ -49,7 +53,7 @@ jobs:
         run: python3 ./mach smoketest --${{ inputs.profile }}
       - name: Run tests
         run: |
-          python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \
+          python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG ${{ inputs.wpt-tests-to-run }} \
             --${{ inputs.profile }} --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
@@ -59,7 +63,7 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: wpt-filtered-results-mac-${{ inputs.layout }}
+          name: wpt-filtered-results-mac-${{ inputs.wpt-layout }}
           path: |
             filtered-test-wpt.${{ matrix.chunk_id }}.json
             unexpected-test-wpt.${{ matrix.chunk_id }}.log
@@ -67,7 +71,7 @@ jobs:
         uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: wpt-logs-mac-${{ inputs.layout }}
+          name: wpt-logs-mac-${{ inputs.wpt-layout }}
           path: |
             test-wpt.${{ matrix.chunk_id }}.log
             filtered-wpt-results.${{ matrix.chunk_id }}.json
@@ -83,18 +87,18 @@ jobs:
           fetch-depth: 2
       - uses: actions/download-artifact@v3
         with:
-          name: wpt-filtered-results-mac-${{ inputs.layout }}
+          name: wpt-filtered-results-mac-${{ inputs.wpt-layout }}
           path: wpt-filtered-results-mac
       - name: Create aggregated unexpected results
         run: cat wpt-filtered-results-mac/*.log > unexpected-test-wpt.log
       - name: Archive aggregate results
         uses: actions/upload-artifact@v3
         with:
-          name: wpt-filtered-results-mac-${{ inputs.layout }}
+          name: wpt-filtered-results-mac-${{ inputs.wpt-layout }}
           path: |
             unexpected-test-wpt.log
       - name: Comment on PR with results
-        run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.layout }}"
+        run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.wpt-layout }}"
           wpt-filtered-results-mac/*.json
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}

.github/workflows/mac.yml

@@ -7,6 +7,10 @@ on:
         required: false
         default: "release"
         type: string
+      wpt-tests-to-run:
+        default: ""
+        required: false
+        type: string
       wpt-layout:
         required: false
         type: string
@@ -28,6 +32,10 @@ on:
         default: "release"
         type: choice
         options: ["release", "debug", "production"]
+      wpt-tests-to-run:
+        default: ""
+        required: false
+        type: string
       wpt-layout:
         required: false
         type: choice
@@ -147,7 +155,8 @@ jobs:
     uses: ./.github/workflows/mac-wpt.yml
     with:
       profile: ${{ inputs.profile }}
-      layout: "layout-2020"
+      wpt-layout: "layout-2020"
+      wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
     secrets: inherit
 
   wpt-2013:
@@ -157,7 +166,8 @@ jobs:
     uses: ./.github/workflows/mac-wpt.yml
     with:
       profile: ${{ inputs.profile }}
-      layout: "layout-2013"
+      wpt-layout: "layout-2013"
+      wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
     secrets: inherit
 
   result:

.github/workflows/main.yml

@@ -18,11 +18,20 @@ on:
         type: string
   workflow_dispatch:
     inputs:
+      profile:
+        required: false
+        default: "release"
+        type: choice
+        options: ["release", "debug", "production"]
+      wpt-tests-to-run:
+        default: ""
+        required: false
+        type: string
       platform:
         required: false
         type: choice
         options: ["linux", "windows", "macos", "all"]
-      layout:
+      wpt-layout:
         required: false
         type: choice
         options: ["none", "2013", "2020", "all"]
@@ -53,13 +62,15 @@ jobs:
             // We need to pick defaults if the inputs are not provided. Unprovided inputs
             // are empty strings in this template.
             let platform = "${{ inputs.platform }}" || "linux";
-            let layout = "${{ inputs.layout }}" || "none";
+            let profile = "${{ inputs.profile }}" || "release";
+            let wpt_layout = "${{ inputs.wpt-layout }}" || "none";
+            let wpt_tests_to_run = "${{ inputs.wpt-tests-to-run }}" || "";
             let unit_tests = Boolean(${{ inputs.unit-tests }})
 
             // Merge queue runs and pushes to `main` should always trigger a full build and test.
             if (["push", "merge_group"].includes(context.eventName)) {
               platform = "all";
-              layout = "all";
+              wpt_layout = "all";
               unit_tests = true;
             }
 
@@ -70,10 +81,12 @@ jobs:
               platforms = [ platform ];
             }
 
-            let returnValue =  {
+            let returnValue = {
               platforms,
-              layout,
+              wpt_layout,
               unit_tests,
+              profile,
+              wpt_tests_to_run,
             };
 
             console.log("Using configuration: " + JSON.stringify(returnValue));
@@ -85,7 +98,7 @@ jobs:
     if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'windows') }}
     uses: ./.github/workflows/windows.yml
     with:
-      profile: "release"
+      profile: ${{ fromJson(needs.decision.outputs.configuration).profile }}
       unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
     secrets: inherit
 
@@ -95,7 +108,8 @@ jobs:
     if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'macos') }}
     uses: ./.github/workflows/mac.yml
     with:
-      profile: "release"
+      wpt-tests-to-run: ${{ fromJson(needs.decision.outputs.configuration).wpt_tests_to_run }}
+      profile: ${{ fromJson(needs.decision.outputs.configuration).profile }}
       unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
     secrets: inherit
 
@@ -105,9 +119,9 @@ jobs:
     if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'linux') }}
     uses: ./.github/workflows/linux.yml
     with:
-      profile: "release"
-      wpt: 'test'
-      layout: ${{ fromJson(needs.decision.outputs.configuration).layout }}
+      wpt-tests-to-run: ${{ fromJson(needs.decision.outputs.configuration).wpt_tests_to_run }}
+      profile: ${{ fromJson(needs.decision.outputs.configuration).profile }}
+      wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).wpt_layout }}
       unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
     secrets: inherit
 

.github/workflows/scheduled-wpt-import.yml

@@ -15,8 +15,8 @@ jobs:
     name: Linux
     uses: ./.github/workflows/linux.yml
     with:
-      wpt: 'sync'
-      layout: 'all'
+      wpt-sync-from-upstream: true
+      wpt-layout: 'all'
       unit-tests: false
 
   sync:

.github/workflows/try.yml

@@ -22,7 +22,7 @@ jobs:
             function makeComment(body) {
                 console.log(body);
 
-                if (github.event_name != 'pull_request_target')
+                if (context.eventName != 'pull_request_target')
                   return;
 
                 github.rest.issues.createComment({
@@ -61,7 +61,7 @@ jobs:
                 if (tryString.includes("full")) {
                   configuration.platforms = ["linux", "macos", "windows"];
                   configuration.unit_tests = true;
-                  configuration.layout = "all";
+                  configuration.wpt_layout = "all";
                   return configuration;
                 }
 
@@ -79,20 +79,22 @@ jobs:
                 if (tryString.includes("wpt")) {
                   addPlatformToConfiguration("linux", configuration);
                   if (tryString.includes("2020")) {
-                    configuration.layout = combineWPTLayoutOptions(configuration.layout, "2020");
+                    configuration.wpt_layout = combineWPTLayoutOptions(configuration.layout, "2020");
                   } else {
-                    configuration.layout = combineWPTLayoutOptions(configuration.layout, "2013");
+                    configuration.wpt_layout = combineWPTLayoutOptions(configuration.layout, "2013");
                   }
                 }
             }
 
             let configuration = {
               platforms: [],
-              layout: "none",
+              wpt_layout: "none",
               unit_tests: false,
+              profile: "release",
+              wpt_tests_to_run: "",
             };
 
-            if (github.event_name == 'pull_request_target') {
+            if (context.eventName == 'pull_request_target') {
               for (const label of context.payload.pull_request.labels) {
                 if (!label.name.startsWith("T-")) {
                   continue;