Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-06 16:45:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

Customizable wpt options and profile in workflows (#30912)

Browse source 
* wpt -> wpt-sync

* Add profile configuration option

* Add wpt (for custom tests selection) to CI

* Renaming of wpt-* options

* fixup! do renames also in scripts

* fixup! profile

* Fix try labels
This commit is contained in:
Samson 2023-12-22 13:49:39 +01:00 • committed by GitHub
parent 81f5157522
commit 1f0f50b22b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 105 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

42
.github/workflows/linux-wpt.yml vendored
View file

@ -5,21 +5,25 @@ on:
profile: profile:
required: true required: true
type: string type: string
wpt: wpt-tests-to-run:
default: ""
required: false required: false
type: string type: string
layout: wpt-sync-from-upstream:
required: false
type: boolean
wpt-layout:
required: true required: true
type: string type: string
env: env:
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
SHELL: /bin/bash SHELL: /bin/bash
WPT_COMMAND_LINE_ARG: "${{ inputs.layout == 'layout-2013' && '--legacy-layout' || '' }}" WPT_COMMAND_LINE_ARG: "${{ inputs.wpt-layout == 'layout-2013' && '--legacy-layout' || '' }}"
jobs: jobs:
linux-wpt: linux-wpt:
name: WPT ${{ inputs.layout }} name: WPT ${{ inputs.wpt-layout }}
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
env: env:
max_chunk_id: 20 max_chunk_id: 20
@ -63,13 +67,13 @@ jobs:
sudo apt install ./libffi6_3.2.1-8_amd64.deb sudo apt install ./libffi6_3.2.1-8_amd64.deb
python3 ./mach bootstrap-gstreamer python3 ./mach bootstrap-gstreamer
- name: Sync from upstream WPT - name: Sync from upstream WPT
if: ${{ inputs.wpt == 'sync' }} if: ${{ inputs.wpt-sync-from-upstream == 'true' }}
run: | run: |
./mach update-wpt --sync --patch ./mach update-wpt --sync --patch
- name: Run tests - name: Run tests
if: ${{ inputs.wpt != 'sync' }} if: ${{ inputs.wpt-sync-from-upstream != 'true' }}
run: | run: |
python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG ${{ inputs.wpt-tests-to-run }} \
--${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \ --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \
--total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
--log-raw test-wpt.${{ matrix.chunk_id }}.log \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \
@ -79,7 +83,7 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_CONTEXT: ${{ toJson(github) }}
INTERMITTENT_TRACKER_DASHBOARD_SECRET: ${{ secrets.INTERMITTENT_TRACKER_DASHBOARD_SECRET }} INTERMITTENT_TRACKER_DASHBOARD_SECRET: ${{ secrets.INTERMITTENT_TRACKER_DASHBOARD_SECRET }}
- name: Run tests (sync) - name: Run tests (sync)
if: ${{ inputs.wpt == 'sync' }} if: ${{ inputs.wpt-sync-from-upstream == 'true' }}
run: | run: |
python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \
--${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \ --${{ inputs.profile }} --processes $(nproc) --timeout-multiplier 2 \
@ -88,24 +92,24 @@ jobs:
--always-succeed --always-succeed
- name: Archive filtered results - name: Archive filtered results
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
if: ${{ always() && inputs.wpt != 'sync' }} if: ${{ always() && inputs.wpt-sync-from-upstream != 'true' }}
with: with:
name: wpt-filtered-results-linux-${{ inputs.layout }} name: wpt-filtered-results-linux-${{ inputs.wpt-layout }}
path: | path: |
filtered-test-wpt.${{ matrix.chunk_id }}.json filtered-test-wpt.${{ matrix.chunk_id }}.json
unexpected-test-wpt.${{ matrix.chunk_id }}.log unexpected-test-wpt.${{ matrix.chunk_id }}.log
- name: Archive logs - name: Archive logs
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
if: ${{ failure() && inputs.wpt != 'sync' }} if: ${{ failure() && inputs.wpt-sync-from-upstream != 'true' }}
with: with:
name: wpt-logs-linux-${{ inputs.layout }} name: wpt-logs-linux-${{ inputs.wpt-layout }}
path: | path: |
test-wpt.${{ matrix.chunk_id }}.log test-wpt.${{ matrix.chunk_id }}.log
- name: Archive logs - name: Archive logs
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
if: ${{ inputs.wpt == 'sync' }} if: ${{ inputs.wpt-sync-from-upstream == 'true' }}
with: with:
name: wpt-logs-linux-${{ inputs.layout }} name: wpt-logs-linux-${{ inputs.wpt-layout }}
path: | path: |
test-wpt.${{ matrix.chunk_id }}.log test-wpt.${{ matrix.chunk_id }}.log
wpt-jsonsummary.${{ matrix.chunk_id }}.log wpt-jsonsummary.${{ matrix.chunk_id }}.log
@ -113,7 +117,7 @@ jobs:
report-test-results: report-test-results:
name: Report WPT Results name: Report WPT Results
runs-on: ubuntu-latest runs-on: ubuntu-latest
if: ${{ always() && !cancelled() && inputs.wpt == 'test' }} if: ${{ always() && !cancelled() && inputs.wpt-sync-from-upstream != 'true' }}
needs: needs:
- "linux-wpt" - "linux-wpt"
steps: steps:
@ -122,20 +126,20 @@ jobs:
fetch-depth: 2 fetch-depth: 2
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: wpt-filtered-results-linux-${{ inputs.layout }} name: wpt-filtered-results-linux-${{ inputs.wpt-layout }}
path: wpt-filtered-results-linux path: wpt-filtered-results-linux
- name: Create aggregated unexpected results - name: Create aggregated unexpected results
run: | run: |
cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.layout }}.log cat wpt-filtered-results-linux/*.log > unexpected-test-wpt-${{ inputs.wpt-layout }}.log
- name: Archive aggregate results - name: Archive aggregate results
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: wpt-filtered-results-linux name: wpt-filtered-results-linux
path: | path: |
unexpected-test-wpt-${{ inputs.layout }}.log unexpected-test-wpt-${{ inputs.wpt-layout }}.log
- name: Comment on PR with results - name: Comment on PR with results
run: | run: |
etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.layout }}" wpt-filtered-results-linux/*.json etc/ci/report_aggregated_expected_results.py --tag="linux-wpt-${{ inputs.wpt-layout }}" wpt-filtered-results-linux/*.json
env: env:
GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_CONTEXT: ${{ toJson(github) }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

36
.github/workflows/linux.yml vendored
View file

@ -6,10 +6,15 @@ on:
required: false required: false
default: "release" default: "release"
type: string type: string
wpt: wpt-tests-to-run:
default: ""
required: false required: false
type: string type: string
layout: wpt-sync-from-upstream:
required: false
default: false
type: boolean
wpt-layout:
required: false required: false
type: string type: string
unit-tests: unit-tests:
@ -30,12 +35,15 @@ on:
default: "release" default: "release"
type: choice type: choice
options: ["release", "debug", "production"] options: ["release", "debug", "production"]
wpt: wpt-tests-to-run:
default: "test" default: ""
required: false required: false
type: choice type: string
options: ["test", "sync"] wpt-sync-from-upstream:
layout: default: false
required: false
type: boolean
wpt-layout:
required: false required: false
type: choice type: choice
options: ["none", "2013", "2020", "all"] options: ["none", "2013", "2020", "all"]
@ -140,25 +148,27 @@ jobs:
path: target.tar.gz path: target.tar.gz
wpt-2020: wpt-2020:
if: ${{ inputs.layout == '2020' || inputs.layout == 'all' }} if: ${{ inputs.wpt-layout == '2020' || inputs.wpt-layout == 'all' }}
name: Linux WPT Tests 2020 name: Linux WPT Tests 2020
needs: ["build"] needs: ["build"]
uses: ./.github/workflows/linux-wpt.yml uses: ./.github/workflows/linux-wpt.yml
with: with:
wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
profile: ${{ inputs.profile }} profile: ${{ inputs.profile }}
wpt: ${{ inputs.wpt }} wpt-sync-from-upstream: ${{ inputs.wpt-sync-from-upstream }}
layout: "layout-2020" wpt-layout: "layout-2020"
secrets: inherit secrets: inherit
wpt-2013: wpt-2013:
if: ${{ inputs.layout == '2013' || inputs.layout == 'all' }} if: ${{ inputs.wpt-layout == '2013' || inputs.wpt-layout == 'all' }}
name: Linux WPT Tests 2013 name: Linux WPT Tests 2013
needs: ["build"] needs: ["build"]
uses: ./.github/workflows/linux-wpt.yml uses: ./.github/workflows/linux-wpt.yml
with: with:
wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
profile: ${{ inputs.profile }} profile: ${{ inputs.profile }}
wpt: ${{ inputs.wpt }} wpt-sync-from-upstream: ${{ inputs.wpt-sync-from-upstream }}
layout: "layout-2013" wpt-layout: "layout-2013"
secrets: inherit secrets: inherit
result: result:

22
.github/workflows/mac-wpt.yml vendored
View file

@ -3,21 +3,25 @@ name: Mac WPT Tests
on: on:
workflow_call: workflow_call:
inputs: inputs:
wpt-tests-to-run:
default: ""
required: false
type: string
profile: profile:
required: true required: true
type: string type: string
layout: wpt-layout:
required: true required: true
type: string type: string
env: env:
RUST_BACKTRACE: 1 RUST_BACKTRACE: 1
SHELL: /bin/bash SHELL: /bin/bash
WPT_COMMAND_LINE_ARG: "${{ inputs.layout == 'layout-2013' && '--legacy-layout' || '' }}" WPT_COMMAND_LINE_ARG: "${{ inputs.wpt-layout == 'layout-2013' && '--legacy-layout' || '' }}"
jobs: jobs:
mac-wpt: mac-wpt:
name: WPT ${{ inputs.layout }} name: WPT ${{ inputs.wpt-layout }}
runs-on: macos-13 runs-on: macos-13
env: env:
max_chunk_id: 5 max_chunk_id: 5
@ -49,7 +53,7 @@ jobs:
run: python3 ./mach smoketest --${{ inputs.profile }} run: python3 ./mach smoketest --${{ inputs.profile }}
- name: Run tests - name: Run tests
run: | run: |
python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG \ python3 ./mach test-wpt $WPT_COMMAND_LINE_ARG ${{ inputs.wpt-tests-to-run }} \
--${{ inputs.profile }} --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \ --${{ inputs.profile }} --processes $(sysctl -n hw.logicalcpu) --timeout-multiplier 8 \
--total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
--log-raw test-wpt.${{ matrix.chunk_id }}.log \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \
@ -59,7 +63,7 @@ jobs:
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
if: always() if: always()
with: with:
name: wpt-filtered-results-mac-${{ inputs.layout }} name: wpt-filtered-results-mac-${{ inputs.wpt-layout }}
path: | path: |
filtered-test-wpt.${{ matrix.chunk_id }}.json filtered-test-wpt.${{ matrix.chunk_id }}.json
unexpected-test-wpt.${{ matrix.chunk_id }}.log unexpected-test-wpt.${{ matrix.chunk_id }}.log
@ -67,7 +71,7 @@ jobs:
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
if: failure() if: failure()
with: with:
name: wpt-logs-mac-${{ inputs.layout }} name: wpt-logs-mac-${{ inputs.wpt-layout }}
path: | path: |
test-wpt.${{ matrix.chunk_id }}.log test-wpt.${{ matrix.chunk_id }}.log
filtered-wpt-results.${{ matrix.chunk_id }}.json filtered-wpt-results.${{ matrix.chunk_id }}.json
@ -83,18 +87,18 @@ jobs:
fetch-depth: 2 fetch-depth: 2
- uses: actions/download-artifact@v3 - uses: actions/download-artifact@v3
with: with:
name: wpt-filtered-results-mac-${{ inputs.layout }} name: wpt-filtered-results-mac-${{ inputs.wpt-layout }}
path: wpt-filtered-results-mac path: wpt-filtered-results-mac
- name: Create aggregated unexpected results - name: Create aggregated unexpected results
run: cat wpt-filtered-results-mac/*.log > unexpected-test-wpt.log run: cat wpt-filtered-results-mac/*.log > unexpected-test-wpt.log
- name: Archive aggregate results - name: Archive aggregate results
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
name: wpt-filtered-results-mac-${{ inputs.layout }} name: wpt-filtered-results-mac-${{ inputs.wpt-layout }}
path: | path: |
unexpected-test-wpt.log unexpected-test-wpt.log
- name: Comment on PR with results - name: Comment on PR with results
run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.layout }}" run: etc/ci/report_aggregated_expected_results.py --tag="mac-wpt-${{ inputs.wpt-layout }}"
wpt-filtered-results-mac/*.json wpt-filtered-results-mac/*.json
env: env:
GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_CONTEXT: ${{ toJson(github) }}

14
.github/workflows/mac.yml vendored
View file

@ -7,6 +7,10 @@ on:
required: false required: false
default: "release" default: "release"
type: string type: string
wpt-tests-to-run:
default: ""
required: false
type: string
wpt-layout: wpt-layout:
required: false required: false
type: string type: string
@ -28,6 +32,10 @@ on:
default: "release" default: "release"
type: choice type: choice
options: ["release", "debug", "production"] options: ["release", "debug", "production"]
wpt-tests-to-run:
default: ""
required: false
type: string
wpt-layout: wpt-layout:
required: false required: false
type: choice type: choice
@ -147,7 +155,8 @@ jobs:
uses: ./.github/workflows/mac-wpt.yml uses: ./.github/workflows/mac-wpt.yml
with: with:
profile: ${{ inputs.profile }} profile: ${{ inputs.profile }}
layout: "layout-2020" wpt-layout: "layout-2020"
wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
secrets: inherit secrets: inherit
wpt-2013: wpt-2013:
@ -157,7 +166,8 @@ jobs:
uses: ./.github/workflows/mac-wpt.yml uses: ./.github/workflows/mac-wpt.yml
with: with:
profile: ${{ inputs.profile }} profile: ${{ inputs.profile }}
layout: "layout-2013" wpt-layout: "layout-2013"
wpt-tests-to-run: ${{ inputs.wpt-tests-to-run }}
secrets: inherit secrets: inherit
result: result:

34
.github/workflows/main.yml vendored
View file

@ -18,11 +18,20 @@ on:
type: string type: string
workflow_dispatch: workflow_dispatch:
inputs: inputs:
profile:
required: false
default: "release"
type: choice
options: ["release", "debug", "production"]
wpt-tests-to-run:
default: ""
required: false
type: string
platform: platform:
required: false required: false
type: choice type: choice
options: ["linux", "windows", "macos", "all"] options: ["linux", "windows", "macos", "all"]
layout: wpt-layout:
required: false required: false
type: choice type: choice
options: ["none", "2013", "2020", "all"] options: ["none", "2013", "2020", "all"]
@ -53,13 +62,15 @@ jobs:
// We need to pick defaults if the inputs are not provided. Unprovided inputs // We need to pick defaults if the inputs are not provided. Unprovided inputs
// are empty strings in this template. // are empty strings in this template.
let platform = "${{ inputs.platform }}" || "linux"; let platform = "${{ inputs.platform }}" || "linux";
let layout = "${{ inputs.layout }}" || "none"; let profile = "${{ inputs.profile }}" || "release";
let wpt_layout = "${{ inputs.wpt-layout }}" || "none";
let wpt_tests_to_run = "${{ inputs.wpt-tests-to-run }}" || "";
let unit_tests = Boolean(${{ inputs.unit-tests }}) let unit_tests = Boolean(${{ inputs.unit-tests }})
// Merge queue runs and pushes to `main` should always trigger a full build and test. // Merge queue runs and pushes to `main` should always trigger a full build and test.
if (["push", "merge_group"].includes(context.eventName)) { if (["push", "merge_group"].includes(context.eventName)) {
platform = "all"; platform = "all";
layout = "all"; wpt_layout = "all";
unit_tests = true; unit_tests = true;
} }
@ -70,10 +81,12 @@ jobs:
platforms = [ platform ]; platforms = [ platform ];
} }
let returnValue = { let returnValue = {
platforms, platforms,
layout, wpt_layout,
unit_tests, unit_tests,
profile,
wpt_tests_to_run,
}; };
console.log("Using configuration: " + JSON.stringify(returnValue)); console.log("Using configuration: " + JSON.stringify(returnValue));
@ -85,7 +98,7 @@ jobs:
if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'windows') }} if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'windows') }}
uses: ./.github/workflows/windows.yml uses: ./.github/workflows/windows.yml
with: with:
profile: "release" profile: ${{ fromJson(needs.decision.outputs.configuration).profile }}
unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
secrets: inherit secrets: inherit
@ -95,7 +108,8 @@ jobs:
if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'macos') }} if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'macos') }}
uses: ./.github/workflows/mac.yml uses: ./.github/workflows/mac.yml
with: with:
profile: "release" wpt-tests-to-run: ${{ fromJson(needs.decision.outputs.configuration).wpt_tests_to_run }}
profile: ${{ fromJson(needs.decision.outputs.configuration).profile }}
unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
secrets: inherit secrets: inherit
@ -105,9 +119,9 @@ jobs:
if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'linux') }} if: ${{ contains(fromJson(needs.decision.outputs.configuration).platforms, 'linux') }}
uses: ./.github/workflows/linux.yml uses: ./.github/workflows/linux.yml
with: with:
profile: "release" wpt-tests-to-run: ${{ fromJson(needs.decision.outputs.configuration).wpt_tests_to_run }}
wpt: 'test' profile: ${{ fromJson(needs.decision.outputs.configuration).profile }}
layout: ${{ fromJson(needs.decision.outputs.configuration).layout }} wpt-layout: ${{ fromJson(needs.decision.outputs.configuration).wpt_layout }}
unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }} unit-tests: ${{ fromJson(needs.decision.outputs.configuration).unit_tests }}
secrets: inherit secrets: inherit

4
.github/workflows/scheduled-wpt-import.yml vendored
View file

@ -15,8 +15,8 @@ jobs:
name: Linux name: Linux
uses: ./.github/workflows/linux.yml uses: ./.github/workflows/linux.yml
with: with:
wpt: 'sync' wpt-sync-from-upstream: true
layout: 'all' wpt-layout: 'all'
unit-tests: false unit-tests: false
sync: sync:

14
.github/workflows/try.yml vendored
View file

@ -22,7 +22,7 @@ jobs:
function makeComment(body) { function makeComment(body) {
console.log(body); console.log(body);
if (github.event_name != 'pull_request_target') if (context.eventName != 'pull_request_target')
return; return;
github.rest.issues.createComment({ github.rest.issues.createComment({
@ -61,7 +61,7 @@ jobs:
if (tryString.includes("full")) { if (tryString.includes("full")) {
configuration.platforms = ["linux", "macos", "windows"]; configuration.platforms = ["linux", "macos", "windows"];
configuration.unit_tests = true; configuration.unit_tests = true;
configuration.layout = "all"; configuration.wpt_layout = "all";
return configuration; return configuration;
} }
@ -79,20 +79,22 @@ jobs:
if (tryString.includes("wpt")) { if (tryString.includes("wpt")) {
addPlatformToConfiguration("linux", configuration); addPlatformToConfiguration("linux", configuration);
if (tryString.includes("2020")) { if (tryString.includes("2020")) {
configuration.layout = combineWPTLayoutOptions(configuration.layout, "2020"); configuration.wpt_layout = combineWPTLayoutOptions(configuration.layout, "2020");
} else { } else {
configuration.layout = combineWPTLayoutOptions(configuration.layout, "2013"); configuration.wpt_layout = combineWPTLayoutOptions(configuration.layout, "2013");
} }
} }
} }
let configuration = { let configuration = {
platforms: [], platforms: [],
layout: "none", wpt_layout: "none",
unit_tests: false, unit_tests: false,
profile: "release",
wpt_tests_to_run: "",
}; };
if (github.event_name == 'pull_request_target') { if (context.eventName == 'pull_request_target') {
for (const label of context.payload.pull_request.labels) { for (const label of context.payload.pull_request.labels) {
if (!label.name.startsWith("T-")) { if (!label.name.startsWith("T-")) {
continue; continue;