From 8bb739b818dc21a2ef71d56a3b97a040cc44f384 Mon Sep 17 00:00:00 2001 From: Delan Azabani Date: Sun, 8 Sep 2024 23:06:44 +0800 Subject: [PATCH] CI: use self-hosted runners for Linux build jobs (#33321) * CI: use self-hosted runners for Linux build jobs Signed-off-by: Delan Azabani * Set ccache and incremental env variables when not self-hosted Signed-off-by: Delan Azabani * Force GitHub-hosted runner when in upload mode Signed-off-by: Delan Azabani * Revert s/python/python3/ now that our image has python Signed-off-by: Delan Azabani * Remove stray comment in timeout workflow Signed-off-by: Delan Azabani * Update description of runner-select job Signed-off-by: Delan Azabani * Apply suggestions from code review Address couple minor naming / formatting nits Signed-off-by: Martin Robinson --------- Signed-off-by: Delan Azabani Signed-off-by: Martin Robinson Co-authored-by: Martin Robinson --- .github/workflows/linux.yml | 74 ++++++++++--- .../workflows/self-hosted-runner-select.yml | 98 +++++++++++++++++ .../workflows/self-hosted-runner-timeout.yml | 39 +++++++ .github/workflows/windows.yml | 100 ++---------------- 4 files changed, 207 insertions(+), 104 deletions(-) create mode 100644 .github/workflows/self-hosted-runner-select.yml create mode 100644 .github/workflows/self-hosted-runner-timeout.yml diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 6a0002361c8..ae7548e3112 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -59,43 +59,89 @@ on: env: RUST_BACKTRACE: 1 SHELL: /bin/bash - SCCACHE_GHA_ENABLED: "true" - RUSTC_WRAPPER: "sccache" - CCACHE: "sccache" - CARGO_INCREMENTAL: 0 jobs: + # Runs the underlying job (“workload”) on a self-hosted runner if available, + # with the help of a `runner-select` job and a `runner-timeout` job. + runner-select: + uses: ./.github/workflows/self-hosted-runner-select.yml + secrets: inherit + with: + # Ubuntu 22.04 has glibc 2.34 so the binaries produced + # won't run on systems with older glibc e.g wpt.fyi + # runners which still use 20.04. + github-hosted-runner-label: ubuntu-${{ inputs.upload && '20.04' || '22.04' }} + self-hosted-image-name: servo-ubuntu2204 + force-github-hosted-runner: ${{ inputs.upload }} + runner-timeout: + needs: + - runner-select + uses: ./.github/workflows/self-hosted-runner-timeout.yml + secrets: inherit + with: + selected-runner-label: ${{ needs.runner-select.outputs.selected-runner-label }} + is-self-hosted: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} + build: - name: Linux Build - # Ubuntu 22.04 has glibc 2.34 so the binaries produced - # won't run on systems with older glibc e.g wpt.fyi - # runners which still use 20.04. - runs-on: ubuntu-${{ inputs.upload && '20.04' || '22.04' }} + needs: + - runner-select + name: Linux Build [${{ needs.runner-select.outputs.unique-id }}] + runs-on: ${{ needs.runner-select.outputs.selected-runner-label }} steps: + - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} + run: | + echo SCCACHE_GHA_ENABLED=true >> $GITHUB_ENV + echo RUSTC_WRAPPER=sccache >> $GITHUB_ENV + echo CCACHE=sccache >> $GITHUB_ENV + echo CARGO_INCREMENTAL=0 >> $GITHUB_ENV + - uses: actions/checkout@v4 - if: github.event_name != 'pull_request_target' + if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name != 'pull_request_target' }} # This is necessary to checkout the pull request if this run was triggered via a # `pull_request_target` event. - uses: actions/checkout@v4 - if: github.event_name == 'pull_request_target' + if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name == 'pull_request_target' }} with: ref: ${{ github.event.pull_request.head.sha }} + + # Faster checkout for self-hosted runner that uses prebaked repo. + - if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name != 'pull_request_target' }} + run: git fetch --depth=1 origin $env:GITHUB_SHA + - if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name == 'pull_request_target' }} + run: git fetch --depth=1 origin refs/pull/${{ github.event_number }}/head + - if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} + # Same as `git switch --detach FETCH_HEAD`, but fixes up dirty working + # trees, in case the runner image was baked with a dirty working tree. + run: | + git switch --detach + git reset --hard FETCH_HEAD + + # Install missing tools in a GitHub-hosted runner. - name: Run sccache-cache + if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} uses: mozilla-actions/sccache-action@v0.0.4 - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle - if: ${{ !inputs.upload }} # not needed on ubuntu 20.04 used for nightly + if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV - uses: actions/setup-python@v5 + if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} with: python-version: '3.10' - - name: Install crown - run: cargo install --path support/crown - name: Bootstrap Python + if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: python3 -m pip install --upgrade pip - name: Bootstrap dependencies + if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }} run: | sudo apt update python3 ./mach bootstrap --skip-lints + + # Always install crown, even on self-hosted runners, because it is tightly + # coupled to the rustc version, and we may have the wrong version if the + # commit we are building uses a different rustc version. + - name: Install crown + run: cargo install --path support/crown + - name: Build (${{ inputs.profile }}) run: | python3 ./mach build --use-crown --locked --${{ inputs.profile }} diff --git a/.github/workflows/self-hosted-runner-select.yml b/.github/workflows/self-hosted-runner-select.yml new file mode 100644 index 00000000000..bfb6d2ad96c --- /dev/null +++ b/.github/workflows/self-hosted-runner-select.yml @@ -0,0 +1,98 @@ +name: Select Self-hosted Runner +on: + workflow_call: + inputs: + github-hosted-runner-label: + required: true + type: string + self-hosted-image-name: + required: true + type: string + self-hosted-runner-scope: + required: false + type: string + default: /orgs/${{ github.repository_owner }}/actions/runners + monitor-api-base-url: + required: false + type: string + default: https://ci0.servo.org + force-github-hosted-runner: + required: false + type: boolean + default: false + outputs: + unique-id: + value: ${{ jobs.runner-select.outputs.unique-id }} + selected-runner-label: + value: ${{ jobs.runner-select.outputs.selected-runner-label }} + is-self-hosted: + value: ${{ jobs.runner-select.outputs.is-self-hosted }} + +jobs: + # Selects a self-hosted runner if available, or else a GitHub-hosted runner. + # We generate a unique id for the workload, then ask our monitor API to + # reserve a self-hosted runner for us. + runner-select: + name: Select Runner + runs-on: ubuntu-latest + outputs: + unique-id: ${{ steps.select.outputs.unique_id }} + selected-runner-label: ${{ steps.select.outputs.selected_runner_label }} + is-self-hosted: ${{ steps.select.outputs.is_self_hosted }} + steps: + - name: Select and reserve best available runner + id: select + run: | + github_hosted_runner_label='${{ inputs.github-hosted-runner-label }}' + self_hosted_image_name='${{ inputs.self-hosted-image-name }}' + self_hosted_runner_scope='${{ inputs.self-hosted-runner-scope }}' + monitor_api_base_url='${{ inputs.monitor-api-base-url }}' + + set -euo pipefail + + fall_back_to_github_hosted() { + echo 'Falling back to GitHub-hosted runner' + echo "selected_runner_label=$github_hosted_runner_label" | tee -a $GITHUB_OUTPUT + echo 'is_self_hosted=false' | tee -a $GITHUB_OUTPUT + exit 0 + } + + # Generate a unique id that allows the workload job to find the runner + # we are reserving for it (via runner labels), and allows the timeout + # job to find the workload job run (via the job’s friendly name), even + # if there are multiple instances in the workflow call tree. + unique_id=$(uuidgen) + echo "unique_id=$unique_id" | tee -a $GITHUB_OUTPUT + + # Disable self-hosted runners by creating a repository variable named + # NO_SELF_HOSTED_RUNNERS with any non-empty value. + # + if [ -n '${{ vars.NO_SELF_HOSTED_RUNNERS }}' ]; then + echo 'NO_SELF_HOSTED_RUNNERS is set!' + fall_back_to_github_hosted + fi + + if [ '${{ inputs.force-github-hosted-runner }}' = true ]; then + echo 'inputs.force-github-hosted-runner is set!' + fall_back_to_github_hosted + fi + + # Use the monitor API to reserve a runner. If we get an object with + # runner details, we succeeded. If we get null, we failed. + take_runner_url=$monitor_api_base_url/$self_hosted_image_name/$unique_id/${{ github.repository }}/${{ github.run_id }} + result=$(mktemp) + echo + echo POST "$take_runner_url" + if ! curl -sS --fail-with-body --connect-timeout 5 --max-time 30 -X POST "$take_runner_url" \ + -H 'Authorization: Bearer ${{ secrets.SERVO_CI_MONITOR_API_TOKEN }}' > $result \ + || ! jq -e . $result > /dev/null; then + cat $result + echo + echo + echo 'No self-hosted runners available!' + fall_back_to_github_hosted + fi + + echo + echo "selected_runner_label=reserved-for:$unique_id" | tee -a $GITHUB_OUTPUT + echo 'is_self_hosted=true' | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/self-hosted-runner-timeout.yml b/.github/workflows/self-hosted-runner-timeout.yml new file mode 100644 index 00000000000..24c4009ab2d --- /dev/null +++ b/.github/workflows/self-hosted-runner-timeout.yml @@ -0,0 +1,39 @@ +name: Detect Self-hosted Runner Timeout +on: + workflow_call: + inputs: + selected-runner-label: + required: true + type: string + is-self-hosted: + required: true + type: boolean + +jobs: + # In the unlikely event a self-hosted runner was selected and reserved but it + # goes down before the workload starts, cancel the workflow run. + runner-timeout: + if: ${{ inputs.is-self-hosted }} + name: Detect Runner Timeout + runs-on: ubuntu-latest + steps: + - name: Wait a bit + run: sleep 30 + + - name: Cancel if workload job is still queued + run: | + run_url=/repos/${{ github.repository }}/actions/runs/${{ github.run_id }} + export GH_TOKEN=${{ secrets.GITHUB_TOKEN }} + + if [ "$(gh api "$run_url/jobs" \ + | jq -er --arg id '${{ inputs.unique-id }}' \ + '.jobs[] | select(.name | contains("[" + $id + "]")) | .status' + )" = queued ]; then + echo 'Timeout waiting for runner assignment!' + echo 'Hint: does this repo have permission to access the runner group?' + echo 'Hint: https://github.com/organizations/servo/settings/actions/runner-groups' + echo + echo 'Cancelling workflow run' + gh api "$run_url/cancel" --method POST + exit 1 + fi diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index b7d643e5769..e2d706d514f 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -46,102 +46,22 @@ env: RUSTUP_WINDOWS_PATH_ADD_BIN: 1 jobs: - # Automatic runner selection for job: build # Runs the underlying job (“workload”) on a self-hosted runner if available, # with the help of a `runner-select` job and a `runner-timeout` job. - - # Selects a self-hosted runner if available, or else a GitHub-hosted runner. - # We generate a unique id for the workload, find an idle self-hosted runner - # with the given `image:` label that wasn’t already reserved by another - # `runner-select` job run, and reserve it with a `reserved-for:` label. runner-select: - name: Select Runner - runs-on: ubuntu-latest - outputs: - unique-id: ${{ steps.select.outputs.unique_id }} - selected-runner-label: ${{ steps.select.outputs.selected_runner_label }} - is-self-hosted: ${{ steps.select.outputs.is_self_hosted }} - steps: - - name: Select and reserve best available runner - id: select - # Set the variables below to your desired runner images, runner scope - # (org or repo), and monitor API URL. - run: | - github_hosted_runner_label=windows-2022 - self_hosted_image_name=servo-windows10 - self_hosted_runner_scope=/orgs/${{ github.repository_owner }}/actions/runners - monitor_api_base_url=https://ci0.servo.org - - set -euo pipefail - - fall_back_to_github_hosted() { - echo 'Falling back to GitHub-hosted runner' - echo "selected_runner_label=$github_hosted_runner_label" | tee -a $GITHUB_OUTPUT - echo 'is_self_hosted=false' | tee -a $GITHUB_OUTPUT - exit 0 - } - - # Generate a unique id that allows the workload job to find the runner - # we are reserving for it (via runner labels), and allows the timeout - # job to find the workload job run (via the job’s friendly name), even - # if there are multiple instances in the workflow call tree. - unique_id=$(uuidgen) - echo "unique_id=$unique_id" | tee -a $GITHUB_OUTPUT - - # Disable self-hosted runners by creating a repository variable named - # NO_SELF_HOSTED_RUNNERS with any non-empty value. - # - if [ -n '${{ vars.NO_SELF_HOSTED_RUNNERS }}' ]; then - echo 'NO_SELF_HOSTED_RUNNERS is set!' - fall_back_to_github_hosted - fi - - # Use the monitor API to reserve a runner. If we get an object with - # runner details, we succeeded. If we get null, we failed. - take_runner_url=$monitor_api_base_url/$self_hosted_image_name/$unique_id/${{ github.repository }}/${{ github.run_id }} - echo - echo POST "$take_runner_url" - if ! curl -fsS --max-time 10 -X POST "$take_runner_url" \ - -H 'Authorization: Bearer ${{ secrets.SERVO_CI_MONITOR_API_TOKEN }}' \ - | jq -e; then - echo - echo 'No self-hosted runners available!' - fall_back_to_github_hosted - fi - - echo - echo "selected_runner_label=reserved-for:$unique_id" | tee -a $GITHUB_OUTPUT - echo 'is_self_hosted=true' | tee -a $GITHUB_OUTPUT - - # In the unlikely event a self-hosted runner was selected and reserved but it - # goes down before the workload starts, cancel the workflow run. + uses: ./.github/workflows/self-hosted-runner-select.yml + secrets: inherit + with: + github-hosted-runner-label: windows-2022 + self-hosted-image-name: servo-windows10 runner-timeout: needs: - runner-select - if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} - name: Detect Runner Timeout - runs-on: ubuntu-latest - steps: - - name: Wait a bit - run: sleep 30 - - - name: Cancel if workload job is still queued - run: | - run_url=/repos/${{ github.repository }}/actions/runs/${{ github.run_id }} - export GH_TOKEN=${{ secrets.GITHUB_TOKEN }} - - if [ "$(gh api "$run_url/jobs" \ - | jq -er --arg id '${{ needs.runner-select.outputs.unique-id }}' \ - '.jobs[] | select(.name | contains("[" + $id + "]")) | .status' - )" = queued ]; then - echo 'Timeout waiting for runner assignment!' - echo 'Hint: does this repo have permission to access the runner group?' - echo 'Hint: https://github.com/organizations/servo/settings/actions/runner-groups' - echo - echo 'Cancelling workflow run' - gh api "$run_url/cancel" --method POST - exit 1 - fi + uses: ./.github/workflows/self-hosted-runner-timeout.yml + secrets: inherit + with: + selected-runner-label: ${{ needs.runner-select.outputs.selected-runner-label }} + is-self-hosted: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }} build: needs: