CI: use self-hosted runners for Linux build jobs (#33321)

* CI: use self-hosted runners for Linux build jobs

Signed-off-by: Delan Azabani <dazabani@igalia.com>

* Set ccache and incremental env variables when not self-hosted

Signed-off-by: Delan Azabani <dazabani@igalia.com>

* Force GitHub-hosted runner when in upload mode

Signed-off-by: Delan Azabani <dazabani@igalia.com>

* Revert s/python/python3/ now that our image has python

Signed-off-by: Delan Azabani <dazabani@igalia.com>

* Remove stray comment in timeout workflow

Signed-off-by: Delan Azabani <dazabani@igalia.com>

* Update description of runner-select job

Signed-off-by: Delan Azabani <dazabani@igalia.com>

* Apply suggestions from code review

Address couple minor naming / formatting nits

Signed-off-by: Martin Robinson <mrobinson@igalia.com>

---------

Signed-off-by: Delan Azabani <dazabani@igalia.com>
Signed-off-by: Martin Robinson <mrobinson@igalia.com>
Co-authored-by: Martin Robinson <mrobinson@igalia.com>

.github/workflows/linux.yml

@@ -59,43 +59,89 @@ on:
 env:
   RUST_BACKTRACE: 1
   SHELL: /bin/bash
-  SCCACHE_GHA_ENABLED: "true"
-  RUSTC_WRAPPER: "sccache"
-  CCACHE: "sccache"
-  CARGO_INCREMENTAL: 0
 
 jobs:
-  build:
+  # Runs the underlying job (“workload”) on a self-hosted runner if available,
-    name: Linux Build
+  # with the help of a `runner-select` job and a `runner-timeout` job.
+  runner-select:
+    uses: ./.github/workflows/self-hosted-runner-select.yml
+    secrets: inherit
+    with:
       # Ubuntu 22.04 has glibc 2.34 so the binaries produced
       # won't run on systems with older glibc e.g wpt.fyi
       # runners which still use 20.04.
-    runs-on: ubuntu-${{ inputs.upload && '20.04' || '22.04' }}
+      github-hosted-runner-label: ubuntu-${{ inputs.upload && '20.04' || '22.04' }}
+      self-hosted-image-name: servo-ubuntu2204
+      force-github-hosted-runner: ${{ inputs.upload }}
+  runner-timeout:
+    needs:
+      - runner-select
+    uses: ./.github/workflows/self-hosted-runner-timeout.yml
+    secrets: inherit
+    with:
+      selected-runner-label: ${{ needs.runner-select.outputs.selected-runner-label }}
+      is-self-hosted: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
+
+  build:
+    needs:
+      - runner-select
+    name: Linux Build [${{ needs.runner-select.outputs.unique-id }}]
+    runs-on: ${{ needs.runner-select.outputs.selected-runner-label }}
     steps:
+      - if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
+        run: |
+          echo SCCACHE_GHA_ENABLED=true >> $GITHUB_ENV
+          echo RUSTC_WRAPPER=sccache >> $GITHUB_ENV
+          echo CCACHE=sccache >> $GITHUB_ENV
+          echo CARGO_INCREMENTAL=0 >> $GITHUB_ENV
+
       - uses: actions/checkout@v4
-        if: github.event_name != 'pull_request_target'
+        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name != 'pull_request_target' }}
       # This is necessary to checkout the pull request if this run was triggered via a
       # `pull_request_target` event.
       - uses: actions/checkout@v4
-        if: github.event_name == 'pull_request_target'
+        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name == 'pull_request_target' }}
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+
+      # Faster checkout for self-hosted runner that uses prebaked repo.
+      - if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name != 'pull_request_target' }}
+        run: git fetch --depth=1 origin $env:GITHUB_SHA
+      - if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) && github.event_name == 'pull_request_target' }}
+        run: git fetch --depth=1 origin refs/pull/${{ github.event_number }}/head
+      - if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
+        # Same as `git switch --detach FETCH_HEAD`, but fixes up dirty working
+        # trees, in case the runner image was baked with a dirty working tree.
+        run: |
+          git switch --detach
+          git reset --hard FETCH_HEAD
+
+      # Install missing tools in a GitHub-hosted runner.
       - name: Run sccache-cache
+        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
         uses: mozilla-actions/sccache-action@v0.0.4
       - name: Set LIBCLANG_PATH env # needed for bindgen in mozangle
-        if: ${{ !inputs.upload  }} # not needed on ubuntu 20.04 used for nightly
+        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) && !inputs.upload }} # not needed on ubuntu 20.04 used for nightly
         run: echo "LIBCLANG_PATH=/usr/lib/llvm-14/lib" >> $GITHUB_ENV
       - uses: actions/setup-python@v5
+        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
         with:
           python-version: '3.10'
-      - name: Install crown
-        run: cargo install --path support/crown
       - name: Bootstrap Python
+        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
         run: python3 -m pip install --upgrade pip
       - name: Bootstrap dependencies
+        if: ${{ ! fromJSON(needs.runner-select.outputs.is-self-hosted) }}
         run: |
           sudo apt update
           python3 ./mach bootstrap --skip-lints
+
+      # Always install crown, even on self-hosted runners, because it is tightly
+      # coupled to the rustc version, and we may have the wrong version if the
+      # commit we are building uses a different rustc version.
+      - name: Install crown
+        run: cargo install --path support/crown
+
       - name: Build (${{ inputs.profile }})
         run: |
           python3 ./mach build --use-crown --locked --${{ inputs.profile }}

.github/workflows/self-hosted-runner-select.yml

@@ -0,0 +1,98 @@
+name: Select Self-hosted Runner
+on:
+  workflow_call:
+    inputs:
+      github-hosted-runner-label:
+        required: true
+        type: string
+      self-hosted-image-name:
+        required: true
+        type: string
+      self-hosted-runner-scope:
+        required: false
+        type: string
+        default: /orgs/${{ github.repository_owner }}/actions/runners
+      monitor-api-base-url:
+        required: false
+        type: string
+        default: https://ci0.servo.org
+      force-github-hosted-runner:
+        required: false
+        type: boolean
+        default: false
+    outputs:
+      unique-id:
+        value: ${{ jobs.runner-select.outputs.unique-id }}
+      selected-runner-label:
+        value: ${{ jobs.runner-select.outputs.selected-runner-label }}
+      is-self-hosted:
+        value: ${{ jobs.runner-select.outputs.is-self-hosted }}
+
+jobs:
+  # Selects a self-hosted runner if available, or else a GitHub-hosted runner.
+  # We generate a unique id for the workload, then ask our monitor API to
+  # reserve a self-hosted runner for us.
+  runner-select:
+    name: Select Runner
+    runs-on: ubuntu-latest
+    outputs:
+      unique-id: ${{ steps.select.outputs.unique_id }}
+      selected-runner-label: ${{ steps.select.outputs.selected_runner_label }}
+      is-self-hosted: ${{ steps.select.outputs.is_self_hosted }}
+    steps:
+      - name: Select and reserve best available runner
+        id: select
+        run: |
+          github_hosted_runner_label='${{ inputs.github-hosted-runner-label }}'
+          self_hosted_image_name='${{ inputs.self-hosted-image-name }}'
+          self_hosted_runner_scope='${{ inputs.self-hosted-runner-scope }}'
+          monitor_api_base_url='${{ inputs.monitor-api-base-url }}'
+
+          set -euo pipefail
+
+          fall_back_to_github_hosted() {
+            echo 'Falling back to GitHub-hosted runner'
+            echo "selected_runner_label=$github_hosted_runner_label" | tee -a $GITHUB_OUTPUT
+            echo 'is_self_hosted=false' | tee -a $GITHUB_OUTPUT
+            exit 0
+          }
+
+          # Generate a unique id that allows the workload job to find the runner
+          # we are reserving for it (via runner labels), and allows the timeout
+          # job to find the workload job run (via the job’s friendly name), even
+          # if there are multiple instances in the workflow call tree.
+          unique_id=$(uuidgen)
+          echo "unique_id=$unique_id" | tee -a $GITHUB_OUTPUT
+
+          # Disable self-hosted runners by creating a repository variable named
+          # NO_SELF_HOSTED_RUNNERS with any non-empty value.
+          # <https://github.com/servo/servo/settings/variables/actions>
+          if [ -n '${{ vars.NO_SELF_HOSTED_RUNNERS }}' ]; then
+            echo 'NO_SELF_HOSTED_RUNNERS is set!'
+            fall_back_to_github_hosted
+          fi
+
+          if [ '${{ inputs.force-github-hosted-runner }}' = true ]; then
+            echo 'inputs.force-github-hosted-runner is set!'
+            fall_back_to_github_hosted
+          fi
+
+          # Use the monitor API to reserve a runner. If we get an object with
+          # runner details, we succeeded. If we get null, we failed.
+          take_runner_url=$monitor_api_base_url/$self_hosted_image_name/$unique_id/${{ github.repository }}/${{ github.run_id }}
+          result=$(mktemp)
+          echo
+          echo POST "$take_runner_url"
+          if ! curl -sS --fail-with-body --connect-timeout 5 --max-time 30 -X POST "$take_runner_url" \
+              -H 'Authorization: Bearer ${{ secrets.SERVO_CI_MONITOR_API_TOKEN }}' > $result \
+              || ! jq -e . $result > /dev/null; then
+            cat $result
+            echo
+            echo
+            echo 'No self-hosted runners available!'
+            fall_back_to_github_hosted
+          fi
+
+          echo
+          echo "selected_runner_label=reserved-for:$unique_id" | tee -a $GITHUB_OUTPUT
+          echo 'is_self_hosted=true' | tee -a $GITHUB_OUTPUT

.github/workflows/self-hosted-runner-timeout.yml

@@ -0,0 +1,39 @@
+name: Detect Self-hosted Runner Timeout
+on:
+  workflow_call:
+    inputs:
+      selected-runner-label:
+        required: true
+        type: string
+      is-self-hosted:
+        required: true
+        type: boolean
+
+jobs:
+  # In the unlikely event a self-hosted runner was selected and reserved but it
+  # goes down before the workload starts, cancel the workflow run.
+  runner-timeout:
+    if: ${{ inputs.is-self-hosted }}
+    name: Detect Runner Timeout
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait a bit
+        run: sleep 30
+
+      - name: Cancel if workload job is still queued
+        run: |
+          run_url=/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          export GH_TOKEN=${{ secrets.GITHUB_TOKEN }}
+
+          if [ "$(gh api "$run_url/jobs" \
+            | jq -er --arg id '${{ inputs.unique-id }}' \
+              '.jobs[] | select(.name | contains("[" + $id + "]")) | .status'
+          )" = queued ]; then
+            echo 'Timeout waiting for runner assignment!'
+            echo 'Hint: does this repo have permission to access the runner group?'
+            echo 'Hint: https://github.com/organizations/servo/settings/actions/runner-groups'
+            echo
+            echo 'Cancelling workflow run'
+            gh api "$run_url/cancel" --method POST
+            exit 1
+          fi

.github/workflows/windows.yml

@@ -46,102 +46,22 @@ env:
   RUSTUP_WINDOWS_PATH_ADD_BIN: 1
 
 jobs:
-  # Automatic runner selection for job: build
   # Runs the underlying job (“workload”) on a self-hosted runner if available,
   # with the help of a `runner-select` job and a `runner-timeout` job.
-
-  # Selects a self-hosted runner if available, or else a GitHub-hosted runner.
-  # We generate a unique id for the workload, find an idle self-hosted runner
-  # with the given `image:` label that wasn’t already reserved by another
-  # `runner-select` job run, and reserve it with a `reserved-for:<id>` label.
   runner-select:
-    name: Select Runner
+    uses: ./.github/workflows/self-hosted-runner-select.yml
-    runs-on: ubuntu-latest
+    secrets: inherit
-    outputs:
+    with:
-      unique-id: ${{ steps.select.outputs.unique_id }}
+      github-hosted-runner-label: windows-2022
-      selected-runner-label: ${{ steps.select.outputs.selected_runner_label }}
+      self-hosted-image-name: servo-windows10
-      is-self-hosted: ${{ steps.select.outputs.is_self_hosted }}
-    steps:
-      - name: Select and reserve best available runner
-        id: select
-        # Set the variables below to your desired runner images, runner scope
-        # (org or repo), and monitor API URL.
-        run: |
-          github_hosted_runner_label=windows-2022
-          self_hosted_image_name=servo-windows10
-          self_hosted_runner_scope=/orgs/${{ github.repository_owner }}/actions/runners
-          monitor_api_base_url=https://ci0.servo.org
-
-          set -euo pipefail
-
-          fall_back_to_github_hosted() {
-            echo 'Falling back to GitHub-hosted runner'
-            echo "selected_runner_label=$github_hosted_runner_label" | tee -a $GITHUB_OUTPUT
-            echo 'is_self_hosted=false' | tee -a $GITHUB_OUTPUT
-            exit 0
-          }
-
-          # Generate a unique id that allows the workload job to find the runner
-          # we are reserving for it (via runner labels), and allows the timeout
-          # job to find the workload job run (via the job’s friendly name), even
-          # if there are multiple instances in the workflow call tree.
-          unique_id=$(uuidgen)
-          echo "unique_id=$unique_id" | tee -a $GITHUB_OUTPUT
-
-          # Disable self-hosted runners by creating a repository variable named
-          # NO_SELF_HOSTED_RUNNERS with any non-empty value.
-          # <https://github.com/servo/servo/settings/variables/actions>
-          if [ -n '${{ vars.NO_SELF_HOSTED_RUNNERS }}' ]; then
-            echo 'NO_SELF_HOSTED_RUNNERS is set!'
-            fall_back_to_github_hosted
-          fi
-
-          # Use the monitor API to reserve a runner. If we get an object with
-          # runner details, we succeeded. If we get null, we failed.
-          take_runner_url=$monitor_api_base_url/$self_hosted_image_name/$unique_id/${{ github.repository }}/${{ github.run_id }}
-          echo
-          echo POST "$take_runner_url"
-          if ! curl -fsS --max-time 10 -X POST "$take_runner_url" \
-              -H 'Authorization: Bearer ${{ secrets.SERVO_CI_MONITOR_API_TOKEN }}' \
-              | jq -e; then
-            echo
-            echo 'No self-hosted runners available!'
-            fall_back_to_github_hosted
-          fi
-
-          echo
-          echo "selected_runner_label=reserved-for:$unique_id" | tee -a $GITHUB_OUTPUT
-          echo 'is_self_hosted=true' | tee -a $GITHUB_OUTPUT
-
-  # In the unlikely event a self-hosted runner was selected and reserved but it
-  # goes down before the workload starts, cancel the workflow run.
   runner-timeout:
     needs:
       - runner-select
-    if: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
+    uses: ./.github/workflows/self-hosted-runner-timeout.yml
-    name: Detect Runner Timeout
+    secrets: inherit
-    runs-on: ubuntu-latest
+    with:
-    steps:
+      selected-runner-label: ${{ needs.runner-select.outputs.selected-runner-label }}
-      - name: Wait a bit
+      is-self-hosted: ${{ fromJSON(needs.runner-select.outputs.is-self-hosted) }}
-        run: sleep 30
-
-      - name: Cancel if workload job is still queued
-        run: |
-          run_url=/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          export GH_TOKEN=${{ secrets.GITHUB_TOKEN }}
-
-          if [ "$(gh api "$run_url/jobs" \
-            | jq -er --arg id '${{ needs.runner-select.outputs.unique-id }}' \
-              '.jobs[] | select(.name | contains("[" + $id + "]")) | .status'
-          )" = queued ]; then
-            echo 'Timeout waiting for runner assignment!'
-            echo 'Hint: does this repo have permission to access the runner group?'
-            echo 'Hint: https://github.com/organizations/servo/settings/actions/runner-groups'
-            echo
-            echo 'Cancelling workflow run'
-            gh api "$run_url/cancel" --method POST
-            exit 1
-          fi
 
   build:
     needs: