From f1ba708cf77f7ee7dd7c718f48c4f0532bba1845 Mon Sep 17 00:00:00 2001 From: Mukilan Thiyagarajan Date: Sun, 9 Apr 2023 10:14:45 +0530 Subject: [PATCH] Upload nightly builds to Github Releases This change extends the `mach upload-nightly` command to publish the nightly builds for all platforms as GH Release assets. The GH releases are made on a separate repository so that we can persist older nightly builds without having to accumulate git tags for them. Some design tradeoffs in this approach are: 1. To allow the 'latest' link from servo.org to remain stable, the release assets are named 'servo-latest.{ext}' instead of containing the release tag/date. 2. The release is created as draft and published atomically when all platforms have been built successfully. This allows us to link to the 'latest' alias from servo.org while gauranteeing that it contains builds for all platforms. The other option here would be to have code in servo.org UI that uses GH API to find the most recent release with a successful build for a given platform. 3. The tags in the nightly repo are all based on the same commit that has no relation to servo code base. Signed-off-by: Mukilan Thiyagarajan --- .github/workflows/linux.yml | 10 ++++- .github/workflows/mac.yml | 9 ++++- .github/workflows/nightly.yml | 63 ++++++++++++++++++++++++++++++++ .github/workflows/windows.yml | 9 ++++- python/requirements.txt | 1 + python/servo/package_commands.py | 54 ++++++++++++++++++++------- 6 files changed, 129 insertions(+), 17 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 66149b5b86e..517691d2e85 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -16,6 +16,9 @@ on: required: false default: false type: boolean + github-release-id: + required: false + type: string workflow_dispatch: inputs: layout: @@ -82,9 +85,14 @@ jobs: path: target/release/servo-tech-demo.tar.gz - name: Upload if: ${{ inputs.upload }} - run: python3 ./mach upload-nightly ${{ env.PACKAGE }} --secret-from-environment + run: | + python3 ./mach upload-nightly ${{ env.PACKAGE }} \ + --secret-from-environment \ + --github-release-id ${{ inputs.github-release-id }} env: S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} + NIGHTLY_REPO_TOKEN: ${{ secrets.NIGHTLY_REPO_TOKEN }} + NIGHTLY_REPO: ${{ github.repository_owner }}/servo-nightly-builds - name: Package binary run: tar -czf target.tar.gz target/release/servo resources - name: Archive binary diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 933683c713c..f8348df28ba 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -14,6 +14,9 @@ on: required: false default: false type: boolean + github-release-id: + required: false + type: string workflow_dispatch: inputs: layout: @@ -91,10 +94,14 @@ jobs: path: target/release/servo-tech-demo.dmg - name: Upload if: ${{ inputs.upload }} - run: python3 ./mach upload-nightly mac --secret-from-environment + run: | + python3 ./mach upload-nightly mac --secret-from-environment \ + --github-release-id ${{ inputs.github-release-id }} env: S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} GITHUB_HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_TOKEN }} + NIGHTLY_REPO_TOKEN: ${{ secrets.NIGHTLY_REPO_TOKEN }} + NIGHTLY_REPO: ${{ github.repository_owner }}/servo-nightly-builds - name: Package binary run: gtar -czf target.tar.gz target/release/servo target/release/*.dylib resources - name: Archive binary diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 12aec1ea428..a09de65c789 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -11,42 +11,105 @@ env: SHELL: /bin/bash jobs: + create-draft-release: + # This job is only useful when run on upstream servo. + if: github.repository == 'servo/servo' || github.event_name == 'workflow_dispatch' + name: Create Draft GH Release + runs-on: ubuntu-20.04 + steps: + - id: create-release + run: | + NIGHTLY_TAG=$(date "+%F") + RELEASE_URL=$(gh release create "${NIGHTLY_TAG}" \ + --draft \ + --title "${NIGHTLY_TAG}" \ + --notes 'Nightly builds based on servo/servo@${{ github.sha }}' \ + --repo ${NIGHTLY_REPO}) + TEMP_TAG=$(basename "$RELEASE_URL") + RELEASE_ID=$( \ + gh api -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/${NIGHTLY_REPO}/releases/tags/${TEMP_TAG}" \ + | jq '.id' \ + ) + echo "RELEASE_ID=${RELEASE_ID}" >> ${GITHUB_OUTPUT} + env: + GITHUB_TOKEN: ${{ secrets.NIGHTLY_REPO_TOKEN }} + NIGHTLY_REPO: ${{ github.repository_owner }}/servo-nightly-builds + outputs: + release-id: ${{ steps.create-release.outputs.RELEASE_ID }} + + publish-nightly-release: + # This job is only useful when run on upstream servo. + if: github.repository == 'servo/servo' || github.event_name == 'workflow_dispatch' + name: Publish GH Release for nightly + runs-on: ubuntu-20.04 + steps: + - run: | + gh api \ + --method PATCH \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${NIGHTLY_REPO}/releases/${RELEASE_ID} \ + -F draft=false + env: + GITHUB_TOKEN: ${{ secrets.NIGHTLY_REPO_TOKEN }} + NIGHTLY_REPO: ${{ github.repository_owner }}/servo-nightly-builds + RELEASE_ID: ${{ needs.create-draft-release.outputs.release-id }} + needs: + - create-draft-release + - upload-linux + - upload-win + - upload-mac + upload-win: # This job is only useful when run on upstream servo. if: github.repository == 'servo/servo' || github.event_name == 'workflow_dispatch' name: Upload nightly (Windows) + needs: + - create-draft-release uses: ./.github/workflows/windows.yml with: layout: '2013' upload: true + github-release-id: ${{ needs.create-draft-release.outputs.release-id }} secrets: inherit upload-mac: # This job is only useful when run on upstream servo. if: github.repository == 'servo/servo' || github.event_name == 'workflow_dispatch' name: Upload nightly (macOS) + needs: + - create-draft-release uses: ./.github/workflows/mac.yml with: layout: '2013' upload: true + github-release-id: ${{ needs.create-draft-release.outputs.release-id }} secrets: inherit upload-linux: # This job is only useful when run on upstream servo. if: github.repository == 'servo/servo' || github.event_name == 'workflow_dispatch' name: Upload nightly (Linux) + needs: + - create-draft-release uses: ./.github/workflows/linux.yml with: layout: '2013' upload: true + github-release-id: ${{ needs.create-draft-release.outputs.release-id }} secrets: inherit upload-linux-2020: # This job is only useful when run on upstream servo. if: github.repository == 'servo/servo' || github.event_name == 'workflow_dispatch' name: Upload nightly (Linux layout2020) + needs: + - create-draft-release uses: ./.github/workflows/linux.yml with: layout: '2020' upload: true + github-release-id: ${{ needs.create-draft-release.outputs.release-id }} secrets: inherit diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f4c3c0364de..63fe277161a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -14,6 +14,9 @@ on: required: false default: false type: boolean + github-release-id: + required: false + type: string workflow_dispatch: inputs: layout: @@ -83,9 +86,13 @@ jobs: - name: Upload if: ${{ inputs.upload }} working-directory: "C:\\a\\${{ github.event.repository.name }}\\${{ github.event.repository.name }}" - run: python mach upload-nightly windows-msvc --secret-from-environment + run: | + python mach upload-nightly windows-msvc --secret-from-environment ` + --github-release-id ${{ inputs.github-release-id }} env: S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} + NIGHTLY_REPO_TOKEN: ${{ secrets.NIGHTLY_REPO_TOKEN }} + NIGHTLY_REPO: ${{ github.repository_owner }}/servo-nightly-builds build_result: name: homu build finished diff --git a/python/requirements.txt b/python/requirements.txt index b58140060e3..ce46d904b83 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -23,6 +23,7 @@ colorama == 0.3.7 # For package uploading boto3 == 1.17.27 +PyGithub == 1.58.1 # Default root CAs on Windows CI do not trust CloudFront certificates, # connecting to https://static.rust-lang.org would fail: diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 53910dcd644..405bdfb8c3e 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -10,6 +10,8 @@ from __future__ import absolute_import, print_function, unicode_literals from datetime import datetime +from github import Github + import base64 import hashlib import io @@ -594,7 +596,11 @@ class PackageCommands(CommandBase): @CommandArgument('--secret-from-environment', action='store_true', help='Retrieve the appropriate secrets from the environment.') - def upload_nightly(self, platform, secret_from_environment): + @CommandArgument('--github-release-id', + default=None, + type=int, + help='The github release to upload the nightly builds.') + def upload_nightly(self, platform, secret_from_environment, github_release_id): import boto3 def get_s3_secret(): @@ -612,7 +618,24 @@ class PackageCommands(CommandBase): path.basename(package) ) - def upload_to_s3(platform, package, timestamp): + def upload_to_github_release(platform, package, package_hash_fileobj): + if not github_release_id: + return + extension = path.basename(package).partition('.')[2] + g = Github(os.environ['NIGHTLY_REPO_TOKEN']) + nightly_repo = g.get_repo(os.environ['NIGHTLY_REPO']) + release = nightly_repo.get_release(github_release_id) + if '2020' in platform: + asset_name = f'servo-latest-layout-2020.{extension}' + else: + asset_name = f'servo-latest.{extension}' + release.upload_asset(package, name=asset_name) + release.upload_asset_from_memory( + package_hash_fileobj, + package_hash_fileobj.getbuffer().nbytes, + name=f'{asset_name}.sha256') + + def upload_to_s3(platform, package, package_hash_fileobj, timestamp): (aws_access_key, aws_secret_access_key) = get_s3_secret() s3 = boto3.client( 's3', @@ -635,17 +658,6 @@ class PackageCommands(CommandBase): extension = path.basename(package).partition('.')[2] latest_upload_key = '{}/servo-latest.{}'.format(nightly_dir, extension) - # Compute the hash - SHA_BUF_SIZE = 1048576 # read in 1 MiB chunks - sha256_digest = hashlib.sha256() - with open(package, 'rb') as package_file: - while True: - data = package_file.read(SHA_BUF_SIZE) - if not data: - break - sha256_digest.update(data) - package_hash = sha256_digest.hexdigest() - package_hash_fileobj = io.BytesIO(package_hash.encode('utf-8')) latest_hash_upload_key = f'{latest_upload_key}.sha256' s3.upload_file(package, BUCKET, package_upload_key) @@ -763,7 +775,21 @@ class PackageCommands(CommandBase): package ), file=sys.stderr) return 1 - upload_to_s3(platform, package, timestamp) + + # Compute the hash + SHA_BUF_SIZE = 1048576 # read in 1 MiB chunks + sha256_digest = hashlib.sha256() + with open(package, 'rb') as package_file: + while True: + data = package_file.read(SHA_BUF_SIZE) + if not data: + break + sha256_digest.update(data) + package_hash = sha256_digest.hexdigest() + package_hash_fileobj = io.BytesIO(package_hash.encode('utf-8')) + + upload_to_s3(platform, package, package_hash_fileobj, timestamp) + upload_to_github_release(platform, package, package_hash_fileobj) if platform == 'maven': for package in PACKAGES[platform]: