Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-08-06 22:15:33 +01:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #28424 - jdm:nightlies, r=jdm

Browse source 
Replace more taskcluster jobs

Fixes #28283. Fixes #28282. Fixes #28279.
This commit is contained in:
bors-servo 2021-05-24 16:41:44 -04:00 • committed by GitHub
commit 90b1ad4d1f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 164 additions and 278 deletions
Download patch file Download diff file Expand all files Collapse all files

30
.github/workflows/docs.yml vendored Normal file
View file

@ -0,0 +1,30 @@
name: Docs upload
on:
push:
branches: ["master"]
workflow_dispatch:
jobs:
upload-docs:
name: Upload docs to GitHub Pages
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Compile docs
run: python3 ./mach doc
env:
RUSTDOCFLAGS: --disable-minification
- name: Upload docs
run: |
cd target/doc
git init
git add .
git -c user.name="Workflow" -c user.email="" \
commit -q -m "Rebuild Servo documentation"
git remote set-url origin https://git:${DOC_SERVO_ORG}@github.com/servo/doc.servo.org
git push --force origin FETCH_HEAD:gh-pages
env:
DOC_SERVO_ORG: ${{ secrets.DOC_SERVO_ORG }}

110
.github/workflows/nightly.yml vendored Normal file
View file

@ -0,0 +1,110 @@
name: Nightly builds
on:
schedule:
# Run at 5:30 am, daily.
- cron: '15 5 * * *'
workflow_dispatch:
env:
RUST_BACKTRACE: 1
SHELL: /bin/bash
jobs:
build-linux-with-rust-nightly:
name: Build (Linux) + rustc nightly
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Update rustc
run: echo nightly > rust-toolchain
- name: Bootstrap
run: |
python3 -m pip install --upgrade pip virtualenv
sudo apt update
python3 ./mach bootstrap
- name: Release build
run: python3 ./mach build --release
- name: Unit tests
run: python3 ./mach test-unit --release
upload-linux:
name: Upload nightly (Linux)
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Bootstrap
run: |
python3 -m pip install --upgrade pip virtualenv
sudo apt update
python3 ./mach bootstrap
- name: Release build
run: python3 ./mach build --release
- name: Package
run: python3 ./mach package --release
- name: Upload
run: python3 ./mach upload-nightly linux --secret-from-environment
env:
S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}
upload-mac:
name: Upload nightly (macOS)
runs-on: macos-10.15
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Bootstrap
run: |
python3 -m pip install --upgrade pip virtualenv
brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile
brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile-build
rm -rf /usr/local/etc/openssl
rm -rf /usr/local/etc/openssl@1.1
brew install openssl@1.1 gnu-tar
- name: Release build
run: |
export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include"
export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib"
export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/"
export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH"
python3 ./mach build --release
- name: Package
run: python3 ./mach package --release
- name: Smoketest
run: ./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg
- name: Upload
run: python3 ./mach upload-nightly mac --secret-from-environment
env:
S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}
GITHUB_HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_TOKEN }}
upload-win:
name: Upload nightly (Windows)
runs-on: windows-2019
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Copy to C drive
run: cp D:\a C:\ -Recurse
- name: Bootstrap
working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
run: |
python -m pip install --upgrade pip virtualenv
python mach fetch
- name: Release build
working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
run: python mach build --release --media-stack=dummy
- name: Package
working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
run: python mach package --release
- name: Upload
working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
run: python mach upload-nightly windows-msvc --secret-from-environment
env:
S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}

12
etc/ci/workflow.mako
View file

@ -1,11 +1,9 @@
name: CI
on:
# Triggers the workflow on push or pull request events but only for the master branch
# Triggers the workflow on push events but only for the master branch
push:
branches: [ "master", "github-actions-dev", "auto", "try", "try-linux", "try-mac", "try-windows", "try-wpt", "linux-wpt-tests"]
pull_request:
branches: [ "master", "github-actions-dev" ]
branches: [ "auto", "try", "try-linux", "try-mac", "try-windows", "try-wpt"]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
@ -63,6 +61,12 @@ jobs:
python3 ./mach build --release
- name: Smoketest
run: python3 ./mach smoketest
- name: Unit tests
run: python3 ./mach test-unit --release
- name: Test package
run: python3 ./mach package --release
- name: Package smoketest
run: ./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg
- name: Package binary
run: gtar -czf target.tar.gz target/release/servo target/release/*.dylib resources
- name: Archive binary

240
etc/taskcluster/decision_task.py
View file

@ -36,8 +36,6 @@ def tasks(task_for):
windows_unit,
windows_arm64,
windows_uwp_x64,
macos_unit,
linux_release,
]
by_branch_name = {
"auto": all_tests,
@ -46,15 +44,14 @@ def tasks(task_for):
# Add functions here as needed, in your push to that branch
],
"master": [
upload_docs,
layout_2020_regressions_report,
],
# The "try-*" keys match those in `servo_try_choosers` in Homu’s config:
# https://github.com/servo/saltfs/blob/master/homu/map.jinja
"try-mac": [macos_unit],
"try-linux": [linux_tidy_unit, linux_docs_check, linux_release],
"try-mac": [],
"try-linux": [linux_tidy_unit, linux_docs_check],
"try-windows": [windows_unit, windows_arm64, windows_uwp_x64],
"try-arm": [windows_arm64],
"try-wpt": [],
@ -89,10 +86,6 @@ def tasks(task_for):
# https://tools.taskcluster.net/hooks/project-servo/daily
elif task_for == "daily":
daily_tasks_setup()
with_rust_nightly()
linux_nightly()
windows_nightly()
macos_nightly()
update_wpt()
uwp_nightly()
@ -119,7 +112,6 @@ linux_build_env = {
# https://github.com/servo/servo/issues/24714#issuecomment-552951519
"SCCACHE_MAX_FRAME_LENGTH": str(100 * 1024 * 1024), # 100 MiB
}
macos_build_env = {}
windows_build_env = {
"x86_64": {
"GSTREAMER_1_0_ROOT_X86_64": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\",
@ -198,58 +190,17 @@ def linux_tidy_unit():
def linux_docs_check():
return (
linux_build_task("Docs + check")
.with_treeherder("Linux x64", "Doc+Check")
.with_script("""
RUSTDOCFLAGS="--disable-minification" python3 ./mach doc
(
cd target/doc
git init
git add .
git -c user.name="Taskcluster" -c user.email="" \
commit -q -m "Rebuild Servo documentation"
git bundle create docs.bundle HEAD
)
"""
linux_build_task("Check")
.with_treeherder("Linux x64", "Check")
.with_script('RUSTDOCFLAGS="--disable-minification" python3 ./mach doc')
# Because `rustdoc` needs metadata of dependency crates,
# `cargo doc` does almost all of the work that `cargo check` does.
# Therefore, when running them in this order the second command does very little
# and should finish quickly.
# The reverse order would not increase the total amount of work to do,
# but would reduce the amount of parallelism available.
"""
python3 ./mach check
""")
.with_artifacts("/repo/target/doc/docs.bundle")
.find_or_create("docs." + CONFIG.tree_hash())
)
def upload_docs():
docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.tree_hash())
return (
linux_task("Upload docs to GitHub Pages")
.with_treeherder("Linux x64", "DocUpload")
.with_dockerfile(dockerfile_path("base"))
.with_curl_artifact_script(docs_build_task_id, "docs.bundle")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/doc.servo.org")
.with_env(PY="""if 1:
import urllib.request, json, os
root_url = os.environ["TASKCLUSTER_PROXY_URL"]
url = root_url + "/api/secrets/v1/secret/project/servo/doc.servo.org"
token = json.load(urllib.request.urlopen(url))["secret"]["token"]
open("/root/.git-credentials", "w").write("https://git:%s@github.com/" % token)
""")
.with_script("""
python3 -c "$PY"
git init --bare
git config credential.helper store
git fetch --quiet docs.bundle
git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages
""")
.create()
.with_script("python3 ./mach check")
.find_or_create("check." + CONFIG.tree_hash())
)
@ -269,39 +220,6 @@ def layout_2020_regressions_report():
.create()
)
def macos_unit():
return (
macos_build_task("Dev build + unit tests")
.with_treeherder("macOS x64", "Unit")
.with_script("""
python3 ./mach build --dev --verbose
python3 ./mach test-unit
python3 ./mach package --dev
./etc/ci/macos_package_smoketest.sh target/debug/servo-tech-demo.dmg
./etc/ci/lockfile_changed.sh
""")
.find_or_create("macos_unit." + CONFIG.tree_hash())
)
def with_rust_nightly():
modified_build_env = dict(build_env)
flags = modified_build_env.pop("RUSTFLAGS").split(" ")
flags.remove("-Dwarnings")
if flags: # pragma: no cover
modified_build_env["RUSTFLAGS"] = " ".join(flags)
return (
linux_build_task("with Rust Nightly", build_env=modified_build_env)
.with_treeherder("Linux x64", "RustNightly")
.with_script("""
echo "nightly" > rust-toolchain
python3 ./mach build --dev
python3 ./mach test-unit
""")
.create()
)
appx_artifact = '/'.join([
'repo',
@ -397,71 +315,6 @@ def windows_unit(cached=True, rdp=False):
return task.create()
def windows_nightly(rdp=False):
return (
windows_build_task("Nightly build and upload", rdp=rdp)
.with_treeherder("Windows x64", "Nightly")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/s3-upload-credentials")
.with_script("python mach fetch",
"python mach build --release",
"python mach package --release",
"python mach upload-nightly windows-msvc --secret-from-taskcluster")
.with_artifacts("repo/target/release/msi/Servo.exe",
"repo/target/release/msi/Servo.zip")
.find_or_create("build.windows_x64_nightly." + CONFIG.tree_hash())
)
def linux_nightly():
return (
linux_build_task("Nightly build and upload")
.with_treeherder("Linux x64", "Nightly")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/s3-upload-credentials")
# Not reusing the build made for WPT because it has debug assertions
.with_script(
"python3 ./mach build --release",
"python3 ./mach package --release",
"python3 ./mach upload-nightly linux --secret-from-taskcluster",
)
.with_artifacts("/repo/target/release/servo-tech-demo.tar.gz")
.find_or_create("build.linux_x64_nightly" + CONFIG.tree_hash())
)
def linux_release():
return (
linux_build_task("Release build")
.with_treeherder("Linux x64", "Release")
.with_script(
"python3 ./mach build --release",
"python3 ./mach package --release",
)
.find_or_create("build.linux_x64_release" + CONFIG.tree_hash())
)
def macos_nightly():
return (
macos_build_task("Nightly build and upload")
.with_treeherder("macOS x64", "Nightly")
.with_features("taskclusterProxy")
.with_scopes(
"secrets:get:project/servo/s3-upload-credentials",
"secrets:get:project/servo/github-homebrew-token",
)
.with_script(
"python3 ./mach build --release",
"python3 ./mach package --release",
"./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg",
"python3 ./mach upload-nightly mac --secret-from-taskcluster",
)
.with_artifacts("repo/target/release/servo-tech-demo.dmg")
.find_or_create("build.mac_x64_nightly." + CONFIG.tree_hash())
)
def update_wpt():
build_task = linux_release_build_with_debug_assertions(layout_2020=False)
return (
@ -486,24 +339,6 @@ def update_wpt():
)
def macos_release_build_with_debug_assertions(priority=None):
return (
macos_build_task("Release build, with debug assertions")
.with_treeherder("macOS x64", "Release+A")
.with_priority(priority)
.with_script("\n".join([
"python3 ./mach build --release --verbose --with-debug-assertions",
"./etc/ci/lockfile_changed.sh",
"tar -czf target.tar.gz" +
" target/release/servo" +
" target/release/*.dylib" +
" resources",
]))
.with_artifacts("repo/target.tar.gz")
.find_or_create("build.macos_x64_release_w_assertions." + CONFIG.tree_hash())
) # pragma: no cover
def linux_release_build_with_debug_assertions(layout_2020):
if layout_2020:
name_prefix = "Layout 2020 " # pragma: no cover
@ -568,14 +403,6 @@ def windows_task(name):
)
def macos_task(name):
return (
decisionlib.MacOsGenericWorkerTask(name)
.with_worker_type(CONFIG.macos_worker_type)
.with_treeherder_required()
)
def linux_build_task(name, *, build_env=build_env):
task = (
linux_task(name)
@ -648,58 +475,6 @@ def windows_build_task(name, package=True, arch="x86_64", rdp=False):
return task
def with_homebrew(task, brewfiles):
for brewfile in brewfiles:
task.with_script("time brew bundle install --verbose --no-upgrade --file=" + brewfile)
return task
def macos_build_task(name):
build_task = (
macos_task(name)
# Stray processes eating CPU can slow things down:
# https://github.com/servo/servo/issues/24735
.with_max_run_time_minutes(60 * 2)
.with_env(**build_env, **unix_build_env, **macos_build_env)
.with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects")
.with_python3()
.with_rustup()
.with_index_and_artifacts_expire_in(build_artifacts_expire_in)
# Debugging for surprising generic-worker behaviour
.with_early_script("ls")
.with_script("ls target || true")
)
return (
with_homebrew(build_task, [
"etc/taskcluster/macos/Brewfile",
"etc/taskcluster/macos/Brewfile-build",
])
.with_script("""
export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include"
export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib"
export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/"
export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH"
""")
.with_directory_mount(
"https://github.com/mozilla/sccache/releases/download/"
"0.2.7/sccache-0.2.7-x86_64-apple-darwin.tar.gz",
sha256="f86412abbbcce2d3f23e7d33305469198949f5cf807e6c3258c9e1885b4cb57f",
path="sccache",
)
# Early script in order to run with the initial $PWD
.with_early_script("""
export PATH="$PWD/sccache/sccache-0.2.7-x86_64-apple-darwin:$PATH"
""")
# sccache binaries requires OpenSSL 1.1 and are not compatible with 1.0.
# "Late" script in order to run after Homebrew is installed.
.with_script("""
time brew install openssl@1.1
export DYLD_LIBRARY_PATH="$HOME/homebrew/opt/openssl@1.1/lib"
""")
)
CONFIG.task_name_template = "Servo: %s"
CONFIG.docker_images_expire_in = build_dependencies_artifacts_expire_in
CONFIG.repacked_msi_files_expire_in = build_dependencies_artifacts_expire_in
@ -708,7 +483,6 @@ CONFIG.default_provisioner_id = "proj-servo"
CONFIG.docker_image_build_worker_type = "docker"
CONFIG.windows_worker_type = "win2016"
CONFIG.macos_worker_type = "macos"
if __name__ == "__main__": # pragma: no cover
main(task_for=os.environ["TASK_FOR"])

41
etc/taskcluster/decisionlib.py
View file

@ -28,7 +28,7 @@ import taskcluster
# Public API
__all__ = [
"CONFIG", "SHARED", "Task", "DockerWorkerTask",
"GenericWorkerTask", "WindowsGenericWorkerTask", "MacOsGenericWorkerTask",
"GenericWorkerTask", "WindowsGenericWorkerTask",
"make_repo_bundle",
]
@ -673,45 +673,6 @@ class UnixTaskMixin(Task):
))
class MacOsGenericWorkerTask(UnixTaskMixin, GenericWorkerTask):
"""
Task definition for a `generic-worker` task running on macOS.
Scripts are interpreted with `bash`.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.scripts = []
with_script = chaining(append_to_attr, "scripts")
with_early_script = chaining(prepend_to_attr, "scripts")
def build_command(self):
# generic-worker accepts multiple commands, but unlike on Windows
# the current directory and environment variables
# are not preserved across commands on macOS.
# So concatenate scripts and use a single `bash` command instead.
return [
[
"/bin/bash", "--login", "-x", "-e", "-o", "pipefail", "-c",
deindent("\n".join(self.scripts))
]
] # pragma: no cover
def with_python3(self):
return self.with_early_script("""
python3 -m ensurepip --user
python3 -m pip install --user virtualenv
""")
def with_rustup(self):
return self.with_early_script("""
export PATH="$HOME/.cargo/bin:$PATH"
which rustup || curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y
rustup self update
""")
class DockerWorkerTask(UnixTaskMixin, Task):
"""
Task definition for a worker type that runs the `generic-worker` implementation.

9
python/servo/package_commands.py
View file

@ -601,7 +601,10 @@ class PackageCommands(CommandBase):
@CommandArgument('--secret-from-taskcluster',
action='store_true',
help='Retrieve the appropriate secrets from taskcluster.')
def upload_nightly(self, platform, secret_from_taskcluster):
@CommandArgument('--secret-from-environment',
action='store_true',
help='Retrieve the appropriate secrets from the environment.')
def upload_nightly(self, platform, secret_from_taskcluster, secret_from_environment):
import boto3
def get_s3_secret():
@ -611,6 +614,10 @@ class PackageCommands(CommandBase):
secret = get_taskcluster_secret("s3-upload-credentials")
aws_access_key = secret["aws_access_key_id"]
aws_secret_access_key = secret["aws_secret_access_key"]
elif secret_from_environment:
secret = json.loads(os.environ['S3_UPLOAD_CREDENTIALS'])
aws_access_key = secret["aws_access_key_id"]
aws_secret_access_key = secret["aws_secret_access_key"]
return (aws_access_key, aws_secret_access_key)
def nightly_filename(package, timestamp):