Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-07-02 21:13:39 +01:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #23304 - jdm:nightly-taskcluster, r=SimonSapin

Browse source 
Transfer mac/windows/android nightly builds to Taskcluster

Fixes #23303.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/23304)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2019-05-02 19:11:32 -04:00 • committed by GitHub
commit 38e60136b8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 139 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

40
etc/ci/buildbot_steps.yml
View file

@ -43,16 +43,6 @@ mac-rel-css2:
- ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 6 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
- ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default
mac-nightly:
- ./mach clean-nightlies --keep 3 --force
- ./mach clean-cargo-cache --keep 3 --force
- env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release
- ./mach package --release
- ./mach upload-nightly mac
- ./etc/ci/update-wpt-checkout fetch-and-update-expectations
- ./etc/ci/update-wpt-checkout open-pr
- ./etc/ci/update-wpt-checkout cleanup
linux-rel-intermittent:
env:
CC: gcc-5
@ -144,23 +134,6 @@ android-mac:
- bash ./etc/ci/lockfile_changed.sh
- ./etc/ci/clean_build_artifacts.sh
android-nightly:
env:
HOST_CC: gcc-5
HOST_CXX: g++-5
commands:
- ./mach clean-nightlies --keep 3 --force
- ./mach clean-cargo-cache --keep 3 --force
- ./etc/ci/clean_build_artifacts.sh
- ./mach bootstrap-android --accept-all-licences --build
- env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --release
- env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --release --maven
- env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target=i686-linux-android --release
- env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --target=i686-linux-android --release --maven
- ./mach upload-nightly android
- ./mach upload-nightly maven
- ./etc/ci/clean_build_artifacts.sh
magicleap:
env:
MAGICLEAP_SDK: /Users/servo/magicleap/v0.17.0
@ -248,16 +221,6 @@ arm64:
- bash ./etc/ci/lockfile_changed.sh
- ./etc/ci/clean_build_artifacts.sh
windows-msvc-nightly:
env:
CARGO_HOME: C:\buildbot\.cargo
commands:
- mach.bat clean-cargo-cache --keep 3 --force
- mach.bat clean-nightlies --keep 3 --force
- mach.bat build --release
- mach.bat package --release
- mach.bat upload-nightly windows-msvc
# Moved to Taskcluster
linux-dev: []
mac-dev-unit: []
@ -265,3 +228,6 @@ windows-msvc-dev: []
android: []
android-x86: []
mac-rel-wpt1: []
android-nightly: []
windows-msvc-nightly: []
mac-nightly: []

7
etc/ci/update-wpt-checkout
View file

@ -92,7 +92,12 @@ function unsafe_open_pull_request() {
git checkout "${BRANCH_NAME}" || return 0
if [[ -z "${WPT_SYNC_TOKEN+set}" ]]; then
echo "Github auth token missing from WPT_SYNC_TOKEN."
SECRET_RESPONSE=$(curl $TASKCLUSTER_PROXY_URL/api/secrets/v1/secret/project/servo/wpt-sync)
WPT_SYNC_TOKEN=`echo "${OPEN_PR_RESPONSE}" | jq '.token'`
fi
if [[ -z "${WPT_SYNC_TOKEN}" ]]; then
echo "Github auth token missing from .wpt-token file."
return 1
fi

94
etc/taskcluster/decision_task.py
View file

@ -90,6 +90,10 @@ def main(task_for):
daily_tasks_setup()
with_rust_nightly()
linux_nightly()
android_nightly("arm")
android_nightly("x86")
windows_nightly()
macos_nightly()
# These are disabled in a "real" decision task,
@ -260,6 +264,39 @@ def android_arm32_dev():
)
def android_nightly(job):
details = {
"arm": {
"mach_flag": "--android",
"name": "ARMv7",
"target": "armv7-linux-androideabi",
},
"x86": {
"mach_flag": "--target i686-linux-android",
"name": "x86",
"target": "i686-linux-android",
}
}
return (
android_build_task("Release build")
.with_treeherder("Android " + details[job]["name"], "Nightly")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/s3-upload")
.with_script("""
./mach build {flag} --release
./mach package {flag} --release --maven
./mach upload-nightly android --secret-from-taskcluster
./mach upload-nightly maven --secret-from-taskcluster
""".format(flag=details[job]["mach_flag"]))
.with_artifacts(
"/repo/target/android/%s/release/servoapp.apk" % details[job]["target"],
"/repo/target/android/%s/release/servoview.aar" % details[job]["target"],
)
.find_or_create(("build.android_%s_nightly." + CONFIG.git_sha) % details[job]["name"].lower())
)
def android_arm32_release():
return (
android_build_task("Release build")
@ -354,26 +391,32 @@ def windows_release():
)
def windows_nightly():
return (
windows_build_task("Release build")
.with_treeherder("Windows x64", "Nightly")
.with_scopes("secrets:get:project/servo/s3-upload")
.with_script("mach build --release",
"mach package --release",
"mach upload-nightly windows-msvc --secret-from-taskcluster")
.with_artifacts("repo/target/release/msi/Servo.exe",
"repo/target/release/msi/Servo.zip")
.find_or_create("build.windows_x64_nightly." + CONFIG.git_sha)
)
def linux_nightly():
return (
linux_build_task("Nightly build and upload")
.with_treeherder("Linux x64", "Nightly")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/s3-upload")
.with_env(PY=r"""if 1:
import urllib, json
url = "http://taskcluster/secrets/v1/secret/project/servo/s3-upload"
secret = json.load(urllib.urlopen(url))["secret"]
open("/root/.aws/credentials", "w").write(secret["credentials_file"])
""")
# Not reusing the build made for WPT because it has debug assertions
.with_script("""
./mach build --release
./mach package --release
mkdir /root/.aws
python -c "$PY"
./mach upload-nightly linux
""")
.with_script(
"./mach build --release",
"./mach package --release",
"./mach upload-nightly linux --secret-from-taskcluster",
)
.with_artifacts("/repo/target/release/servo-tech-demo.tar.gz")
.find_or_create("build.linux_x64_nightly" + CONFIG.git_sha)
)
@ -400,6 +443,31 @@ def linux_wpt():
total_chunks=2, processes=24)
def macos_nightly():
return (
macos_build_task("Release build")
.with_treeherder("macOS x64", "Nightly")
.with_features("taskclusterProxy")
.with_scopes(
"secrets:get:project/servo/s3-upload",
"secrets:get:project/servo/github-homebrew-token",
"secrets:get:project/servo/wpt-sync",
)
.with_script(
"./mach build --release",
"./mach package --release",
"./mach upload-nightly mac --secret-from-taskcluster",
)
.with_artifacts("repo/target/release/servo-tech-demo.dmg")
.with_script(
"./etc/ci/update-wpt-checkout fetch-and-update-expectations",
"./etc/ci/update-wpt-checkout open-pr",
"./etc/ci/update-wpt-checkout cleanup",
)
.find_or_create("build.mac_x64_nightly." + CONFIG.git_sha)
)
def macos_wpt():
build_task = (
macos_build_task("Release build")

15
etc/taskcluster/decisionlib.py
View file

@ -286,6 +286,7 @@ class GenericWorkerTask(Task):
super().__init__(*args, **kwargs)
self.max_run_time_minutes = 30
self.env = {}
self.features = {}
self.mounts = []
self.artifacts = []
@ -314,6 +315,7 @@ class GenericWorkerTask(Task):
worker_payload,
env=self.env,
mounts=self.mounts,
features=self.features,
artifacts=[
{
"type": type_,
@ -337,6 +339,15 @@ class GenericWorkerTask(Task):
self.artifacts.extend((type, path) for path in paths)
return self
def with_features(self, *names):
"""
Enable the given `generic-worker` features.
<https://github.com/taskcluster/generic-worker/blob/master/native_windows.yml>
"""
self.features.update({name: True for name in names})
return self
def _mount_content(self, url_or_artifact_name, task_id, sha256):
if task_id:
content = {"taskId": task_id, "artifact": url_or_artifact_name}
@ -693,9 +704,9 @@ class DockerWorkerTask(UnixTaskMixin, Task):
def with_features(self, *names):
"""
Enable the give `docker-worker` features.
Enable the given `docker-worker` features.
<https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features>
<https://github.com/taskcluster/docker-worker/blob/master/docs/features.md>
"""
self.features.update({name: True for name in names})
return self

40
python/servo/package_commands.py
View file

@ -19,6 +19,7 @@ import shutil
import subprocess
import sys
import tempfile
import urllib
from mach.decorators import (
CommandArgument,
@ -538,9 +539,25 @@ class PackageCommands(CommandBase):
@CommandArgument('platform',
choices=PACKAGES.keys(),
help='Package platform type to upload')
def upload_nightly(self, platform):
@CommandArgument('--secret-from-taskcluster',
action='store_true',
help='Retrieve the appropriate secrets from taskcluster.')
def upload_nightly(self, platform, secret_from_taskcluster):
import boto3
def get_taskcluster_secret(name):
url = "http://taskcluster/secrets/v1/secret/project/servo/" + name
return json.load(urllib.urlopen(url))["secret"]
def get_s3_secret():
aws_access_key = None
aws_secret_access_key = None
if secret_from_taskcluster:
secret = get_taskcluster_secret("s3-upload-credentials")
aws_access_key = secret["aws_access_key_id"]
aws_secret_access_key = secret["aws_secret_access_key"]
return (aws_access_key, aws_secret_access_key)
def nightly_filename(package, timestamp):
return '{}-{}'.format(
timestamp.isoformat() + 'Z', # The `Z` denotes UTC
@ -548,7 +565,12 @@ class PackageCommands(CommandBase):
)
def upload_to_s3(platform, package, timestamp):
s3 = boto3.client('s3')
(aws_access_key, aws_secret_access_key) = get_s3_secret()
s3 = boto3.client(
's3',
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_access_key
)
BUCKET = 'servo-builds'
nightly_dir = 'nightly/{}'.format(platform)
@ -565,7 +587,12 @@ class PackageCommands(CommandBase):
s3.copy(copy_source, BUCKET, latest_upload_key)
def update_maven(directory):
s3 = boto3.client('s3')
(aws_access_key, aws_secret_access_key) = get_s3_secret()
s3 = boto3.client(
's3',
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_access_key
)
BUCKET = 'servo-builds'
nightly_dir = 'nightly/maven'
@ -626,13 +653,18 @@ class PackageCommands(CommandBase):
'--message=Version Bump: {}'.format(brew_version),
])
if secret_from_taskcluster:
token = get_taskcluster_secret('github-homebrew-token')["token"]
else:
token = os.environ['GITHUB_HOMEBREW_TOKEN']
push_url = 'https://{}@github.com/servo/homebrew-servo.git'
# TODO(aneeshusa): Use subprocess.DEVNULL with Python 3.3+
with open(os.devnull, 'wb') as DEVNULL:
call_git([
'push',
'-qf',
push_url.format(os.environ['GITHUB_HOMEBREW_TOKEN']),
push_url.format(token),
'master',
], stdout=DEVNULL, stderr=DEVNULL)