Auto merge of #23304 - jdm:nightly-taskcluster, r=SimonSapin

Transfer mac/windows/android nightly builds to Taskcluster

Fixes #23303.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/23304)
<!-- Reviewable:end -->

etc/ci/buildbot_steps.yml

@@ -43,16 +43,6 @@ mac-rel-css2:
   - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 6 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
   - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default
 
-mac-nightly:
-  - ./mach clean-nightlies --keep 3 --force
-  - ./mach clean-cargo-cache --keep 3 --force
-  - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release
-  - ./mach package --release
-  - ./mach upload-nightly mac
-  - ./etc/ci/update-wpt-checkout fetch-and-update-expectations
-  - ./etc/ci/update-wpt-checkout open-pr
-  - ./etc/ci/update-wpt-checkout cleanup
-
 linux-rel-intermittent:
  env:
    CC: gcc-5
@@ -144,23 +134,6 @@ android-mac:
   - bash ./etc/ci/lockfile_changed.sh
   - ./etc/ci/clean_build_artifacts.sh
 
-android-nightly:
- env:
-   HOST_CC: gcc-5
-   HOST_CXX: g++-5
- commands:
-  - ./mach clean-nightlies --keep 3 --force
-  - ./mach clean-cargo-cache --keep 3 --force
-  - ./etc/ci/clean_build_artifacts.sh
-  - ./mach bootstrap-android --accept-all-licences --build
-  - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --android --release
-  - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --android --release --maven
-  - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach build --target=i686-linux-android --release
-  - env --unset ANDROID_NDK --unset ANDROID_SDK ./mach package --target=i686-linux-android --release --maven
-  - ./mach upload-nightly android
-  - ./mach upload-nightly maven
-  - ./etc/ci/clean_build_artifacts.sh
-
 magicleap:
  env:
   MAGICLEAP_SDK: /Users/servo/magicleap/v0.17.0
@@ -248,16 +221,6 @@ arm64:
   - bash ./etc/ci/lockfile_changed.sh
   - ./etc/ci/clean_build_artifacts.sh
 
-windows-msvc-nightly:
-  env:
-    CARGO_HOME: C:\buildbot\.cargo
-  commands:
-  - mach.bat clean-cargo-cache --keep 3 --force
-  - mach.bat clean-nightlies --keep 3 --force
-  - mach.bat build --release
-  - mach.bat package --release
-  - mach.bat upload-nightly windows-msvc
-
 # Moved to Taskcluster
 linux-dev: []
 mac-dev-unit: []
@@ -265,3 +228,6 @@ windows-msvc-dev: []
 android: []
 android-x86: []
 mac-rel-wpt1: []
+android-nightly: []
+windows-msvc-nightly: []
+mac-nightly: []

etc/ci/update-wpt-checkout

@@ -92,7 +92,12 @@ function unsafe_open_pull_request() {
     git checkout "${BRANCH_NAME}" || return 0
 
     if [[ -z "${WPT_SYNC_TOKEN+set}" ]]; then
-        echo "Github auth token missing from WPT_SYNC_TOKEN."
+        SECRET_RESPONSE=$(curl $TASKCLUSTER_PROXY_URL/api/secrets/v1/secret/project/servo/wpt-sync)
+        WPT_SYNC_TOKEN=`echo "${OPEN_PR_RESPONSE}" | jq '.token'`
+    fi
+
+    if [[ -z "${WPT_SYNC_TOKEN}" ]]; then
+        echo "Github auth token missing from .wpt-token file."
         return 1
     fi
 

etc/taskcluster/decision_task.py

@@ -90,6 +90,10 @@ def main(task_for):
         daily_tasks_setup()
         with_rust_nightly()
         linux_nightly()
+        android_nightly("arm")
+        android_nightly("x86")
+        windows_nightly()
+        macos_nightly()
 
 
 # These are disabled in a "real" decision task,
@@ -260,6 +264,39 @@ def android_arm32_dev():
     )
 
 
+def android_nightly(job):
+    details = {
+        "arm": {
+            "mach_flag": "--android",
+            "name": "ARMv7",
+            "target": "armv7-linux-androideabi",
+        },
+        "x86": {
+            "mach_flag": "--target i686-linux-android",
+            "name": "x86",
+            "target": "i686-linux-android",
+        }
+    }
+
+    return (
+        android_build_task("Release build")
+        .with_treeherder("Android " + details[job]["name"], "Nightly")
+        .with_features("taskclusterProxy")
+        .with_scopes("secrets:get:project/servo/s3-upload")
+        .with_script("""
+            ./mach build {flag} --release
+            ./mach package {flag} --release --maven
+            ./mach upload-nightly android --secret-from-taskcluster
+            ./mach upload-nightly maven --secret-from-taskcluster
+        """.format(flag=details[job]["mach_flag"]))
+        .with_artifacts(
+            "/repo/target/android/%s/release/servoapp.apk" % details[job]["target"],
+            "/repo/target/android/%s/release/servoview.aar" % details[job]["target"],
+        )
+        .find_or_create(("build.android_%s_nightly." + CONFIG.git_sha) % details[job]["name"].lower())
+    )
+
+
 def android_arm32_release():
     return (
         android_build_task("Release build")
@@ -354,26 +391,32 @@ def windows_release():
     )
 
 
+def windows_nightly():
+    return (
+        windows_build_task("Release build")
+        .with_treeherder("Windows x64", "Nightly")
+        .with_scopes("secrets:get:project/servo/s3-upload")
+        .with_script("mach build --release",
+                     "mach package --release",
+                     "mach upload-nightly windows-msvc --secret-from-taskcluster")
+        .with_artifacts("repo/target/release/msi/Servo.exe",
+                        "repo/target/release/msi/Servo.zip")
+        .find_or_create("build.windows_x64_nightly." + CONFIG.git_sha)
+    )
+
+
 def linux_nightly():
     return (
         linux_build_task("Nightly build and upload")
         .with_treeherder("Linux x64", "Nightly")
         .with_features("taskclusterProxy")
         .with_scopes("secrets:get:project/servo/s3-upload")
-        .with_env(PY=r"""if 1:
-            import urllib, json
-            url = "http://taskcluster/secrets/v1/secret/project/servo/s3-upload"
-            secret = json.load(urllib.urlopen(url))["secret"]
-            open("/root/.aws/credentials", "w").write(secret["credentials_file"])
-        """)
         # Not reusing the build made for WPT because it has debug assertions
-        .with_script("""
+        .with_script(
-            ./mach build --release
+            "./mach build --release",
-            ./mach package --release
+            "./mach package --release",
-            mkdir /root/.aws
+            "./mach upload-nightly linux --secret-from-taskcluster",
-            python -c "$PY"
+        )
-            ./mach upload-nightly linux
-        """)
         .with_artifacts("/repo/target/release/servo-tech-demo.tar.gz")
         .find_or_create("build.linux_x64_nightly" + CONFIG.git_sha)
     )
@@ -400,6 +443,31 @@ def linux_wpt():
                total_chunks=2, processes=24)
 
 
+def macos_nightly():
+    return (
+        macos_build_task("Release build")
+        .with_treeherder("macOS x64", "Nightly")
+        .with_features("taskclusterProxy")
+        .with_scopes(
+            "secrets:get:project/servo/s3-upload",
+            "secrets:get:project/servo/github-homebrew-token",
+            "secrets:get:project/servo/wpt-sync",
+        )
+        .with_script(
+            "./mach build --release",
+            "./mach package --release",
+            "./mach upload-nightly mac --secret-from-taskcluster",
+        )
+        .with_artifacts("repo/target/release/servo-tech-demo.dmg")
+        .with_script(
+            "./etc/ci/update-wpt-checkout fetch-and-update-expectations",
+            "./etc/ci/update-wpt-checkout open-pr",
+            "./etc/ci/update-wpt-checkout cleanup",
+        )
+        .find_or_create("build.mac_x64_nightly." + CONFIG.git_sha)
+    )
+
+
 def macos_wpt():
     build_task = (
         macos_build_task("Release build")

etc/taskcluster/decisionlib.py

@@ -286,6 +286,7 @@ class GenericWorkerTask(Task):
         super().__init__(*args, **kwargs)
         self.max_run_time_minutes = 30
         self.env = {}
+        self.features = {}
         self.mounts = []
         self.artifacts = []
 
@@ -314,6 +315,7 @@ class GenericWorkerTask(Task):
             worker_payload,
             env=self.env,
             mounts=self.mounts,
+            features=self.features,
             artifacts=[
                 {
                     "type": type_,
@@ -337,6 +339,15 @@ class GenericWorkerTask(Task):
         self.artifacts.extend((type, path) for path in paths)
         return self
 
+    def with_features(self, *names):
+        """
+        Enable the given `generic-worker` features.
+
+        <https://github.com/taskcluster/generic-worker/blob/master/native_windows.yml>
+        """
+        self.features.update({name: True for name in names})
+        return self
+
     def _mount_content(self, url_or_artifact_name, task_id, sha256):
         if task_id:
             content = {"taskId": task_id, "artifact": url_or_artifact_name}
@@ -693,9 +704,9 @@ class DockerWorkerTask(UnixTaskMixin, Task):
 
     def with_features(self, *names):
         """
-        Enable the give `docker-worker` features.
+        Enable the given `docker-worker` features.
 
-        <https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features>
+        <https://github.com/taskcluster/docker-worker/blob/master/docs/features.md>
         """
         self.features.update({name: True for name in names})
         return self

python/servo/package_commands.py

@@ -19,6 +19,7 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import urllib
 
 from mach.decorators import (
     CommandArgument,
@@ -538,9 +539,25 @@ class PackageCommands(CommandBase):
     @CommandArgument('platform',
                      choices=PACKAGES.keys(),
                      help='Package platform type to upload')
-    def upload_nightly(self, platform):
+    @CommandArgument('--secret-from-taskcluster',
+                     action='store_true',
+                     help='Retrieve the appropriate secrets from taskcluster.')
+    def upload_nightly(self, platform, secret_from_taskcluster):
         import boto3
 
+        def get_taskcluster_secret(name):
+            url = "http://taskcluster/secrets/v1/secret/project/servo/" + name
+            return json.load(urllib.urlopen(url))["secret"]
+
+        def get_s3_secret():
+            aws_access_key = None
+            aws_secret_access_key = None
+            if secret_from_taskcluster:
+                secret = get_taskcluster_secret("s3-upload-credentials")
+                aws_access_key = secret["aws_access_key_id"]
+                aws_secret_access_key = secret["aws_secret_access_key"]
+            return (aws_access_key, aws_secret_access_key)
+
         def nightly_filename(package, timestamp):
             return '{}-{}'.format(
                 timestamp.isoformat() + 'Z',  # The `Z` denotes UTC
@@ -548,7 +565,12 @@ class PackageCommands(CommandBase):
             )
 
         def upload_to_s3(platform, package, timestamp):
-            s3 = boto3.client('s3')
+            (aws_access_key, aws_secret_access_key) = get_s3_secret()
+            s3 = boto3.client(
+                's3',
+                aws_access_key_id=aws_access_key,
+                aws_secret_access_key=aws_secret_access_key
+            )
             BUCKET = 'servo-builds'
 
             nightly_dir = 'nightly/{}'.format(platform)
@@ -565,7 +587,12 @@ class PackageCommands(CommandBase):
             s3.copy(copy_source, BUCKET, latest_upload_key)
 
         def update_maven(directory):
-            s3 = boto3.client('s3')
+            (aws_access_key, aws_secret_access_key) = get_s3_secret()
+            s3 = boto3.client(
+                's3',
+                aws_access_key_id=aws_access_key,
+                aws_secret_access_key=aws_secret_access_key
+            )
             BUCKET = 'servo-builds'
 
             nightly_dir = 'nightly/maven'
@@ -626,13 +653,18 @@ class PackageCommands(CommandBase):
                     '--message=Version Bump: {}'.format(brew_version),
                 ])
 
+                if secret_from_taskcluster:
+                    token = get_taskcluster_secret('github-homebrew-token')["token"]
+                else:
+                    token = os.environ['GITHUB_HOMEBREW_TOKEN']
+
                 push_url = 'https://{}@github.com/servo/homebrew-servo.git'
                 # TODO(aneeshusa): Use subprocess.DEVNULL with Python 3.3+
                 with open(os.devnull, 'wb') as DEVNULL:
                     call_git([
                         'push',
                         '-qf',
-                        push_url.format(os.environ['GITHUB_HOMEBREW_TOKEN']),
+                        push_url.format(token),
                         'master',
                     ], stdout=DEVNULL, stderr=DEVNULL)