Auto merge of #22046 - servo:tc-android-x86, r=Manishearth+asajeffrey

Taskcluster: build for Android x86 and test in emulator

This adds to Taskcluster the equivalent of Buildbot’s `android-x86` job. CC https://github.com/servo/saltfs/issues/559

Like on Buildbot, the Android emulator runs on a Packet.net server. However we can compiler in a separate task, and run that task on fast AWS instances. The Packet server is provisionned and installed automatically with Terraform scripts forked from https://github.com/taskcluster/taskcluster-infrastructure/tree/master/modules/docker-worker.

Taskcluster’s “livelog” functionality seems to be broken on Packet tasks, so logs are only available after a task is completed.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22046)
<!-- Reviewable:end -->

.taskcluster.yml

@@ -23,12 +23,8 @@ tasks:
           owner: &task_owner ${event.pusher.name}@users.noreply.github.com
           source: &task_source ${event.compare}
         scopes:
-          - "queue:scheduler-id:taskcluster-github"
-
           # Granted to role "repo:github.com/servo/servo:branch:*"
-          - "queue:create-task:highest:aws-provisioner-v1/servo-*"
-          - "queue:route:index.project.servo.servo.*"
-          - "docker-worker:cache:servo-*"
+          - "assume:project:servo:decision-task/trusted"
 
         payload:
           maxRunTime: {$eval: '20 * 60'}

etc/taskcluster/README.md

@@ -149,12 +149,24 @@ Servo admins have scope `auth:update-role:repo:github.com/servo/*` which allows
 to edit that role in the web UI and grant more scopes to these tasks
 (if that person has the new scope themselves).
 
+The [`project:servo:decision-task/base`][base]
+and [`project:servo:decision-task/trusted`][trusted] roles
+centralize the set of scopes granted to the decision task.
+This avoids maintaining them seprately in the `repo:…` roles,
+in the `hook-id:…` role,
+and in the `taskcluster.yml` file.
+Only the `base` role is granted to tasks executed when a pull request is opened.
+These tasks are less trusted because they run before the code has been reviewed,
+and anyone can open a PR.
+
 [Scopes]: https://docs.taskcluster.net/docs/manual/design/apis/hawk/scopes
 [web UI]: https://tools.taskcluster.net/
 [credentials]: https://tools.taskcluster.net/credentials
 [Roles]: https://docs.taskcluster.net/docs/manual/design/apis/hawk/roles
 [expand]: https://docs.taskcluster.net/docs/reference/platform/taskcluster-auth/docs/roles
 [branches]: https://tools.taskcluster.net/auth/roles/repo%3Agithub.com%2Fservo%2Fservo%3Abranch%3A*
+[base]: https://tools.taskcluster.net/auth/roles/project%3Aservo%3Adecision-task%2Fbase
+[trusted]: https://tools.taskcluster.net/auth/roles/project%3Aservo%3Adecision-task%2Ftrusted
 
 
 ## Daily tasks

etc/taskcluster/curl-artifact.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-task_id="${1}"
-artifact="${2}"
-shift 2
-queue="https://queue.taskcluster.net/v1"
-url="${queue}/task/${task_id}/artifacts/public/${artifact}"
-echo "Fetching ${url}" >&2
-curl \
-    --retry 5 \
-    --connect-timeout 10 \
-    --location \
-    --fail \
-    "${url}" \
-    "${@}"

etc/taskcluster/decision_task.py

@@ -13,6 +13,7 @@ def main(task_for, mock=False):
         if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]:
             linux_tidy_unit()
             android_arm32()
+            android_x86()
             windows_dev()
             if mock:
                 windows_release()
@@ -82,14 +83,8 @@ def with_rust_nightly():
 
 def android_arm32():
     return (
-        linux_build_task("Android ARMv7: build")
-        # file: NDK parses $(file $SHELL) to tell x64 host from x86
-        # wget: servo-media-gstreamer’s build script
-        .with_script("""
-            apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget
-            ./mach bootstrap-android --accept-all-licences --build
-            ./mach build --android --release
-        """)
+        android_build_task("Android ARMv7: release build")
+        .with_script("./mach build --android --release")
         .with_artifacts(
             "/repo/target/armv7-linux-androideabi/release/servoapp.apk",
             "/repo/target/armv7-linux-androideabi/release/servoview.aar",
@@ -98,6 +93,36 @@ def android_arm32():
     )
 
 
+def android_x86():
+    build_task = (
+        android_build_task("Android x86: release build")
+        .with_script("./mach build --target i686-linux-android --release")
+        .with_artifacts(
+            "/repo/target/i686-linux-android/release/servoapp.apk",
+            "/repo/target/i686-linux-android/release/servoview.aar",
+        )
+        .find_or_create("build.android_x86_release." + CONFIG.git_sha)
+    )
+    return (
+        DockerWorkerTask("Android x86: tests in emulator")
+        .with_provisioner_id("proj-servo")
+        .with_worker_type("docker-worker-kvm")
+        .with_capabilities(privileged=True)
+        .with_scopes("project:servo:docker-worker-kvm:capability:privileged")
+        .with_dockerfile(dockerfile_path("run-android-emulator"))
+        .with_repo()
+        .with_curl_artifact_script(build_task, "servoapp.apk", "target/i686-linux-android/release")
+        .with_script("""
+            ./mach bootstrap-android --accept-all-licences --emulator-x86
+            ./mach test-android-startup --release
+            ./mach test-wpt-android --release \
+                /_mozilla/mozilla/DOMParser.html \
+                /_mozilla/mozilla/webgl/context_creation_error.html
+        """)
+        .create()
+    )
+
+
 def windows_dev():
     return (
         windows_build_task("Windows x64: dev build + unit tests")
@@ -192,11 +217,9 @@ def linux_run_task(name, build_task, script):
     return (
         linux_task(name)
         .with_dockerfile(dockerfile_path("run"))
-        .with_early_script("""
-            ./etc/taskcluster/curl-artifact.sh ${BUILD_TASK_ID} target.tar.gz | tar -xz
-        """)
-        .with_env(BUILD_TASK_ID=build_task)
-        .with_dependencies(build_task)
+        .with_repo()
+        .with_curl_artifact_script(build_task, "target.tar.gz")
+        .with_script("tar -xzf target.tar.gz")
         .with_script(script)
         .with_index_and_artifacts_expire_in(log_artifacts_expire_in)
         .with_artifacts(*[
@@ -266,6 +289,19 @@ def linux_build_task(name):
     )
 
 
+def android_build_task(name):
+    return (
+        linux_build_task(name)
+        # file: NDK parses $(file $SHELL) to tell x64 host from x86
+        # wget: servo-media-gstreamer’s build script
+        .with_script("""
+            apt-get update -q
+            apt-get install -y --no-install-recommends openjdk-8-jdk-headless file wget
+            ./mach bootstrap-android --accept-all-licences --build
+        """)
+    )
+
+
 def windows_build_task(name):
     return (
         windows_task(name)

etc/taskcluster/decisionlib.py

@@ -512,7 +512,9 @@ class DockerWorkerTask(Task):
         self.env = {}
         self.caches = {}
         self.features = {}
+        self.capabilities = {}
         self.artifacts = []
+        self.curl_scripts_count = 0
 
     with_docker_image = chaining(setattr, "docker_image")
     with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes")
@@ -521,6 +523,7 @@ class DockerWorkerTask(Task):
     with_early_script = chaining(prepend_to_attr, "scripts")
     with_caches = chaining(update_attr, "caches")
     with_env = chaining(update_attr, "env")
+    with_capabilities = chaining(update_attr, "capabilities")
 
     def build_worker_payload(self):
         """
@@ -541,6 +544,7 @@ class DockerWorkerTask(Task):
             env=self.env,
             cache=self.caches,
             features=self.features,
+            capabilities=self.capabilities,
             artifacts={
                 "public/" + url_basename(path): {
                     "type": "file",
@@ -560,6 +564,28 @@ class DockerWorkerTask(Task):
         self.features.update({name: True for name in names})
         return self
 
+    def with_curl_script(self, url, file_path):
+        self.curl_scripts_count += 1
+        n = self.curl_scripts_count
+        return self \
+        .with_env(**{
+            "CURL_%s_URL" % n: url,
+            "CURL_%s_PATH" % n: file_path,
+        }) \
+        .with_script("""
+            mkdir -p $(dirname "$CURL_{n}_PATH")
+            curl --retry 5 --connect-timeout 10 -Lf "$CURL_{n}_URL" -o "$CURL_{n}_PATH"
+        """.format(n=n))
+
+    def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
+        return self \
+        .with_dependencies(task_id) \
+        .with_curl_script(
+            "https://queue.taskcluster.net/v1/task/%s/artifacts/public/%s"
+                % (task_id, artifact_name),
+            os.path.join(out_directory, url_basename(artifact_name)),
+        )
+
     def with_repo(self):
         """
         Make a shallow clone the git repository at the start of the task.

etc/taskcluster/docker/run-android-emulator.dockerfile

@@ -0,0 +1,11 @@
+% include base.dockerfile
+
+RUN \
+    apt-get install -qy --no-install-recommends \
+        #
+        # Multiple Android-related tools are in Java
+        openjdk-8-jdk-headless \
+        #
+        # Emulator dependencies
+        libgl1 \
+        libpulse0

etc/taskcluster/packet.net/.gitignore

@@ -0,0 +1,2 @@
+.terraform*
+terraform.tfstate*

etc/taskcluster/packet.net/README.md

@@ -0,0 +1,45 @@
+# docker-worker on Packet.net
+
+This is the configuration for the `proj-servo/docker-worker-kvm` worker type.
+It is similar to `aws-provisioner/docker-worker`,
+except that it runs on a server from Packet.net.
+This server is “real” non-virtualized hardware,
+so that Intel VT-x instructions are available and we can run KVM.
+KVM is required for the Android emulator’s CPU acceleration,
+which in turn is required to run OpenGL ES 3 (not just 2) in the guest system.
+
+## Setup
+
+* [Install Terraform](https://www.terraform.io/downloads.html)
+* [Install taskcluster-cli](https://github.com/taskcluster/taskcluster-cli/#installation)
+* Run ``eval `taskcluster signin` `` (once per open terminal/shell)
+* Run `./terraform_with_vars.py init` (once per checkout of the Servo repository)
+
+## List running servers
+
+* Run `./list_devices.py`
+
+## (Re)deploying a server
+
+* Run `./terraform_with_vars.py plan`
+* If the plan looks good, run `./terraform_with_vars.py apply`
+* Watch the new server being installed. Terraform should finish in 15~20 minutes.
+
+## Taskcluster secrets
+
+`terraform_with_vars.py` uses Taskcluster’s
+[secrets service](https://tools.taskcluster.net/secrets/).
+These secrets include an [authentication token](
+https://app.packet.net/projects/e3d0d8be-9e4c-4d39-90af-38660eb70544/settings/api-keys)
+for Packet.net’s API.
+You’ll need to authenticate with a Taskcluster client ID
+that has scope `secrets:get:project/servo/*`.
+This should be the case if you’re a Servo project administrator (the `project-admin:servo` role).
+
+## Worker’s client ID
+
+Workers are configured to authenticate with client ID
+[project/servo/worker/docker-worker-kvm/1](
+https://tools.taskcluster.net/auth/clients/project%2Fservo%2Fworker%2Fdocker-worker-kvm%2F1).
+This client has the scopes required to run docker-worker
+as well as for tasks that we run on this worker type.

etc/taskcluster/packet.net/docker-worker.tf

@@ -0,0 +1,29 @@
+module "docker_worker_packet" {
+  source = "github.com/servo/taskcluster-infrastructure//modules/docker-worker?ref=424ea4ff13de34df70e5242706fe1e26864cc383"
+
+  packet_project_id = "e3d0d8be-9e4c-4d39-90af-38660eb70544"
+  packet_instance_type = "t1.small.x86"
+  number_of_machines = "1"
+  concurrency = "1"
+
+  provisioner_id = "proj-servo"
+  worker_type = "docker-worker-kvm"
+  worker_group_prefix = "servo-packet"
+
+  taskcluster_client_id    = "${var.taskcluster_client_id}"
+  taskcluster_access_token = "${var.taskcluster_access_token}"
+  ssl_certificate          = "${var.ssl_certificate}"
+  cert_key                 = "${var.cert_key}"
+  ssh_pub_key              = "${var.ssh_pub_key}"
+  ssh_priv_key             = "${var.ssh_priv_key}"
+  private_key              = " "
+  relengapi_token          = " "
+  stateless_hostname       = " "
+}
+
+variable "taskcluster_client_id" {}
+variable "taskcluster_access_token" {}
+variable "ssl_certificate" {}
+variable "cert_key" {}
+variable "ssh_pub_key" {}
+variable "ssh_priv_key" {}

etc/taskcluster/packet.net/list_devices.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import sys
+import urllib.request
+
+import tc
+
+
+SERVO_PROJECT_ID = "e3d0d8be-9e4c-4d39-90af-38660eb70544"
+PACKET_AUTH_TOKEN = None
+
+
+def main():
+    tc.check()
+    global PACKET_AUTH_TOKEN
+    PACKET_AUTH_TOKEN = tc.packet_auth_token()
+    response = api_request("/projects/%s/devices?per_page=1000" % SERVO_PROJECT_ID)
+    for device in response["devices"]:
+        print(device["id"])
+        print("    Hostname:\t" + device["hostname"])
+        print("    Plan:\t" + device["plan"]["name"])
+        print("    OS: \t" + device["operating_system"]["name"])
+        for address in device["ip_addresses"]:
+            if address["public"]:
+                print("    IPv%s:\t%s" % (address["address_family"], address["address"]))
+        print("    Created:\t" + device["created_at"].replace("T", " "))
+        print("    Updated:\t" + device["updated_at"].replace("T", " "))
+    assert response["meta"]["next"] is None
+
+
+def api_request(path, json_data=None, method=None):
+    request = urllib.request.Request("https://api.packet.net" + path, method=method)
+    request.add_header("X-Auth-Token", PACKET_AUTH_TOKEN)
+    if json_data is not None:
+        request.add_header("Content-Type", "application/json")
+        request.data = json.dumps(json_data)
+    with urllib.request.urlopen(request) as response:
+        return json.load(response)
+
+
+if __name__ == "__main__":
+    main(*sys.argv[1:])

etc/taskcluster/packet.net/tc.py

@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import json
+import subprocess
+
+
+def check():
+    try:
+        subprocess.check_output(["taskcluster", "version"])
+    except FileNotFoundError:  # noqa: F821
+        sys.exit("taskcluster CLI tool not available. Install it from "
+                 "https://github.com/taskcluster/taskcluster-cli#installation")
+
+    if "TASKCLUSTER_CLIENT_ID" not in os.environ or "TASKCLUSTER_ACCESS_TOKEN" not in os.environ:
+        sys.exit("Taskcluster API credentials not available. Run this command and try again:\n\n"
+                 "eval `taskcluster signin`\n")
+
+
+def packet_auth_token():
+    return secret("project/servo/packet.net-api-key")["key"]
+
+
+def secret(name):
+    return api("secrets", "get", name)["secret"]
+
+
+def api(*args):
+    args = ["taskcluster", "api"] + list(args)
+    output = subprocess.check_output(args)
+    if output:
+        return json.loads(output)

etc/taskcluster/packet.net/terraform_with_vars.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import base64
+import subprocess
+
+import tc
+
+
+def main(*args):
+    tc.check()
+    ssh_key = tc.secret("project/servo/ssh-keys/docker-worker-kvm")
+    tc_creds = tc.secret("project/servo/tc-client/worker/docker-worker-kvm/1")
+    win2016 = tc.api("awsProvisioner", "workerType", "servo-win2016")
+    files_by_desc = {f.get("description"): f for f in win2016["secrets"]["files"]}
+
+    def decode(description):
+        f = files_by_desc[description]
+        assert f["encoding"] == "base64"
+        return base64.b64decode(f["content"])
+
+    terraform_vars = dict(
+        ssh_pub_key=ssh_key["public"],
+        ssh_priv_key=ssh_key["private"],
+        taskcluster_client_id=tc_creds["client_id"],
+        taskcluster_access_token=tc_creds["access_token"],
+        packet_api_key=tc.packet_auth_token(),
+        ssl_certificate=decode("SSL certificate for livelog"),
+        cert_key=decode("SSL key for livelog"),
+    )
+    env = dict(os.environ)
+    env["PACKET_AUTH_TOKEN"] = terraform_vars["packet_api_key"]
+    env.update({"TF_VAR_" + k: v for k, v in terraform_vars.items()})
+    sys.exit(subprocess.call(["terraform"] + list(args), env=env))
+
+
+if __name__ == "__main__":
+    main(*sys.argv[1:])