Auto merge of #22185 - servo:tc-mac, r=Manishearth

Taskcluster: add dev build and unit tests on macOS

For simplicity, Salt is used in [agentless](https://docs.saltstack.com/en/getstarted/ssh/) mode for now.

I’m also hoping to have most of the dependencies be installed by tasks for themselves, so that changes can be made in the same `servo/servo` PR that uses them without redeploying servers.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22185)
<!-- Reviewable:end -->

etc/taskcluster/decision_task.py

@@ -13,7 +13,8 @@ def main(task_for, mock=False):
         if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]:
             linux_tidy_unit()
             android_arm32()
-            windows_dev()
+            windows_unit()
+            macos_unit()
             if mock:
                 windows_release()
                 linux_wpt()
@@ -40,12 +41,15 @@ build_env = {
     "RUSTFLAGS": "-Dwarnings",
     "CARGO_INCREMENTAL": "0",
 }
-linux_build_env = {
+unix_build_env = {
     "CCACHE": "sccache",
     "RUSTC_WRAPPER": "sccache",
     "SCCACHE_IDLE_TIMEOUT": "1200",
+}
+linux_build_env = {
     "SHELL": "/bin/dash",  # For SpiderMonkey’s build system
 }
+macos_build_env = {}
 windows_build_env = {
     "LIB": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\lib;%LIB%",
 }
@@ -73,6 +77,15 @@ def linux_tidy_unit():
     """).create()
 
 
+def macos_unit():
+    macos_build_task("macOS x64: dev build + unit tests").with_script("""
+        ./mach build --dev
+        ./mach test-unit
+        ./mach package --dev
+        ./etc/ci/lockfile_changed.sh
+    """).create()
+
+
 def with_rust_nightly():
     return linux_build_task("Linux x64: with Rust Nightly").with_script("""
         echo "nightly" > rust-toolchain
@@ -123,7 +136,7 @@ def android_x86():
     )
 
 
-def windows_dev():
+def windows_unit():
     return (
         windows_build_task("Windows x64: dev build + unit tests")
         .with_script(
@@ -268,6 +281,14 @@ def windows_task(name):
     return WindowsGenericWorkerTask(name).with_worker_type("servo-win2016")
 
 
+def macos_task(name):
+    return (
+        MacOsGenericWorkerTask(name)
+        .with_provisioner_id("proj-servo")
+        .with_worker_type("macos")
+    )
+
+
 def linux_build_task(name):
     return (
         linux_task(name)
@@ -283,7 +304,7 @@ def linux_build_task(name):
         .with_index_and_artifacts_expire_in(build_artifacts_expire_in)
         .with_max_run_time_minutes(60)
         .with_dockerfile(dockerfile_path("build"))
-        .with_env(**build_env, **linux_build_env)
+        .with_env(**build_env, **unix_build_env, **linux_build_env)
         .with_repo()
         .with_index_and_artifacts_expire_in(build_artifacts_expire_in)
     )
@@ -325,6 +346,48 @@ def windows_build_task(name):
     )
 
 
+def macos_build_task(name):
+    return (
+        macos_task(name)
+        # Allow long runtime in case the cache expired for all those Homebrew dependencies
+        .with_max_run_time_minutes(60 * 2)
+        .with_env(**build_env, **unix_build_env, **macos_build_env)
+        .with_repo()
+        .with_python2()
+        .with_rustup()
+        .with_script("""
+            mkdir -p "$HOME/homebrew"
+            export PATH="$HOME/homebrew/bin:$PATH"
+            which homebrew || curl -L https://github.com/Homebrew/brew/tarball/master \
+                | tar xz --strip 1 -C "$HOME/homebrew"
+
+            time brew install automake autoconf@2.13 pkg-config cmake yasm llvm
+            time brew install gstreamer gst-plugins-base gst-plugins-good gst-plugins-bad \
+                gst-libav gst-rtsp-server \
+                --with-orc --with-libogg --with-opus --with-pango --with-theora --with-libvorbis
+            export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include"
+            export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib"
+        """)
+
+        .with_directory_mount(
+            "https://github.com/mozilla/sccache/releases/download/"
+                "0.2.7/sccache-0.2.7-x86_64-apple-darwin.tar.gz",
+            sha256="f86412abbbcce2d3f23e7d33305469198949f5cf807e6c3258c9e1885b4cb57f",
+            path="sccache",
+        )
+        # Early script in order to run with the initial $PWD
+        .with_early_script("""
+            export PATH="$PWD/sccache/sccache-0.2.7-x86_64-apple-darwin:$PATH"
+        """)
+        # sccache binaries requires OpenSSL 1.1 and are not compatible with 1.0.
+        # "Late" script in order to run after Homebrew is installed.
+        .with_script("""
+            time brew install openssl@1.1
+            export DYLD_LIBRARY_PATH="$HOME/homebrew/opt/openssl@1.1/lib"
+        """)
+    )
+
+
 CONFIG.task_name_template = "Servo: %s"
 CONFIG.index_prefix = "project.servo.servo"
 CONFIG.docker_image_buil_worker_type = "servo-docker-worker"

etc/taskcluster/decisionlib.py

@@ -27,7 +27,7 @@ import taskcluster
 # Public API
 __all__ = [
     "CONFIG", "SHARED", "Task", "DockerWorkerTask",
-    "GenericWorkerTask", "WindowsGenericWorkerTask",
+    "GenericWorkerTask", "WindowsGenericWorkerTask", "MacOsGenericWorkerTask",
 ]
 
 
@@ -496,6 +496,59 @@ class WindowsGenericWorkerTask(GenericWorkerTask):
         .with_path_from_homedir("python2", "python2\\Scripts")
 
 
+class MacOsGenericWorkerTask(GenericWorkerTask):
+    """
+    Task definition for a `generic-worker` task running on macOS.
+
+    Scripts are interpreted with `bash`.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.scripts = []
+
+    with_script = chaining(append_to_attr, "scripts")
+    with_early_script = chaining(prepend_to_attr, "scripts")
+
+    def build_command(self):
+        # generic-worker accepts multiple commands, but unlike on Windows
+        # the current directory and environment variables
+        # are not preserved across commands on macOS.
+        # So concatenate scripts and use a single `bash` command instead.
+        return [
+            [
+                "/bin/bash", "--login", "-x", "-e", "-c",
+                deindent("\n".join(self.scripts))
+            ]
+        ]
+
+    def with_repo(self):
+        """
+        Make a shallow clone the git repository at the start of the task.
+        This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`,
+        and creates the clone in a `repo` directory in the task’s directory.
+        """
+        return self \
+        .with_env(**git_env()) \
+        .with_early_script("""
+            git init repo
+            cd repo
+            git fetch --depth 1 "$GIT_URL" "$GIT_REF"
+            git reset --hard "$GIT_SHA"
+        """)
+
+    def with_python2(self):
+        return self.with_early_script("""
+            export PATH="$HOME/Library/Python/2.7/bin:$PATH"
+            python -m ensurepip --user
+            pip install --user virtualenv
+        """)
+
+    def with_rustup(self):
+        return self.with_early_script("""
+            export PATH="$HOME/.cargo/bin:$PATH"
+            which rustup || curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y
+        """)
+
 
 class DockerWorkerTask(Task):
     """

etc/taskcluster/macos/.gitignore

@@ -0,0 +1 @@
+.salt

etc/taskcluster/macos/README.md

@@ -0,0 +1,49 @@
+# macOS
+
+This is the configuration for the `proj-servo/macos` worker type.
+These macOS workers are configured with SaltStack in [agentless] mode.
+
+[agentless]: https://docs.saltstack.com/en/getstarted/ssh/index.html
+
+Either run `./salt-ssh`
+to automatically install `salt-ssh` in `mach`’s existing Python virtualenv,
+or install `salt-ssh` through some other mean and run in from this directory.
+
+```sh
+cd etc/taskcluster/macos
+./salt-ssh '*' test.ping
+./salt-ssh '*' state.apply test=True
+```
+
+## (Re)deploying a server
+
+* Place an order or file a ticket with MacStadium to get a new hardware or reinstall an OS.
+
+* Change the administrator password to one generated with
+  `</dev/urandom tr -d -c 'a-zA-Z' | head -c 8; echo`
+  (this short because of VNC),
+  and save it in the shared 1Password account.
+
+* Give the public IPv4 address a DNS name through Cloudflare.
+
+* Add a correponding entry in the `config/roster` file.
+
+* Log in through VNC, and run `xcode-select --install`
+
+
+## Taskcluster secrets
+
+This SaltStack configuration has a custom module that uses Taskcluster’s
+[secrets service](https://tools.taskcluster.net/secrets/).
+These secrets include an [authentication token](
+You’ll need to authenticate with a Taskcluster client ID
+that has scope `secrets:get:project/servo/*`.
+This should be the case if you’re a Servo project administrator (the `project-admin:servo` role).
+
+
+## Worker’s client ID
+
+Workers are configured to authenticate with client ID
+[`project/servo/worker/macos/1`](
+https://tools.taskcluster.net/auth/clients/project%2Fservo%2Fworker%macos%2F1).
+This client has the scopes required to run tasks for this worker type.

etc/taskcluster/macos/Saltfile

@@ -0,0 +1,3 @@
+salt-ssh:
+  config_dir: ./config
+  state_verbose: False

etc/taskcluster/macos/config/master

@@ -0,0 +1,13 @@
+root_dir: .salt
+file_roots:
+  base:
+    - states
+extension_modules: ../modules
+ext_pillar:
+  - taskcluster_secrets:
+roster_defaults:
+    # https://github.com/saltstack/salt/issues/50477
+    minion_opts:
+        providers:
+            user: mac_user
+            group: mac_group

etc/taskcluster/macos/config/roster

@@ -0,0 +1,2 @@
+mac1:
+    host: servo-tc-mac1.servo.org

etc/taskcluster/macos/modules/pillar/taskcluster_secrets.py

@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "packet.net"))
+import tc
+
+
+def ext_pillar(_minion_id, _pillar, *_args):
+    tc.check()
+    data = {}
+    data.update(tc.secret("project/servo/tc-client/worker/macos/1"))
+    data.update(tc.livelog())
+    return data

etc/taskcluster/macos/salt-ssh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cd "$(dirname $0)"
+VENV_BIN="../../../python/_virtualenv/bin"
+[ -x "${VENV_BIN}/salt-ssh" ] || "${VENV_BIN}/pip" install salt-ssh
+"${VENV_BIN}/salt-ssh" "${@}"

etc/taskcluster/macos/states/generic-worker.plist.jinja

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>             <string>net.generic.worker</string>
+    <key>KeepAlive</key>         <true/>
+    <key>StandardOutPath</key>   <string>stdout.log</string>
+    <key>StandardErrorPath</key> <string>stderr.log</string>
+    <key>WorkingDirectory</key>  <string>{{ home }}</string>
+    <key>UserName</key>          <string>{{ user }}</string>
+    <key>ProgramArguments</key>  <array>
+        <string>{{ bin }}/generic-worker</string>
+        <string>run</string>
+        <string>--config</string>
+        <string>{{ etc }}/config.json</string>
+    </array>
+</dict>
+</plist>

etc/taskcluster/macos/states/generic-worker.sls

@@ -0,0 +1,92 @@
+{% set bin = "/usr/local/bin" %}
+{% set etc = "/etc/generic-worker" %}
+{% set user = "worker" %}
+{% set home = "/Users/" + user %}
+
+GMT:
+  timezone.system
+
+{{ bin }}/generic-worker:
+  file.managed:
+    - name:
+    - source: https://github.com/taskcluster/generic-worker/releases/download/v11.0.1/generic-worker-darwin-amd64
+    - source_hash: sha256=059331865670d3722a710f0b6f4dae97d347811cc347d1810c6dfc1b413c4b48
+    - mode: 755
+    - makedirs: True
+
+{{ bin }}/livelog:
+  file.managed:
+    - source: https://github.com/taskcluster/livelog/releases/download/v1.1.0/livelog-darwin-amd64
+    - source_hash: sha256=be5d4b998b208afd802ac6ce6c4d4bbf0fb3816bb039a300626abbc999dfe163
+    - mode: 755
+    - makedirs: True
+
+{{ user }} group:
+  group.present:
+    - name: {{ user }}
+
+{{ user }}:
+  user.present:
+    - home: {{ home }}
+    - gid_from_name: True
+
+# `user.present`’s `createhome` is apparently not supported on macOS
+{{ home }}:
+  file.directory:
+    - user: {{ user }}
+
+{{ etc }}/config.json:
+  file.serialize:
+    - makedirs: True
+    - group: {{ user }}
+    - mode: 640
+    - show_changes: False
+    - formatter: json
+    - dataset:
+        provisionerId: proj-servo
+        workerType: macos
+        workerGroup: servo-macos
+        workerId: mac1
+        tasksDir: {{ home }}/tasks
+        publicIP: {{ salt.network.ip_addrs()[0] }}
+        signingKeyLocation: {{ home }}/key
+        clientId: {{ pillar["client_id"] }}
+        accessToken: {{ pillar["access_token"] }}
+        livelogExecutable: {{ bin }}/livelog
+        livelogCertificate: {{ etc }}/livelog.crt
+        livelogKey: {{ etc }}/livelog.key
+        livelogSecret: {{ pillar["livelog_secret"] }}
+    - watch_in:
+      - service: net.generic.worker
+
+{{ etc }}/livelog.crt:
+  file.managed:
+    - contents_pillar: livelog_cert
+    - group: {{ user }}
+    - mode: 640
+
+{{ etc }}/livelog.key:
+  file.managed:
+    - contents_pillar: livelog_key
+    - group: {{ user }}
+    - mode: 640
+
+{{ bin }}/generic-worker new-openpgp-keypair --file {{ home }}/key:
+  cmd.run:
+    - creates: {{ home }}/key
+    - runas: {{ user }}
+
+/Library/LaunchAgents/net.generic.worker.plist:
+  file.managed:
+    - mode: 644
+    - template: jinja
+    - source: salt://generic-worker.plist.jinja
+    - context:
+      bin: {{ bin }}
+      etc: {{ etc }}
+      home: {{ home }}
+      user: {{ user }}
+
+net.generic.worker:
+  service.running:
+    - enable: True

etc/taskcluster/macos/states/top.sls

@@ -0,0 +1,3 @@
+base:
+  'mac*':
+    - generic-worker

etc/taskcluster/packet.net/README.md

@@ -39,7 +39,7 @@ This should be the case if you’re a Servo project administrator (the `project-
 ## Worker’s client ID
 
 Workers are configured to authenticate with client ID
-[project/servo/worker/docker-worker-kvm/1](
+[`project/servo/worker/docker-worker-kvm/1`](
 https://tools.taskcluster.net/auth/clients/project%2Fservo%2Fworker%2Fdocker-worker-kvm%2F1).
 This client has the scopes required to run docker-worker
 as well as for tasks that we run on this worker type.

etc/taskcluster/packet.net/tc.py

@@ -5,6 +5,7 @@
 import os
 import sys
 import json
+import base64
 import subprocess
 
 
@@ -20,6 +21,18 @@ def check():
                  "eval `taskcluster signin`\n")
 
 
+def livelog():
+    win2016 = api("awsProvisioner", "workerType", "servo-win2016")
+    files = win2016["secrets"]["files"]
+    assert all(f["encoding"] == "base64" for f in files)
+    files = {f.get("description"): f["content"] for f in files}
+    return {
+        "livelog_cert": base64.b64decode(files["SSL certificate for livelog"]),
+        "livelog_key": base64.b64decode(files["SSL key for livelog"]),
+        "livelog_secret": win2016["secrets"]["generic-worker"]["config"]["livelogSecret"],
+    }
+
+
 def packet_auth_token():
     return secret("project/servo/packet.net-api-key")["key"]
 

etc/taskcluster/packet.net/terraform_with_vars.py

@@ -6,7 +6,6 @@
 
 import os
 import sys
-import base64
 import subprocess
 
 import tc
@@ -16,13 +15,7 @@ def main(*args):
     tc.check()
     ssh_key = tc.secret("project/servo/ssh-keys/docker-worker-kvm")
     tc_creds = tc.secret("project/servo/tc-client/worker/docker-worker-kvm/1")
-    win2016 = tc.api("awsProvisioner", "workerType", "servo-win2016")
+    livelog = tc.livelog()
-    files_by_desc = {f.get("description"): f for f in win2016["secrets"]["files"]}
-
-    def decode(description):
-        f = files_by_desc[description]
-        assert f["encoding"] == "base64"
-        return base64.b64decode(f["content"])
 
     terraform_vars = dict(
         ssh_pub_key=ssh_key["public"],
@@ -30,8 +23,8 @@ def main(*args):
         taskcluster_client_id=tc_creds["client_id"],
         taskcluster_access_token=tc_creds["access_token"],
         packet_api_key=tc.packet_auth_token(),
-        ssl_certificate=decode("SSL certificate for livelog"),
+        ssl_certificate=livelog["livelog_cert_base64"],
-        cert_key=decode("SSL key for livelog"),
+        cert_key=livelog["livelog_key_base64"],
     )
     env = dict(os.environ)
     env["PACKET_AUTH_TOKEN"] = terraform_vars["packet_api_key"]