Add Terraform config for docker-worker on Packet.net

etc/taskcluster/packet.net/.gitignore

@@ -0,0 +1,2 @@
+.terraform*
+terraform.tfstate*

etc/taskcluster/packet.net/README.md

@@ -0,0 +1,45 @@
+# docker-worker on Packet.net
+
+This is the configuration for the `proj-servo/docker-worker-kvm` worker type.
+It is similar to `aws-provisioner/docker-worker`,
+except that it runs on a server from Packet.net.
+This server is “real” non-virtualized hardware,
+so that Intel VT-x instructions are available and we can run KVM.
+KVM is required for the Android emulator’s CPU acceleration,
+which in turn is required to run OpenGL ES 3 (not just 2) in the guest system.
+
+## Setup
+
+* [Install Terraform](https://www.terraform.io/downloads.html)
+* [Install taskcluster-cli](https://github.com/taskcluster/taskcluster-cli/#installation)
+* Run ``eval `taskcluster signin` `` (once per open terminal/shell)
+* Run `./terraform_with_vars.py init` (once per checkout of the Servo repository)
+
+## List running servers
+
+* Run `./list_devices.py`
+
+## (Re)deploying a server
+
+* Run `./terraform_with_vars.py plan`
+* If the plan looks good, run `./terraform_with_vars.py apply`
+* Watch the new server being installed. Terraform should finish in 15~20 minutes.
+
+## Taskcluster secrets
+
+`terraform_with_vars.py` uses Taskcluster’s
+[secrets service](https://tools.taskcluster.net/secrets/).
+These secrets include an [authentication token](
+https://app.packet.net/projects/e3d0d8be-9e4c-4d39-90af-38660eb70544/settings/api-keys)
+for Packet.net’s API.
+You’ll need to authenticate with a Taskcluster client ID
+that has scope `secrets:get:project/servo/*`.
+This should be the case if you’re a Servo project administrator (the `project-admin:servo` role).
+
+## Worker’s client ID
+
+Workers are configured to authenticate with client ID
+[project/servo/worker/docker-worker-kvm/1](
+https://tools.taskcluster.net/auth/clients/project%2Fservo%2Fworker%2Fdocker-worker-kvm%2F1).
+This client has the scopes required to run docker-worker
+as well as for tasks that we run on this worker type.

etc/taskcluster/packet.net/docker-worker.tf

@@ -0,0 +1,29 @@
+module "docker_worker_packet" {
+  source = "github.com/servo/taskcluster-infrastructure//modules/docker-worker?ref=424ea4ff13de34df70e5242706fe1e26864cc383"
+
+  packet_project_id = "e3d0d8be-9e4c-4d39-90af-38660eb70544"
+  packet_instance_type = "t1.small.x86"
+  number_of_machines = "1"
+  concurrency = "1"
+
+  provisioner_id = "proj-servo"
+  worker_type = "docker-worker-kvm"
+  worker_group_prefix = "servo-packet"
+
+  taskcluster_client_id    = "${var.taskcluster_client_id}"
+  taskcluster_access_token = "${var.taskcluster_access_token}"
+  ssl_certificate          = "${var.ssl_certificate}"
+  cert_key                 = "${var.cert_key}"
+  ssh_pub_key              = "${var.ssh_pub_key}"
+  ssh_priv_key             = "${var.ssh_priv_key}"
+  private_key              = " "
+  relengapi_token          = " "
+  stateless_hostname       = " "
+}
+
+variable "taskcluster_client_id" {}
+variable "taskcluster_access_token" {}
+variable "ssl_certificate" {}
+variable "cert_key" {}
+variable "ssh_pub_key" {}
+variable "ssh_priv_key" {}

etc/taskcluster/packet.net/list_devices.py

@@ -5,20 +5,21 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
-import os
 import sys
 import urllib.request
 
+import tc
+
 
 SERVO_PROJECT_ID = "e3d0d8be-9e4c-4d39-90af-38660eb70544"
+PACKET_AUTH_TOKEN = None
 
 
 def main():
-    auth_token = os.environ.get("PACKET_AUTH_TOKEN")
-    if not auth_token:
-        sys.exit("$PACKET_AUTH_TOKEN is not set. See:\n"
-                 "https://app.packet.net/projects/%s/settings/api-keys" % SERVO_PROJECT_ID)
-    response = api_request(auth_token, "/projects/%s/devices?per_page=1000" % SERVO_PROJECT_ID)
+    tc.check()
+    global PACKET_AUTH_TOKEN
+    PACKET_AUTH_TOKEN = tc.packet_auth_token()
+    response = api_request("/projects/%s/devices?per_page=1000" % SERVO_PROJECT_ID)
     for device in response["devices"]:
         print(device["id"])
         print("  Host:\t" + device["hostname"])
@@ -30,9 +31,9 @@ def main():
     assert response["meta"]["next"] is None
 
 
-def api_request(auth_token, path, json_data=None, method=None):
+def api_request(path, json_data=None, method=None):
     request = urllib.request.Request("https://api.packet.net" + path, method=method)
-    request.add_header("X-Auth-Token", auth_token)
+    request.add_header("X-Auth-Token", PACKET_AUTH_TOKEN)
     if json_data is not None:
         request.add_header("Content-Type", "application/json")
         request.data = json.dumps(json_data)

etc/taskcluster/packet.net/tc.py

@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import json
+import subprocess
+
+
+def check():
+    try:
+        subprocess.check_output(["taskcluster", "version"])
+    except FileNotFoundError:  # noqa: F821
+        sys.exit("taskcluster CLI tool not available. Install it from "
+                 "https://github.com/taskcluster/taskcluster-cli#installation")
+
+    if "TASKCLUSTER_CLIENT_ID" not in os.environ or "TASKCLUSTER_ACCESS_TOKEN" not in os.environ:
+        sys.exit("Taskcluster API credentials not available. Run this command and try again:\n\n"
+                 "eval `taskcluster signin`\n")
+
+
+def packet_auth_token():
+    return secret("project/servo/packet.net-api-key")["key"]
+
+
+def secret(name):
+    return api("secrets", "get", name)["secret"]
+
+
+def api(*args):
+    args = ["taskcluster", "api"] + list(args)
+    output = subprocess.check_output(args)
+    if output:
+        return json.loads(output)

etc/taskcluster/packet.net/terraform_with_vars.py

@@ -0,0 +1,43 @@
+#!/usr/bin/python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+import base64
+import subprocess
+
+import tc
+
+
+def main(*args):
+    tc.check()
+    ssh_key = tc.secret("project/servo/ssh-keys/docker-worker-kvm")
+    tc_creds = tc.secret("project/servo/tc-client/worker/docker-worker-kvm/1")
+    win2016 = tc.api("awsProvisioner", "workerType", "servo-win2016")
+    files_by_desc = {f.get("description"): f for f in win2016["secrets"]["files"]}
+
+    def decode(description):
+        f = files_by_desc[description]
+        assert f["encoding"] == "base64"
+        return base64.b64decode(f["content"])
+
+    terraform_vars = dict(
+        ssh_pub_key=ssh_key["public"],
+        ssh_priv_key=ssh_key["private"],
+        taskcluster_client_id=tc_creds["client_id"],
+        taskcluster_access_token=tc_creds["access_token"],
+        packet_api_key=tc.packet_auth_token(),
+        ssl_certificate=decode("SSL certificate for livelog"),
+        cert_key=decode("SSL key for livelog"),
+    )
+    env = dict(os.environ)
+    env["PACKET_AUTH_TOKEN"] = terraform_vars["packet_api_key"]
+    env.update({"TF_VAR_" + k: v for k, v in terraform_vars.items()})
+    sys.exit(subprocess.call(["terraform"] + list(args), env=env))
+
+
+if __name__ == "__main__":
+    main(*sys.argv[1:])