mirror of
https://github.com/servo/servo.git
synced 2025-07-03 21:43:41 +01:00
Start splitting generic and project-specific decision task logic
This commit is contained in:
parent
3e280b0fd4
commit
6a011f0e30
2 changed files with 161 additions and 150 deletions
151
decision-task.py
151
decision-task.py
|
@ -1,11 +1,6 @@
|
|||
# coding: utf8
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import taskcluster
|
||||
from decisionlib import *
|
||||
|
||||
|
||||
def main():
|
||||
|
@ -40,149 +35,5 @@ def main():
|
|||
)
|
||||
|
||||
|
||||
# https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
|
||||
IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
|
||||
"0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"
|
||||
|
||||
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
|
||||
DECISION_TASK_ID = os.environ["TASK_ID"]
|
||||
|
||||
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy
|
||||
QUEUE = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"})
|
||||
INDEX = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"})
|
||||
|
||||
DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4"
|
||||
DOCKER_IMAGE_CACHE_EXPIRY = "1 week"
|
||||
|
||||
REPO = os.path.dirname(__file__)
|
||||
|
||||
|
||||
def create_task_with_in_tree_dockerfile(*, image, **kwargs):
|
||||
image_build_task = build_image(image)
|
||||
kwargs.setdefault("dependencies", []).append(image_build_task)
|
||||
image = {
|
||||
"type": "task-image",
|
||||
"taskId": image_build_task,
|
||||
"path": "public/" + DOCKER_IMAGE_ARTIFACT_FILENAME,
|
||||
}
|
||||
return create_task(image=image, **kwargs)
|
||||
|
||||
|
||||
def build_image(image_name):
|
||||
with open(os.path.join(REPO, image_name + ".dockerfile"), "rb") as f:
|
||||
dockerfile = f.read()
|
||||
digest = hashlib.sha256(dockerfile).hexdigest()
|
||||
route = "project.servo.servo-taskcluster-experiments.docker-image." + digest
|
||||
|
||||
try:
|
||||
result = INDEX.findTask(route)
|
||||
return result["taskId"]
|
||||
except taskcluster.TaskclusterRestFailure as e:
|
||||
if e.status_code != 404:
|
||||
raise
|
||||
|
||||
image_build_task = create_task(
|
||||
task_name="docker image build task for image: " + image_name,
|
||||
command="""
|
||||
echo "$DOCKERFILE" | docker build -t taskcluster-built -
|
||||
docker save taskcluster-built | lz4 > /%s
|
||||
""" % DOCKER_IMAGE_ARTIFACT_FILENAME,
|
||||
env={
|
||||
"DOCKERFILE": dockerfile,
|
||||
},
|
||||
artifacts=[
|
||||
(
|
||||
DOCKER_IMAGE_ARTIFACT_FILENAME,
|
||||
"/" + DOCKER_IMAGE_ARTIFACT_FILENAME,
|
||||
DOCKER_IMAGE_CACHE_EXPIRY
|
||||
),
|
||||
],
|
||||
max_run_time_minutes=20,
|
||||
image=IMAGE_BUILDER_IMAGE,
|
||||
features={
|
||||
"dind": True, # docker-in-docker
|
||||
},
|
||||
with_repo=False,
|
||||
routes=[
|
||||
"index." + route,
|
||||
],
|
||||
extra={
|
||||
"index": {
|
||||
"expires": taskcluster.fromNowJSON(DOCKER_IMAGE_CACHE_EXPIRY),
|
||||
},
|
||||
},
|
||||
)
|
||||
return image_build_task
|
||||
|
||||
|
||||
def create_task(*, task_name, command, image, max_run_time_minutes,
|
||||
artifacts=None, dependencies=None, env=None, cache=None, scopes=None,
|
||||
routes=None, extra=None, features=None,
|
||||
with_repo=True):
|
||||
env = env or {}
|
||||
|
||||
if with_repo:
|
||||
for k in ["GITHUB_EVENT_CLONE_URL", "GITHUB_EVENT_COMMIT_SHA"]:
|
||||
env.setdefault(k, os.environ[k])
|
||||
|
||||
command = """
|
||||
git clone $GITHUB_EVENT_CLONE_URL repo
|
||||
cd repo
|
||||
git checkout $GITHUB_EVENT_COMMIT_SHA
|
||||
""" + command
|
||||
|
||||
payload = {
|
||||
"taskGroupId": DECISION_TASK_ID,
|
||||
"dependencies": [DECISION_TASK_ID] + (dependencies or []),
|
||||
"schedulerId": "taskcluster-github",
|
||||
"provisionerId": "aws-provisioner-v1",
|
||||
"workerType": "servo-docker-worker",
|
||||
|
||||
"created": taskcluster.fromNowJSON(""),
|
||||
"deadline": taskcluster.fromNowJSON("1 day"),
|
||||
"metadata": {
|
||||
"name": "Taskcluster experiments for Servo: " + task_name,
|
||||
"description": "",
|
||||
"owner": os.environ["GITHUB_EVENT_OWNER"],
|
||||
"source": os.environ["GITHUB_EVENT_SOURCE"],
|
||||
},
|
||||
"scopes": scopes or [],
|
||||
"routes": routes or [],
|
||||
"extra": extra or {},
|
||||
"payload": {
|
||||
"cache": cache or {},
|
||||
"maxRunTime": max_run_time_minutes * 60,
|
||||
"image": image,
|
||||
"command": [
|
||||
"/bin/bash",
|
||||
"--login",
|
||||
"-x",
|
||||
"-e",
|
||||
"-c",
|
||||
deindent(command)
|
||||
],
|
||||
"env": env,
|
||||
"artifacts": {
|
||||
"public/" + artifact_name: {
|
||||
"type": "file",
|
||||
"path": path,
|
||||
"expires": taskcluster.fromNowJSON(expires),
|
||||
}
|
||||
for artifact_name, path, expires in artifacts or []
|
||||
},
|
||||
"features": features or {},
|
||||
},
|
||||
}
|
||||
|
||||
task_id = taskcluster.slugId().decode("utf8")
|
||||
QUEUE.createTask(task_id, payload)
|
||||
print("Scheduled %s: %s" % (task_name, task_id))
|
||||
return task_id
|
||||
|
||||
|
||||
def deindent(string):
|
||||
return re.sub("\n +", "\n ", string)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue