Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-07-05 22:43:40 +01:00
Code Issues Projects Releases Packages Wiki Activity

Move project-specific stuff out of decisionlib

Browse source
This commit is contained in:
Simon Sapin 2018-09-13 16:52:05 +02:00
parent e566906bef
commit 46ed832b21
2 changed files with 54 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

12
decision-task.py
View file

@ -1,15 +1,19 @@
# coding: utf8 # coding: utf8
import os.path
from decisionlib import DecisionTask from decisionlib import DecisionTask
def main(): def main():
decision = DecisionTask() decision = DecisionTask(
project_name="Taskcluster experimenfts for Servo", # Used in task names
docker_image_cache_expiry="1 week",
)
build_task = decision.create_task_with_in_tree_dockerfile( build_task = decision.create_task_with_in_tree_dockerfile(
task_name="build task", task_name="build task",
command="./build-task.sh", command="./build-task.sh",
image="servo-x86_64-linux", dockerfile=dockerfile("servo-x86_64-linux"),
max_run_time_minutes=20, max_run_time_minutes=20,
artifacts=[ artifacts=[
@ -37,5 +41,9 @@ def main():
) )
def dockerfile(name):
return os.path.join(os.path.dirname(__file__), name + ".dockerfile")
if __name__ == "__main__": if __name__ == "__main__":
main() main()

83
decisionlib.py
View file

@ -12,70 +12,62 @@ import sys
import taskcluster import taskcluster
# Used in task names
PROJECT_NAME = "Taskcluster experiments for Servo"
DOCKER_IMAGE_CACHE_EXPIRY = "1 week"
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
DECISION_TASK_ID = os.environ["TASK_ID"]
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy
QUEUE = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"})
INDEX = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"})
# https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
DOCKER_IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
"0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"
DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4"
REPO = os.path.dirname(__file__)
class DecisionTask: class DecisionTask:
def create_task_with_in_tree_dockerfile(self, *, image, **kwargs): DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4"
image_build_task = self.build_image(image)
# https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
DOCKER_IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
"0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"
def __init__(self, project_name, docker_image_cache_expiry="1 year"):
self.project_name = project_name
self.docker_image_cache_expiry = docker_image_cache_expiry
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy
self.queue_service = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"})
self.index_service = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"})
def create_task_with_in_tree_dockerfile(self, *, dockerfile, **kwargs):
image_build_task = self.build_image(dockerfile)
kwargs.setdefault("dependencies", []).append(image_build_task) kwargs.setdefault("dependencies", []).append(image_build_task)
image = { image = {
"type": "task-image", "type": "task-image",
"taskId": image_build_task, "taskId": image_build_task,
"path": "public/" + DOCKER_IMAGE_ARTIFACT_FILENAME, "path": "public/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME,
} }
return self.create_task(image=image, **kwargs) return self.create_task(image=image, **kwargs)
def build_image(self, dockerfile):
def build_image(self, image_name): with open(dockerfile, "rb") as f:
with open(os.path.join(REPO, image_name + ".dockerfile"), "rb") as f:
dockerfile = f.read() dockerfile = f.read()
digest = hashlib.sha256(dockerfile).hexdigest() digest = hashlib.sha256(dockerfile).hexdigest()
route = "project.servo.servo-taskcluster-experiments.docker-image." + digest route = "project.servo.servo-taskcluster-experiments.docker-image." + digest
try: try:
result = INDEX.findTask(route) result = self.index_service.findTask(route)
return result["taskId"] return result["taskId"]
except taskcluster.TaskclusterRestFailure as e: except taskcluster.TaskclusterRestFailure as e:
if e.status_code != 404: if e.status_code != 404:
raise raise
image_build_task = self.create_task( image_build_task = self.create_task(
task_name="docker image build task for image: " + image_name, task_name="docker image build task for image: " + self.image_name(dockerfile),
command=""" command="""
echo "$DOCKERFILE" | docker build -t taskcluster-built - echo "$DOCKERFILE" | docker build -t taskcluster-built -
docker save taskcluster-built | lz4 > /%s docker save taskcluster-built | lz4 > /%s
""" % DOCKER_IMAGE_ARTIFACT_FILENAME, """ % self.DOCKER_IMAGE_ARTIFACT_FILENAME,
env={ env={
"DOCKERFILE": dockerfile, "DOCKERFILE": dockerfile,
}, },
artifacts=[ artifacts=[
( (
DOCKER_IMAGE_ARTIFACT_FILENAME, self.DOCKER_IMAGE_ARTIFACT_FILENAME,
"/" + DOCKER_IMAGE_ARTIFACT_FILENAME, "/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME,
DOCKER_IMAGE_CACHE_EXPIRY self.docker_image_cache_expiry
), ),
], ],
max_run_time_minutes=20, max_run_time_minutes=20,
image=DOCKER_IMAGE_BUILDER_IMAGE, image=self.DOCKER_IMAGE_BUILDER_IMAGE,
features={ features={
"dind": True, # docker-in-docker "dind": True, # docker-in-docker
}, },
@ -85,12 +77,21 @@ class DecisionTask:
], ],
extra={ extra={
"index": { "index": {
"expires": taskcluster.fromNowJSON(DOCKER_IMAGE_CACHE_EXPIRY), "expires": taskcluster.fromNowJSON(self.docker_image_cache_expiry),
}, },
}, },
) )
return image_build_task return image_build_task
def image_name(self, dockerfile):
basename = os.path.basename(dockerfile)
suffix = ".dockerfile"
if basename == "Dockerfile":
return os.path.basename(os.path.dirname(os.path.abspath(dockerfile)))
elif basename.endswith(suffix):
return basename[:-len(suffix)]
else:
return basename
def create_task(self, *, task_name, command, image, max_run_time_minutes, def create_task(self, *, task_name, command, image, max_run_time_minutes,
artifacts=None, dependencies=None, env=None, cache=None, scopes=None, artifacts=None, dependencies=None, env=None, cache=None, scopes=None,
@ -100,6 +101,7 @@ class DecisionTask:
if with_repo: if with_repo:
for k in ["GITHUB_EVENT_CLONE_URL", "GITHUB_EVENT_COMMIT_SHA"]: for k in ["GITHUB_EVENT_CLONE_URL", "GITHUB_EVENT_COMMIT_SHA"]:
# Set in .taskcluster.yml
env.setdefault(k, os.environ[k]) env.setdefault(k, os.environ[k])
command = """ command = """
@ -108,9 +110,12 @@ class DecisionTask:
git checkout $GITHUB_EVENT_COMMIT_SHA git checkout $GITHUB_EVENT_COMMIT_SHA
""" + command """ + command
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
decision_task_id = os.environ["TASK_ID"]
payload = { payload = {
"taskGroupId": DECISION_TASK_ID, "taskGroupId": decision_task_id,
"dependencies": [DECISION_TASK_ID] + (dependencies or []), "dependencies": [decision_task_id] + (dependencies or []),
"schedulerId": "taskcluster-github", "schedulerId": "taskcluster-github",
"provisionerId": "aws-provisioner-v1", "provisionerId": "aws-provisioner-v1",
"workerType": "servo-docker-worker", "workerType": "servo-docker-worker",
@ -118,7 +123,7 @@ class DecisionTask:
"created": taskcluster.fromNowJSON(""), "created": taskcluster.fromNowJSON(""),
"deadline": taskcluster.fromNowJSON("1 day"), "deadline": taskcluster.fromNowJSON("1 day"),
"metadata": { "metadata": {
"name": "%s: %s" % (PROJECT_NAME, task_name), "name": "%s: %s" % (self.project_name, task_name),
"description": "", "description": "",
"owner": os.environ["GITHUB_EVENT_OWNER"], "owner": os.environ["GITHUB_EVENT_OWNER"],
"source": os.environ["GITHUB_EVENT_SOURCE"], "source": os.environ["GITHUB_EVENT_SOURCE"],
@ -152,7 +157,7 @@ class DecisionTask:
} }
task_id = taskcluster.slugId().decode("utf8") task_id = taskcluster.slugId().decode("utf8")
QUEUE.createTask(task_id, payload) self.queue_service.createTask(task_id, payload)
print("Scheduled %s: %s" % (task_name, task_id)) print("Scheduled %s: %s" % (task_name, task_id))
return task_id return task_id