diff --git a/.taskcluster.yml b/.taskcluster.yml index 511bb6ee02e..8fdae1a52e4 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,67 +1,100 @@ version: 1 policy: # https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1#pull-requests - pullRequests: collaborators + pullRequests: public + + # NOTE: when updating this consider whether the daily hook needs similar changes: +# https://tools.taskcluster.net/hooks/project-servo/daily tasks: - - $if: 'tasks_for == "github-push"' - then: - $if: >- - event.ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/master"] || - event.ref[:15] == "refs/heads/try-" + $let: + task_common: + provisionerId: aws-provisioner-v1 + created: {$fromNow: ''} + deadline: {$fromNow: '1 day'} + extra: + treeherder: + machine: {platform: Linux} + labels: [x64] + symbol: Decision + payload: + maxRunTime: {$eval: '20 * 60'} + # https://github.com/servo/taskcluster-bootstrap-docker-images#decision-task + image: "servobrowser/taskcluster-bootstrap:decision-task@sha256:28045b7ec0485ef363f8cb14f194008b47e9ede99f2ea40a1e945e921fce976e" + features: + taskclusterProxy: true + env: + GIT_URL: ${event.repository.clone_url} + TASK_FOR: ${tasks_for} + command: + - /bin/bash + - '--login' + - '-e' + - '-c' + - >- + git init repo && + cd repo && + git fetch --depth 1 "$GIT_URL" "$GIT_REF" && + git reset --hard "$GIT_SHA" && + python3 etc/taskcluster/decision_task.py + in: + - $if: "tasks_for == 'github-push'" then: + $let: + branch: + $if: "event.ref[:11] == 'refs/heads/'" + then: "${event.ref[11:]}" + in: + $if: "branch in ['auto', 'try', 'master'] || branch[:4] == 'try-'" + then: + $mergeDeep: + - {$eval: "task_common"} + - metadata: + name: "Servo: GitHub push decision task" + description: "" + owner: ${event.pusher.name}@users.noreply.github.com + source: ${event.compare} + workerType: servo-docker-worker + scopes: + - "assume:repo:github.com/servo/servo:branch:${branch}" + routes: + $let: + treeherder_repo: + $if: "branch[:4] == 'try-'" + then: "servo-${branch}" + else: "servo-try" + in: + - "tc-treeherder.v2._/${treeherder_repo}.${event.after}" + - "tc-treeherder-staging.v2._/${treeherder_repo}.${event.after}" + payload: + env: + GIT_REF: ${event.ref} + GIT_SHA: ${event.after} + TASK_OWNER: ${event.pusher.name}@users.noreply.github.com + TASK_SOURCE: ${event.compare} + - $if: >- + tasks_for == 'github-pull-request' && + event['action'] in ['opened', 'reopened', 'synchronize'] + then: + $mergeDeep: + - {$eval: "task_common"} + - metadata: + name: "Servo: GitHub PR decision task" + description: "" + owner: ${event.sender.login}@users.noreply.github.com + source: ${event.pull_request.url} + workerType: servo-docker-untrusted + scopes: + - "assume:repo:github.com/servo/servo:pull-request" + routes: + - "tc-treeherder.v2._/servo-prs.${event.pull_request.head.sha}" + - "tc-treeherder-staging.v2._/servo-prs.${event.pull_request.head.sha}" + payload: + env: + # We use the merge commit made by GitHub, not the PR’s branch + GIT_REF: refs/pull/${event.pull_request.number}/merge + # `event.pull_request.merge_commit_sha` is tempting but not what we want: + # https://github.com/servo/servo/pull/22597#issuecomment-451518810 + GIT_SHA: FETCH_HEAD + TASK_OWNER: ${event.sender.login}@users.noreply.github.com + TASK_SOURCE: ${event.pull_request.url} - # NOTE: when updating this consider whether the daily hook needs similar changes: - # https://tools.taskcluster.net/hooks/project-servo/daily - - taskGroupId: {$eval: as_slugid("decision_task")} - taskId: {$eval: as_slugid("decision_task")} - provisionerId: aws-provisioner-v1 - workerType: servo-docker-worker - created: {$fromNow: ''} - deadline: {$fromNow: '1 day'} - metadata: - name: "Servo: GitHub push decision task" - description: "" - owner: &task_owner ${event.pusher.name}@users.noreply.github.com - source: &task_source ${event.compare} - scopes: - - "assume:repo:github.com/servo/servo:branch:${event.ref[11:]}" - routes: - $let: - treeherder_repo: - $if: "event.ref[:15] != 'refs/heads/try-'" - then: "servo-${event.ref[11:]}" - else: "servo-try" - in: - - "tc-treeherder.v2._/${treeherder_repo}.${event.after}" - - "tc-treeherder-staging.v2._/${treeherder_repo}.${event.after}" - extra: - treeherder: - machine: {platform: Linux} - labels: [x64] - symbol: Decision - - payload: - maxRunTime: {$eval: '20 * 60'} - # https://github.com/servo/taskcluster-bootstrap-docker-images#decision-task - image: "servobrowser/taskcluster-bootstrap:decision-task@sha256:28045b7ec0485ef363f8cb14f194008b47e9ede99f2ea40a1e945e921fce976e" - features: - taskclusterProxy: true - env: - GIT_URL: ${event.repository.clone_url} - GIT_REF: ${event.ref} - GIT_SHA: ${event.after} - TASK_FOR: ${tasks_for} - TASK_OWNER: *task_owner - TASK_SOURCE: *task_source - command: - - /bin/bash - - '--login' - - '-e' - - '-c' - - >- - git init repo && - cd repo && - git fetch --depth 1 "$GIT_URL" "$GIT_REF" && - git reset --hard "$GIT_SHA" && - python3 etc/taskcluster/decision_task.py diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 94b1cfa9451..e9d568332ee 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -10,11 +10,11 @@ from decisionlib import CONFIG, SHARED def main(task_for): - assert CONFIG.git_ref.startswith("refs/heads/") - branch = CONFIG.git_ref[len("refs/heads/"):] - CONFIG.treeherder_repository_name = "servo-" + ( - branch if not branch.startswith("try-") else "try" - ) + if CONFIG.git_ref.startswith("refs/heads/"): + branch = CONFIG.git_ref[len("refs/heads/"):] + CONFIG.treeherder_repository_name = "servo-" + ( + branch if not branch.startswith("try-") else "try" + ) if task_for == "github-push": # FIXME https://github.com/servo/servo/issues/22325 implement these: @@ -72,6 +72,17 @@ def main(task_for): for function in by_branch_name.get(branch, []): function() + elif task_for == "github-pull-request": + CONFIG.treeherder_repository_name = "servo-prs" + CONFIG.index_read_only = True + + # We want the merge commit that GitHub creates for the PR. + # The event does contain a `pull_request.merge_commit_sha` key, but it is wrong: + # https://github.com/servo/servo/pull/22597#issuecomment-451518810 + CONFIG.git_sha_is_current_head() + + tidy_untrusted() + # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": daily_tasks_setup() @@ -120,6 +131,22 @@ windows_sparse_checkout = [ ] +def tidy_untrusted(): + return ( + decisionlib.DockerWorkerTask("Tidy") + .with_worker_type("servo-docker-untrusted") + .with_treeherder("Linux x64", "Tidy") + .with_max_run_time_minutes(60) + .with_dockerfile(dockerfile_path("build")) + .with_env(**build_env, **unix_build_env, **linux_build_env) + .with_repo() + .with_script(""" + ./mach test-tidy --no-progress --all + """) + .create() + ) + + def linux_tidy_unit_docs(): return ( linux_build_task("Tidy + dev build + unit tests + docs") @@ -452,7 +479,7 @@ def daily_tasks_setup(): "expires": SHARED.from_now_json(log_artifacts_expire_in), }) - # Unlike when reacting to a GitHub event, + # Unlike when reacting to a GitHub push event, # the commit hash is not known until we clone the repository. CONFIG.git_sha_is_current_head() @@ -511,7 +538,6 @@ def linux_build_task(name, *, build_env=build_env): .with_dockerfile(dockerfile_path("build")) .with_env(**build_env, **unix_build_env, **linux_build_env) .with_repo() - .with_index_and_artifacts_expire_in(build_artifacts_expire_in) ) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 7d9021c5db2..f23248a4b79 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -38,6 +38,7 @@ class Config: def __init__(self): self.task_name_template = "%s" self.index_prefix = "garbage.servo-decisionlib" + self.index_read_only = False self.scopes_for_all_subtasks = [] self.routes_for_all_subtasks = [] self.docker_image_buil_worker_type = None @@ -265,7 +266,8 @@ class Task: except taskcluster.TaskclusterRestFailure as e: if e.status_code != 404: # pragma: no cover raise - self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) + if not CONFIG.index_read_only: + self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) task_id = self.create() SHARED.found_or_created_indexed_tasks[index_path] = task_id diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py index bfc963ebaf8..559c837ef82 100755 --- a/etc/taskcluster/mock.py +++ b/etc/taskcluster/mock.py @@ -62,4 +62,7 @@ decision_task.main("github-push") print("\n# Daily:") decision_task.main("daily") +print("\n# PR:") +decision_task.main("github-pull-request") + print() diff --git a/etc/taskcluster/simulate_github_events.py b/etc/taskcluster/simulate_github_events.py new file mode 100755 index 00000000000..95faa5b34ae --- /dev/null +++ b/etc/taskcluster/simulate_github_events.py @@ -0,0 +1,72 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +''''set -e +cd "$(dirname $0)" +exec ../../python/_virtualenv/bin/python "$(basename $0)" +''' + +try: + import jsone +except ImportError: + import sys + sys.exit("pip install git+https://github.com/taskcluster/json-e") + +import yaml +import json + +template = yaml.load(open("../../.taskcluster.yml").read().decode("utf8")) +repo = dict( + repository=dict( + clone_url="https://github.com/servo/servo.git", + ), +) +contexts = [ + dict( + tasks_for="github-release", + event=repo, + ), + dict( + tasks_for="github-pull-request", + event=dict( + action="comment", + **repo + ), + ), + dict( + tasks_for="github-push", + event=dict( + ref="refs/heads/master", + compare="https://github.com/servo/servo/compare/1753cda...de09c8f", + after="de09c8fb6ef87dec5932d5fab4adcb421d291a54", + pusher=dict( + name="bors-servo", + ), + **repo + ), + ), + dict( + tasks_for="github-pull-request", + event=dict( + action="synchronize", + pull_request=dict( + number=22583, + url="https://github.com/servo/servo/pull/22583", + head=dict( + sha="51a422c9ef47420eb69c802643b7686bdb498652", + ), + merge_commit_sha="876fcf7a5fe971a9ac0a4ce117906c552c08c095", + ), + sender=dict( + login="jdm", + ), + **repo + ), + ), +] +for context in contexts: + print(context["tasks_for"]) + print(json.dumps(jsone.render(template, context), indent=2))