mirror of
https://github.com/servo/servo.git
synced 2025-06-06 16:45:39 +00:00
Remove more Taskcluster and Treeherder integration
Servo no longer uses Taskcluster and Treeherder, so this change removes script references to those services and support files.
This commit is contained in:
Martin Robinson
parent
d579bd91b8
commit
bc3abf9953
25 changed files with 11 additions and 2174 deletions
6
.github/workflows/mac.yml
vendored
6
.github/workflows/mac.yml
vendored
|
|
@ -63,8 +63,8 @@ jobs:
|
|||
- name: Bootstrap
|
||||
run: |
|
||||
python3 -m pip install --upgrade pip virtualenv
|
||||
brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile
|
||||
brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile-build
|
||||
brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile
|
||||
brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile-build
|
||||
rm -rf /usr/local/etc/openssl
|
||||
rm -rf /usr/local/etc/openssl@1.1
|
||||
brew install openssl@1.1 gnu-tar
|
||||
|
|
@ -133,7 +133,7 @@ jobs:
|
|||
# run: |
|
||||
# gtar -xzf target.tar.gz
|
||||
# python3 -m pip install --upgrade pip virtualenv
|
||||
# brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile
|
||||
# brew bundle install --verbose --no-upgrade --file=etc/homebrew/Brewfile
|
||||
# - name: Smoketest
|
||||
# run: python3 ./mach smoketest
|
||||
# - name: Run tests
|
||||
|
|
|
|||
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -75,4 +75,4 @@ support/hololens/.vs/
|
|||
layout_trace*
|
||||
|
||||
# Package managers
|
||||
etc/taskcluster/macos/Brewfile.lock.json
|
||||
etc/homebrew/Brewfile.lock.json
|
||||
|
|
|
|||
120
.taskcluster.yml
120
.taskcluster.yml
|
|
@ -1,120 +0,0 @@
|
|||
version: 1
|
||||
|
||||
# If and when switching to `reporting: checks-v1` here, also change the `statuses` route to `checks`
|
||||
# in `CONFIG.routes_for_all_subtasks` in `etc/taskcluster/decision_task.py`
|
||||
|
||||
policy:
|
||||
# https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1#pull-requests
|
||||
pullRequests: public
|
||||
|
||||
# NOTE: when updating this consider whether the daily hook needs similar changes:
|
||||
# https://tools.taskcluster.net/hooks/project-servo/daily
|
||||
tasks:
|
||||
$let:
|
||||
task_common:
|
||||
provisionerId:
|
||||
$if: "taskcluster_root_url == 'https://taskcluster.net'"
|
||||
then: aws-provisioner-v1
|
||||
else: proj-servo
|
||||
created: {$fromNow: ''}
|
||||
deadline: {$fromNow: '1 day'}
|
||||
priority: high
|
||||
extra:
|
||||
treeherder:
|
||||
machine: {platform: Linux}
|
||||
labels: [x64]
|
||||
symbol: Decision
|
||||
payload:
|
||||
maxRunTime: {$eval: '20 * 60'}
|
||||
# https://github.com/servo/taskcluster-bootstrap-docker-images#decision-task
|
||||
image: "servobrowser/taskcluster-bootstrap:decision-task@\
|
||||
sha256:7471a998e4462638c8d3e2cf0b4a99c9a5c8ca9f2ec0ae01cc069473b35cde10"
|
||||
features:
|
||||
taskclusterProxy: true
|
||||
artifacts:
|
||||
public/repo.bundle:
|
||||
type: file
|
||||
path: /repo.bundle
|
||||
expires: {$fromNow: '1 day'}
|
||||
env:
|
||||
GIT_URL: ${event.repository.clone_url}
|
||||
TASK_FOR: ${tasks_for}
|
||||
command:
|
||||
- /bin/bash
|
||||
- '--login'
|
||||
- '-e'
|
||||
- '-c'
|
||||
- >-
|
||||
git init repo &&
|
||||
cd repo &&
|
||||
git fetch --depth 1 "$GIT_URL" "$GIT_REF" &&
|
||||
git reset --hard "$GIT_SHA" &&
|
||||
python3 etc/taskcluster/decision_task.py
|
||||
in:
|
||||
- $if: "tasks_for == 'github-push'"
|
||||
then:
|
||||
$let:
|
||||
branch:
|
||||
$if: "event.ref[:11] == 'refs/heads/'"
|
||||
then: "${event.ref[11:]}"
|
||||
in:
|
||||
$if: "branch in ['auto', 'try', 'master'] || branch[:4] == 'try-'"
|
||||
then:
|
||||
$mergeDeep:
|
||||
- {$eval: "task_common"}
|
||||
- metadata:
|
||||
name: "Servo: GitHub push decision task"
|
||||
description: ""
|
||||
owner: ${event.pusher.name}@users.noreply.github.com
|
||||
source: ${event.compare}
|
||||
workerType:
|
||||
$if: "taskcluster_root_url == 'https://taskcluster.net'"
|
||||
then: servo-docker-worker
|
||||
else: docker
|
||||
scopes:
|
||||
- "assume:repo:github.com/servo/servo:branch:${branch}"
|
||||
routes:
|
||||
$let:
|
||||
treeherder_repo:
|
||||
$if: "branch[:4] == 'try-'"
|
||||
then: "servo-try"
|
||||
else: "servo-${branch}"
|
||||
in:
|
||||
- "tc-treeherder.v2._/${treeherder_repo}.${event.after}"
|
||||
- "tc-treeherder-staging.v2._/${treeherder_repo}.${event.after}"
|
||||
payload:
|
||||
env:
|
||||
GIT_REF: ${event.ref}
|
||||
GIT_SHA: ${event.after}
|
||||
TASK_OWNER: ${event.pusher.name}@users.noreply.github.com
|
||||
TASK_SOURCE: ${event.compare}
|
||||
- $if: >-
|
||||
tasks_for == 'github-pull-request' &&
|
||||
event['action'] in ['opened', 'reopened', 'synchronize']
|
||||
then:
|
||||
$mergeDeep:
|
||||
- {$eval: "task_common"}
|
||||
- metadata:
|
||||
name: "Servo: GitHub PR decision task"
|
||||
description: ""
|
||||
owner: ${event.sender.login}@users.noreply.github.com
|
||||
source: ${event.pull_request.url}
|
||||
workerType:
|
||||
$if: "taskcluster_root_url == 'https://taskcluster.net'"
|
||||
then: servo-docker-untrusted
|
||||
else: docker-untrusted
|
||||
scopes:
|
||||
- "assume:repo:github.com/servo/servo:pull-request"
|
||||
routes:
|
||||
- "tc-treeherder.v2._/servo-prs.${event.pull_request.head.sha}"
|
||||
- "tc-treeherder-staging.v2._/servo-prs.${event.pull_request.head.sha}"
|
||||
payload:
|
||||
env:
|
||||
# We use the merge commit made by GitHub, not the PR’s branch
|
||||
GIT_REF: refs/pull/${event.pull_request.number}/merge
|
||||
# `event.pull_request.merge_commit_sha` is tempting but not what we want:
|
||||
# https://github.com/servo/servo/pull/22597#issuecomment-451518810
|
||||
GIT_SHA: FETCH_HEAD
|
||||
TASK_OWNER: ${event.sender.login}@users.noreply.github.com
|
||||
TASK_SOURCE: ${event.pull_request.url}
|
||||
|
||||
|
|
@ -58,8 +58,8 @@ NOTE: run these steps after you've cloned the project locally.
|
|||
|
||||
``` sh
|
||||
cd servo
|
||||
brew bundle install --file=etc/taskcluster/macos/Brewfile
|
||||
brew bundle install --file=etc/taskcluster/macos/Brewfile-build
|
||||
brew bundle install --file=etc/homebrew/Brewfile
|
||||
brew bundle install --file=etc/homebrew/Brewfile-build
|
||||
pip install virtualenv
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -101,11 +101,6 @@ function unsafe_open_pull_request() {
|
|||
# is unnecessary.
|
||||
git checkout "${BRANCH_NAME}" || return 0
|
||||
|
||||
if [[ -z "${WPT_SYNC_TOKEN+set}" && "${TASKCLUSTER_PROXY_URL+set}" == "set" ]]; then
|
||||
SECRET_RESPONSE=$(curl ${TASKCLUSTER_PROXY_URL}/api/secrets/v1/secret/project/servo/wpt-sync)
|
||||
WPT_SYNC_TOKEN=`echo "${SECRET_RESPONSE}" | jq --raw-output '.secret.token'`
|
||||
fi
|
||||
|
||||
if [[ -z "${WPT_SYNC_TOKEN+set}" ]]; then
|
||||
echo "Github auth token missing from WPT_SYNC_TOKEN."
|
||||
return 1
|
||||
|
|
|
|||
|
|
@ -1,277 +0,0 @@
|
|||
# Testing Servo on Taskcluster
|
||||
|
||||
## In-tree and out-of-tree configuration
|
||||
|
||||
Where possible, we prefer keeping Taskcluster-related configuration and code in this directory,
|
||||
set up CI so that testing of a given git branch uses the version in that branch.
|
||||
That way, anyone can make changes (such installing a new system dependency
|
||||
[in a `Dockerfile`](#docker-images)) in the same PR that relies on those changes.
|
||||
|
||||
For some things however that is not practical,
|
||||
or some deployment step that mutates global state is required.
|
||||
That configuration is split between the [mozilla/community-tc-config] and
|
||||
[servo/taskcluster-config] repositories,
|
||||
managed by the Taskcluster team and the Servo team respectively.
|
||||
|
||||
[mozilla/community-tc-config]: https://github.com/mozilla/community-tc-config/blob/master/config/projects.yml
|
||||
[servo/taskcluster-config]: https://github.com/servo/taskcluster-config/tree/master/config
|
||||
|
||||
|
||||
## Homu
|
||||
|
||||
When a pull request is reviewed and the appropriate command is given,
|
||||
[Homu] creates a merge commit of `master` and the PR’s branch, and pushes it to the `auto` branch.
|
||||
One or more CI system (through their own means) get notified of this push by GitHub,
|
||||
start testing the merge commit, and use the [GitHub Status API] to report results.
|
||||
|
||||
Through a [Webhook], Homu gets notified of changes to these statues.
|
||||
If all of the required statuses are reported successful,
|
||||
Homu pushes its merge commit to the `master` branch
|
||||
and goes on to testing the next pull request in its queue.
|
||||
|
||||
[Homu]: https://github.com/servo/servo/wiki/Homu
|
||||
[GitHub Status API]: https://developer.github.com/v3/repos/statuses/
|
||||
[Webhook]: https://developer.github.com/webhooks/
|
||||
|
||||
|
||||
## Taskcluster − GitHub integration
|
||||
|
||||
Taskcluster is very flexible and not necessarily tied to GitHub,
|
||||
but it does have an optional [GitHub integration service] that you can enable
|
||||
on a repository [as a GitHub App].
|
||||
When enabled, this service gets notified for every push, pull request, or GitHub release.
|
||||
It then schedules some tasks based on reading [`.taskcluster.yml`] in the corresponding commit.
|
||||
|
||||
This file contains templates for creating one or more tasks,
|
||||
but the logic it can support is fairly limited.
|
||||
So a common pattern is to have it only run a single initial task called a *decision task*
|
||||
that can have complex logic based on code and data in the repository
|
||||
to build an arbitrary [task graph].
|
||||
|
||||
[GitHub integration service]: https://community-tc.services.mozilla.com/docs/manual/using/github
|
||||
[as a GitHub App]: https://github.com/apps/community-tc-integration/
|
||||
[`.taskcluster.yml`]: https://community-tc.services.mozilla.com/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1
|
||||
[task graph]: https://community-tc.services.mozilla.com/docs/manual/using/task-graph
|
||||
|
||||
|
||||
## Servo’s decision task
|
||||
|
||||
This repository’s [`.taskcluster.yml`][tc.yml] schedules a single task
|
||||
that runs the Python 3 script [`etc/taskcluster/decision_task.py`](decision_task.py).
|
||||
It is called a *decision task* as it is responsible for deciding what other tasks to schedule.
|
||||
|
||||
The Docker image that runs the decision task
|
||||
is hosted on Docker Hub at [`servobrowser/taskcluster-bootstrap`][hub].
|
||||
It is built by [Docker Hub automated builds] based on a `Dockerfile`
|
||||
in the [`taskcluster-bootstrap-docker-images`] GitHub repository.
|
||||
Hopefully, this image does not need to be modified often
|
||||
as it only needs to clone the repository and run Python.
|
||||
|
||||
[tc.yml]: ../../../.taskcluster.yml
|
||||
[hub]: https://hub.docker.com/r/servobrowser/taskcluster-bootstrap/
|
||||
[Docker Hub automated builds]: https://docs.docker.com/docker-hub/builds/
|
||||
[`taskcluster-bootstrap-docker-images`]: https://github.com/servo/taskcluster-bootstrap-docker-images/
|
||||
|
||||
|
||||
## Docker images
|
||||
|
||||
[Similar to Firefox][firefox], Servo’s decision task supports running other tasks
|
||||
in Docker images built on-demand, based on `Dockerfile`s in the main repository.
|
||||
Modifying a `Dockerfile` and relying on those new changes
|
||||
can be done in the same pull request or commit.
|
||||
|
||||
To avoid rebuilding images on every pull request,
|
||||
they are cached based on a hash of the source `Dockerfile`.
|
||||
For now, to support this hashing, we make `Dockerfile`s be self-contained (with one exception).
|
||||
Images are built without a [context],
|
||||
so instructions like [`COPY`] cannot be used because there is nothing to copy from.
|
||||
The exception is that the decision task adds support for a non-standard include directive:
|
||||
when a `Dockerfile` first line is `% include` followed by a filename,
|
||||
that line is replaced with the content of that file.
|
||||
|
||||
For example,
|
||||
[`etc/taskcluster/docker/build.dockerfile`](docker/build.dockerfile) starts like so:
|
||||
|
||||
```Dockerfile
|
||||
% include base.dockerfile
|
||||
|
||||
RUN \
|
||||
apt-get install -qy --no-install-recommends \
|
||||
# […]
|
||||
```
|
||||
|
||||
[firefox]: https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/docker-images.html
|
||||
[context]: https://docs.docker.com/engine/reference/commandline/build/#extended-description
|
||||
[`COPY`]: https://docs.docker.com/engine/reference/builder/#copy
|
||||
|
||||
|
||||
## Build artifacts
|
||||
|
||||
[web-platform-tests] (WPT) is large enough that running all of a it takes a long time.
|
||||
So it supports *chunking*,
|
||||
such as multiple chunks of the test suite can be run in parallel on different machines.
|
||||
As of this writing,
|
||||
Servo’s current Buildbot setup for this has each machine start by compiling its own copy of Servo.
|
||||
On Taskcluster with a decision task,
|
||||
we can have a single build task save its resulting binary executable as an [artifact],
|
||||
together with multiple testing tasks that each depend on the build task
|
||||
(wait until it successfully finishes before they can start)
|
||||
and start by downloading the artifact that was saved earlier.
|
||||
|
||||
The logic for all this is in [`decision_task.py`](decision_task.py)
|
||||
and can be modified in any pull request.
|
||||
|
||||
[web-platform-tests]: https://github.com/web-platform-tests/wpt
|
||||
[artifact]: https://community-tc.services.mozilla.com/docs/manual/using/artifacts
|
||||
|
||||
|
||||
## Log artifacts
|
||||
|
||||
Taskcluster automatically save the `stdio` output of a task as an artifact,
|
||||
and has special support for showing and streaming that output while the task is still running.
|
||||
|
||||
Servo’s decision task additionally looks for `*.log` arguments to its tasks’s commands,
|
||||
assumes they instruct a program to create a log file with that name,
|
||||
and saves those log files as individual artifacts.
|
||||
|
||||
For example, WPT tasks have a `filtered-wpt-errorsummary.log` artifact
|
||||
that is typically the most relevant output when such a task fails.
|
||||
|
||||
|
||||
## Scopes and roles
|
||||
|
||||
[Scopes] are what Taskcluster calls permissions.
|
||||
They control access to everything.
|
||||
|
||||
Anyone logged in in the [web UI] has (access to) a set of scopes,
|
||||
which is visible on the [credentials] page
|
||||
(reachable from clicking on one’s own name on the top-right of any page).
|
||||
|
||||
A running task has a set of scopes allowing it access to various functionality and APIs.
|
||||
It can grant those scopes (and at most only those) to sub-tasks that it schedules
|
||||
(if it has the scope allowing it to schedule new tasks in the first place).
|
||||
|
||||
[Roles] represent each a set of scopes.
|
||||
They can be granted to… things,
|
||||
and then configured separately to modify what scopes they [expand] to.
|
||||
|
||||
For example, when Taskcluster-GitHub schedules tasks based on the `.taskcluster.yml` file
|
||||
in a push to the `auto` branch of this repository,
|
||||
those tasks are granted the scope `assume:repo:github.com/servo/servo:branch:auto`.
|
||||
Scopes that start with `assume:` are special,
|
||||
they expand to the scopes defined in the matching roles.
|
||||
In this case, the [`repo:github.com/servo/servo:branch:*`][branches] role matches.
|
||||
|
||||
The [`project:servo:decision-task/base`][base]
|
||||
and [`project:servo:decision-task/trusted`][trusted] roles
|
||||
centralize the set of scopes granted to the decision task.
|
||||
This avoids maintaining them seprately in the `repo:…` roles,
|
||||
in the `hook-id:…` role,
|
||||
and in the `taskcluster.yml` file.
|
||||
Only the `base` role is granted to tasks executed when a pull request is opened.
|
||||
These tasks are less trusted because they run before the code has been reviewed,
|
||||
and anyone can open a PR.
|
||||
|
||||
Members of the [@servo/taskcluster-admins] GitHub team are granted
|
||||
the scope `assume:project-admin:servo`, which is necessary to deploy changes
|
||||
to those roles from the [servo/taskcluster-config] repository.
|
||||
|
||||
[Scopes]: https://community-tc.services.mozilla.com/docs/manual/design/apis/hawk/scopes
|
||||
[web UI]: https://community-tc.services.mozilla.com/
|
||||
[credentials]: https://community-tc.services.mozilla.com/profile
|
||||
[Roles]: https://community-tc.services.mozilla.com/docs/manual/design/apis/hawk/roles
|
||||
[expand]: https://community-tc.services.mozilla.com/docs/reference/platform/taskcluster-auth/docs/roles
|
||||
[branches]: https://community-tc.services.mozilla.com/auth/roles/repo%3Agithub.com%2Fservo%2Fservo%3Abranch%3A*
|
||||
[base]: https://community-tc.services.mozilla.com/auth/roles/project%3Aservo%3Adecision-task%2Fbase
|
||||
[trusted]: https://community-tc.services.mozilla.com/auth/roles/project%3Aservo%3Adecision-task%2Ftrusted
|
||||
[@servo/taskcluster-admins]: https://github.com/orgs/servo/teams/taskcluster-admins
|
||||
|
||||
|
||||
## Daily tasks
|
||||
|
||||
The [`project-servo/daily`] hook in Taskcluster’s [Hooks service]
|
||||
is used to run some tasks automatically ever 24 hours.
|
||||
In this case as well we use a decision task.
|
||||
The `decision_task.py` script can differentiate this from a GitHub push
|
||||
based on the `$TASK_FOR` environment variable.
|
||||
Daily tasks can also be triggered manually.
|
||||
|
||||
Scopes available to the daily decision task need to be both requested in the hook definition
|
||||
and granted through the [`hook-id:project-servo/daily`] role.
|
||||
|
||||
Because they do not have something similar to GitHub statuses that link to them,
|
||||
daily tasks are indexed under the [`project.servo.daily`] namespace.
|
||||
|
||||
[`project.servo.daily`]: https://tools.taskcluster.net/index/project.servo.daily
|
||||
[`project-servo/daily`]: https://github.com/servo/taskcluster-config/blob/master/config/hooks.yml
|
||||
[Hooks service]: https://community-tc.services.mozilla.com/docs/manual/using/scheduled-tasks
|
||||
[`hook-id:project-servo/daily`]: https://github.com/servo/taskcluster-config/blob/master/config/roles.yml
|
||||
|
||||
|
||||
## Servo’s worker pools
|
||||
|
||||
Each task is assigned to a “worker pool”.
|
||||
Servo has several, for the different environments a task can run in:
|
||||
|
||||
* `docker` and `docker-untrusted` provide a Linux environment with full `root` privileges,
|
||||
in a Docker container running a [Docker image](#docker-images) of the task’s choice,
|
||||
in a short-lived virtual machine,
|
||||
on Google Cloud Platform.
|
||||
|
||||
Instances are started automatically as needed
|
||||
when the existing capacity is insufficient to execute queued tasks.
|
||||
They terminate themselves after being idle without work for a while,
|
||||
or unconditionally after a few days.
|
||||
Because these workers are short-lived,
|
||||
we don’t need to worry about evicting old entries from Cargo’s or rustup’s download cache,
|
||||
for example.
|
||||
|
||||
[The Taskcluster team manages][mozilla/community-tc-config]
|
||||
the configuration and VM image for these two pools.
|
||||
The latter has fewer scopes. It runs automated testing of pull requests
|
||||
as soon as they’re opened or updated, before any review.
|
||||
|
||||
* `win2016` runs Windows Server 2016 on AWS EC2.
|
||||
Like with Docker tasks, workers are short-lived and started automatically.
|
||||
The configuration and VM image for this pool
|
||||
is [managed by the Servo team][servo/taskcluster-config].
|
||||
|
||||
Tasks run as an unprivileged user.
|
||||
Because creating an new the VM image is slow and deploying it mutates global state,
|
||||
when a tool does not require system-wide installation
|
||||
we prefer having each task obtain it as needed by extracting an archive in a directory.
|
||||
See calls of `with_directory_mount` and `with_repacked_msi` in
|
||||
[`decision_task.py`](decision_task.py) and [`decisionlib.py`](decisionlib.py).
|
||||
|
||||
* `macos` runs, you guessed it, macOS.
|
||||
Tasks run on dedicated hardware provided long-term by Macstadium.
|
||||
The system-wide configuration of those machines
|
||||
is [managed by the Servo team][servo/taskcluster-config] through SaltStack.
|
||||
There is a task-owned (but preserved across tasks) install of Homebrew,
|
||||
with `Brewfile`s [in this repository](macos/).
|
||||
|
||||
This [Workers] page lists the current state of each macOS worker.
|
||||
(A similar page exists for other each worker pools, but as of this writing it has
|
||||
[usability issues](https://github.com/taskcluster/taskcluster/issues/1972)
|
||||
with short-lived workers.)
|
||||
|
||||
[Workers]: https://community-tc.services.mozilla.com/provisioners/proj-servo/worker-types/macos
|
||||
|
||||
|
||||
## Taskcluster − Treeherder integration
|
||||
|
||||
See [`treeherder.md`](treeherder.md).
|
||||
|
||||
|
||||
## Self-service, IRC, and Bugzilla
|
||||
|
||||
Taskcluster is designed to be “self-service” as much as possible.
|
||||
Between this repository [servo/taskcluster-config] and [mozilla/community-tc-config],
|
||||
anyone should be able to submit PRs for any part of the configuration.
|
||||
|
||||
Feel free to ask for help on the `#servo` or `#taskcluster` channels on Mozilla IRC.
|
||||
|
||||
For issue reports or feature requests on various bits of Taskcluster *software*,
|
||||
file bugs [in Mozilla’s Bugzilla, under `Taskcluster`][bug].
|
||||
|
||||
[bug]: https://bugzilla.mozilla.org/enter_bug.cgi?product=Taskcluster
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
# coding: utf8
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os.path
|
||||
import decisionlib
|
||||
import functools
|
||||
from decisionlib import CONFIG, SHARED
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
def main(task_for):
|
||||
with decisionlib.make_repo_bundle():
|
||||
tasks(task_for)
|
||||
|
||||
|
||||
def tasks(task_for):
|
||||
if CONFIG.git_ref.startswith("refs/heads/"):
|
||||
branch = CONFIG.git_ref[len("refs/heads/"):]
|
||||
CONFIG.treeherder_repository_name = "servo-" + (
|
||||
branch if not branch.startswith("try-") else "try"
|
||||
)
|
||||
|
||||
# Work around a tc-github bug/limitation:
|
||||
# https://bugzilla.mozilla.org/show_bug.cgi?id=1548781#c4
|
||||
if task_for.startswith("github"):
|
||||
# https://github.com/taskcluster/taskcluster/blob/21f257dc8/services/github/config.yml#L14
|
||||
CONFIG.routes_for_all_subtasks.append("statuses")
|
||||
|
||||
if task_for == "github-push":
|
||||
all_tests = []
|
||||
by_branch_name = {
|
||||
"auto": all_tests,
|
||||
"try": all_tests,
|
||||
"try-taskcluster": [
|
||||
# Add functions here as needed, in your push to that branch
|
||||
],
|
||||
"master": [],
|
||||
|
||||
# The "try-*" keys match those in `servo_try_choosers` in Homu’s config:
|
||||
# https://github.com/servo/saltfs/blob/master/homu/map.jinja
|
||||
|
||||
"try-mac": [],
|
||||
"try-linux": [],
|
||||
"try-windows": [],
|
||||
"try-arm": [],
|
||||
"try-wpt": [],
|
||||
"try-wpt-2020": [],
|
||||
"try-wpt-mac": [],
|
||||
"test-wpt": [],
|
||||
}
|
||||
|
||||
elif task_for == "github-pull-request":
|
||||
CONFIG.treeherder_repository_name = "servo-prs"
|
||||
CONFIG.index_read_only = True
|
||||
CONFIG.docker_image_build_worker_type = None
|
||||
|
||||
# We want the merge commit that GitHub creates for the PR.
|
||||
# The event does contain a `pull_request.merge_commit_sha` key, but it is wrong:
|
||||
# https://github.com/servo/servo/pull/22597#issuecomment-451518810
|
||||
CONFIG.git_sha_is_current_head()
|
||||
|
||||
elif task_for == "try-windows-ami":
|
||||
CONFIG.git_sha_is_current_head()
|
||||
CONFIG.windows_worker_type = os.environ["NEW_AMI_WORKER_TYPE"]
|
||||
|
||||
# https://tools.taskcluster.net/hooks/project-servo/daily
|
||||
elif task_for == "daily":
|
||||
daily_tasks_setup()
|
||||
|
||||
|
||||
ping_on_daily_task_failure = "SimonSapin, nox, emilio"
|
||||
build_artifacts_expire_in = "1 week"
|
||||
build_dependencies_artifacts_expire_in = "1 month"
|
||||
log_artifacts_expire_in = "1 year"
|
||||
|
||||
|
||||
def daily_tasks_setup():
|
||||
# Unlike when reacting to a GitHub push event,
|
||||
# the commit hash is not known until we clone the repository.
|
||||
CONFIG.git_sha_is_current_head()
|
||||
|
||||
# On failure, notify a few people on IRC
|
||||
# https://docs.taskcluster.net/docs/reference/core/taskcluster-notify/docs/usage
|
||||
notify_route = "notify.irc-channel.#servo.on-failed"
|
||||
CONFIG.routes_for_all_subtasks.append(notify_route)
|
||||
CONFIG.scopes_for_all_subtasks.append("queue:route:" + notify_route)
|
||||
CONFIG.task_name_template = "Servo daily: %s. On failure, ping: " + ping_on_daily_task_failure
|
||||
|
||||
|
||||
CONFIG.task_name_template = "Servo: %s"
|
||||
CONFIG.docker_images_expire_in = build_dependencies_artifacts_expire_in
|
||||
CONFIG.repacked_msi_files_expire_in = build_dependencies_artifacts_expire_in
|
||||
CONFIG.index_prefix = "project.servo"
|
||||
CONFIG.default_provisioner_id = "proj-servo"
|
||||
CONFIG.docker_image_build_worker_type = "docker"
|
||||
|
||||
CONFIG.windows_worker_type = "win2016"
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
main(task_for=os.environ["TASK_FOR"])
|
||||
|
|
@ -1,851 +0,0 @@
|
|||
# coding: utf8
|
||||
|
||||
# Copyright 2018 The Servo Project Developers. See the COPYRIGHT
|
||||
# file at the top-level directory of this distribution.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
# option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
"""
|
||||
Project-independent library for Taskcluster decision tasks
|
||||
"""
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import taskcluster
|
||||
|
||||
|
||||
# Public API
|
||||
__all__ = [
|
||||
"CONFIG", "SHARED", "Task", "DockerWorkerTask",
|
||||
"GenericWorkerTask", "WindowsGenericWorkerTask",
|
||||
"make_repo_bundle",
|
||||
]
|
||||
|
||||
|
||||
class Config:
|
||||
"""
|
||||
Global configuration, for users of the library to modify.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.task_name_template = "%s"
|
||||
self.index_prefix = "garbage.servo-decisionlib"
|
||||
self.index_read_only = False
|
||||
self.scopes_for_all_subtasks = []
|
||||
self.routes_for_all_subtasks = []
|
||||
self.docker_image_build_worker_type = None
|
||||
self.docker_images_expire_in = "1 month"
|
||||
self.repacked_msi_files_expire_in = "1 month"
|
||||
self.treeherder_repository_name = None
|
||||
|
||||
# Set by docker-worker:
|
||||
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
|
||||
self.decision_task_id = os.environ.get("TASK_ID")
|
||||
|
||||
# Set in the decision task’s payload, such as defined in .taskcluster.yml
|
||||
self.task_owner = os.environ.get("TASK_OWNER")
|
||||
self.task_source = os.environ.get("TASK_SOURCE")
|
||||
self.git_url = os.environ.get("GIT_URL")
|
||||
self.git_ref = os.environ.get("GIT_REF")
|
||||
self.git_sha = os.environ.get("GIT_SHA")
|
||||
self.git_bundle_shallow_ref = "refs/heads/shallow"
|
||||
|
||||
self.tc_root_url = os.environ.get("TASKCLUSTER_ROOT_URL")
|
||||
self.default_provisioner_id = "proj-example"
|
||||
|
||||
|
||||
def tree_hash(self):
|
||||
if not hasattr(self, "_tree_hash"):
|
||||
# Use the SHA-1 hash of the git "tree" object rather than the commit.
|
||||
# A `@bors-servo retry` command creates a new merge commit with a different commit hash
|
||||
# but with the same tree hash.
|
||||
output = subprocess.check_output(["git", "show", "-s", "--format=%T", "HEAD"])
|
||||
self._tree_hash = output.decode("utf-8").strip()
|
||||
return self._tree_hash
|
||||
|
||||
def git_sha_is_current_head(self):
|
||||
output = subprocess.check_output(["git", "rev-parse", "HEAD"])
|
||||
self.git_sha = output.decode("utf8").strip()
|
||||
|
||||
|
||||
class Shared:
|
||||
"""
|
||||
Global shared state.
|
||||
"""
|
||||
def __init__(self):
|
||||
self.now = datetime.datetime.utcnow()
|
||||
self.found_or_created_indexed_tasks = {}
|
||||
|
||||
options = {"rootUrl": os.environ["TASKCLUSTER_PROXY_URL"]}
|
||||
self.queue_service = taskcluster.Queue(options)
|
||||
self.index_service = taskcluster.Index(options)
|
||||
|
||||
def from_now_json(self, offset):
|
||||
"""
|
||||
Same as `taskcluster.fromNowJSON`, but uses the creation time of `self` for “now”.
|
||||
"""
|
||||
return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))
|
||||
|
||||
|
||||
CONFIG = Config()
|
||||
SHARED = Shared()
|
||||
|
||||
|
||||
def chaining(op, attr):
|
||||
def method(self, *args, **kwargs):
|
||||
op(self, attr, *args, **kwargs)
|
||||
return self
|
||||
return method
|
||||
|
||||
|
||||
def append_to_attr(self, attr, *args): getattr(self, attr).extend(args)
|
||||
def prepend_to_attr(self, attr, *args): getattr(self, attr)[0:0] = list(args)
|
||||
def update_attr(self, attr, **kwargs): getattr(self, attr).update(kwargs)
|
||||
|
||||
|
||||
class Task:
|
||||
"""
|
||||
A task definition, waiting to be created.
|
||||
|
||||
Typical is to use chain the `with_*` methods to set or extend this object’s attributes,
|
||||
then call the `crate` or `find_or_create` method to schedule a task.
|
||||
|
||||
This is an abstract class that needs to be specialized for different worker implementations.
|
||||
"""
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.description = ""
|
||||
self.scheduler_id = "taskcluster-github"
|
||||
self.provisioner_id = CONFIG.default_provisioner_id
|
||||
self.worker_type = "github-worker"
|
||||
self.deadline_in = "1 day"
|
||||
self.expires_in = "1 year"
|
||||
self.index_and_artifacts_expire_in = self.expires_in
|
||||
self.dependencies = []
|
||||
self.scopes = []
|
||||
self.routes = []
|
||||
self.extra = {}
|
||||
self.treeherder_required = False
|
||||
self.priority = None # Defaults to 'lowest'
|
||||
self.git_fetch_url = CONFIG.git_url
|
||||
self.git_fetch_ref = CONFIG.git_ref
|
||||
self.git_checkout_sha = CONFIG.git_sha
|
||||
|
||||
# All `with_*` methods return `self`, so multiple method calls can be chained.
|
||||
with_description = chaining(setattr, "description")
|
||||
with_scheduler_id = chaining(setattr, "scheduler_id")
|
||||
with_provisioner_id = chaining(setattr, "provisioner_id")
|
||||
with_worker_type = chaining(setattr, "worker_type")
|
||||
with_deadline_in = chaining(setattr, "deadline_in")
|
||||
with_expires_in = chaining(setattr, "expires_in")
|
||||
with_index_and_artifacts_expire_in = chaining(setattr, "index_and_artifacts_expire_in")
|
||||
with_priority = chaining(setattr, "priority")
|
||||
|
||||
with_dependencies = chaining(append_to_attr, "dependencies")
|
||||
with_scopes = chaining(append_to_attr, "scopes")
|
||||
with_routes = chaining(append_to_attr, "routes")
|
||||
|
||||
with_extra = chaining(update_attr, "extra")
|
||||
|
||||
def with_index_at(self, index_path):
|
||||
self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path))
|
||||
return self
|
||||
|
||||
def with_treeherder_required(self):
|
||||
self.treeherder_required = True
|
||||
return self
|
||||
|
||||
def with_treeherder(self, category, symbol, group_name=None, group_symbol=None):
|
||||
assert len(symbol) <= 25, symbol
|
||||
self.name = "%s: %s" % (category, self.name)
|
||||
|
||||
# The message schema does not allow spaces in the platfrom or in labels,
|
||||
# but the UI shows them in that order separated by spaces.
|
||||
# So massage the metadata to get the UI to show the string we want.
|
||||
# `labels` defaults to ["opt"] if not provided or empty,
|
||||
# so use a more neutral underscore instead.
|
||||
parts = category.split(" ")
|
||||
platform = parts[0]
|
||||
labels = parts[1:] or ["_"]
|
||||
|
||||
# https://github.com/mozilla/treeherder/blob/master/schemas/task-treeherder-config.yml
|
||||
self.with_extra(treeherder=dict_update_if_truthy(
|
||||
{
|
||||
"machine": {"platform": platform},
|
||||
"labels": labels,
|
||||
"symbol": symbol,
|
||||
},
|
||||
groupName=group_name,
|
||||
groupSymbol=group_symbol,
|
||||
))
|
||||
|
||||
if CONFIG.treeherder_repository_name:
|
||||
assert CONFIG.git_sha
|
||||
suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repository_name, CONFIG.git_sha)
|
||||
self.with_routes(
|
||||
"tc-treeherder" + suffix,
|
||||
"tc-treeherder-staging" + suffix,
|
||||
)
|
||||
|
||||
self.treeherder_required = False # Taken care of
|
||||
return self
|
||||
|
||||
def build_worker_payload(self): # pragma: no cover
|
||||
"""
|
||||
Overridden by sub-classes to return a dictionary in a worker-specific format,
|
||||
which is used as the `payload` property in a task definition request
|
||||
passed to the Queue’s `createTask` API.
|
||||
|
||||
<https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask>
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def create(self):
|
||||
"""
|
||||
Call the Queue’s `createTask` API to schedule a new task, and return its ID.
|
||||
|
||||
<https://docs.taskcluster.net/docs/reference/platform/taskcluster-queue/references/api#createTask>
|
||||
"""
|
||||
worker_payload = self.build_worker_payload()
|
||||
assert not self.treeherder_required, \
|
||||
"make sure to call with_treeherder() for this task: %s" % self.name
|
||||
|
||||
assert CONFIG.decision_task_id
|
||||
assert CONFIG.task_owner
|
||||
assert CONFIG.task_source
|
||||
|
||||
def dedup(xs):
|
||||
seen = set()
|
||||
return [x for x in xs if not (x in seen or seen.add(x))]
|
||||
|
||||
queue_payload = {
|
||||
"taskGroupId": CONFIG.decision_task_id,
|
||||
"dependencies": dedup([CONFIG.decision_task_id] + self.dependencies),
|
||||
"schedulerId": self.scheduler_id,
|
||||
"provisionerId": self.provisioner_id,
|
||||
"workerType": self.worker_type,
|
||||
|
||||
"created": SHARED.from_now_json(""),
|
||||
"deadline": SHARED.from_now_json(self.deadline_in),
|
||||
"expires": SHARED.from_now_json(self.expires_in),
|
||||
"metadata": {
|
||||
"name": CONFIG.task_name_template % self.name,
|
||||
"description": self.description,
|
||||
"owner": CONFIG.task_owner,
|
||||
"source": CONFIG.task_source,
|
||||
},
|
||||
|
||||
"payload": worker_payload,
|
||||
}
|
||||
scopes = self.scopes + CONFIG.scopes_for_all_subtasks
|
||||
routes = self.routes + CONFIG.routes_for_all_subtasks
|
||||
if any(r.startswith("index.") for r in routes):
|
||||
self.extra.setdefault("index", {})["expires"] = \
|
||||
SHARED.from_now_json(self.index_and_artifacts_expire_in)
|
||||
dict_update_if_truthy(
|
||||
queue_payload,
|
||||
scopes=scopes,
|
||||
routes=routes,
|
||||
extra=self.extra,
|
||||
priority=self.priority,
|
||||
)
|
||||
|
||||
task_id = taskcluster.slugId()
|
||||
SHARED.queue_service.createTask(task_id, queue_payload)
|
||||
print("Scheduled %s: %s" % (task_id, self.name))
|
||||
return task_id
|
||||
|
||||
@staticmethod
|
||||
def find(index_path):
|
||||
full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path)
|
||||
task_id = SHARED.index_service.findTask(full_index_path)["taskId"]
|
||||
print("Found task %s indexed at %s" % (task_id, full_index_path))
|
||||
return task_id
|
||||
|
||||
def find_or_create(self, index_path):
|
||||
"""
|
||||
Try to find a task in the Index and return its ID.
|
||||
|
||||
The index path used is `{CONFIG.index_prefix}.{index_path}`.
|
||||
`index_path` defaults to `by-task-definition.{sha256}`
|
||||
with a hash of the worker payload and worker type.
|
||||
|
||||
If no task is found in the index,
|
||||
it is created with a route to add it to the index at that same path if it succeeds.
|
||||
|
||||
<https://docs.taskcluster.net/docs/reference/core/taskcluster-index/references/api#findTask>
|
||||
"""
|
||||
task_id = SHARED.found_or_created_indexed_tasks.get(index_path)
|
||||
if task_id is not None:
|
||||
return task_id
|
||||
|
||||
try:
|
||||
task_id = Task.find(index_path)
|
||||
except taskcluster.TaskclusterRestFailure as e:
|
||||
if e.status_code != 404: # pragma: no cover
|
||||
raise
|
||||
if not CONFIG.index_read_only:
|
||||
self.with_index_at(index_path)
|
||||
task_id = self.create()
|
||||
|
||||
SHARED.found_or_created_indexed_tasks[index_path] = task_id
|
||||
return task_id
|
||||
|
||||
def with_curl_script(self, url, file_path):
|
||||
return self \
|
||||
.with_script("""
|
||||
curl --compressed --retry 5 --connect-timeout 10 -Lf "%s" -o "%s"
|
||||
""" % (url, file_path))
|
||||
|
||||
def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
|
||||
queue_service = CONFIG.tc_root_url + "/api/queue"
|
||||
return self \
|
||||
.with_dependencies(task_id) \
|
||||
.with_curl_script(
|
||||
queue_service + "/v1/task/%s/artifacts/public/%s" % (task_id, artifact_name),
|
||||
os.path.join(out_directory, url_basename(artifact_name)),
|
||||
)
|
||||
|
||||
def with_repo_bundle(self, **kwargs):
|
||||
self.git_fetch_url = "../repo.bundle"
|
||||
self.git_fetch_ref = CONFIG.git_bundle_shallow_ref
|
||||
self.git_checkout_sha = "FETCH_HEAD"
|
||||
return self \
|
||||
.with_curl_artifact_script(CONFIG.decision_task_id, "repo.bundle") \
|
||||
.with_repo(**kwargs)
|
||||
|
||||
|
||||
class GenericWorkerTask(Task):
|
||||
"""
|
||||
Task definition for a worker type that runs the `generic-worker` implementation.
|
||||
|
||||
This is an abstract class that needs to be specialized for different operating systems.
|
||||
|
||||
<https://github.com/taskcluster/generic-worker>
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.max_run_time_minutes = 30
|
||||
self.env = {}
|
||||
self.features = {}
|
||||
self.mounts = []
|
||||
self.artifacts = []
|
||||
|
||||
with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes")
|
||||
with_mounts = chaining(append_to_attr, "mounts")
|
||||
with_env = chaining(update_attr, "env")
|
||||
|
||||
def build_command(self): # pragma: no cover
|
||||
"""
|
||||
Overridden by sub-classes to return the `command` property of the worker payload,
|
||||
in the format appropriate for the operating system.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def build_worker_payload(self):
|
||||
"""
|
||||
Return a `generic-worker` worker payload.
|
||||
|
||||
<https://docs.taskcluster.net/docs/reference/workers/generic-worker/docs/payload>
|
||||
"""
|
||||
worker_payload = {
|
||||
"command": self.build_command(),
|
||||
"maxRunTime": self.max_run_time_minutes * 60
|
||||
}
|
||||
return dict_update_if_truthy(
|
||||
worker_payload,
|
||||
env=self.env,
|
||||
mounts=self.mounts,
|
||||
features=self.features,
|
||||
artifacts=[
|
||||
{
|
||||
"type": type_,
|
||||
"path": path,
|
||||
"name": "public/" + url_basename(path),
|
||||
"expires": SHARED.from_now_json(self.index_and_artifacts_expire_in),
|
||||
}
|
||||
for type_, path in self.artifacts
|
||||
],
|
||||
)
|
||||
|
||||
def with_artifacts(self, *paths, type="file"):
|
||||
"""
|
||||
Add each path in `paths` as a task artifact
|
||||
that expires in `self.index_and_artifacts_expire_in`.
|
||||
|
||||
`type` can be `"file"` or `"directory"`.
|
||||
|
||||
Paths are relative to the task’s home directory.
|
||||
"""
|
||||
for path in paths:
|
||||
if (type, path) in self.artifacts:
|
||||
raise ValueError("Duplicate artifact: " + path) # pragma: no cover
|
||||
self.artifacts.append(tuple((type, path)))
|
||||
return self
|
||||
|
||||
def with_features(self, *names):
|
||||
"""
|
||||
Enable the given `generic-worker` features.
|
||||
|
||||
<https://github.com/taskcluster/generic-worker/blob/master/native_windows.yml>
|
||||
"""
|
||||
self.features.update({name: True for name in names})
|
||||
return self
|
||||
|
||||
def _mount_content(self, url_or_artifact_name, task_id, sha256):
|
||||
if task_id:
|
||||
content = {"taskId": task_id, "artifact": url_or_artifact_name}
|
||||
else:
|
||||
content = {"url": url_or_artifact_name}
|
||||
if sha256:
|
||||
content["sha256"] = sha256
|
||||
return content
|
||||
|
||||
def with_file_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None):
|
||||
"""
|
||||
Make `generic-worker` download a file before the task starts
|
||||
and make it available at `path` (which is relative to the task’s home directory).
|
||||
|
||||
If `sha256` is provided, `generic-worker` will hash the downloaded file
|
||||
and check it against the provided signature.
|
||||
|
||||
If `task_id` is provided, this task will depend on that task
|
||||
and `url_or_artifact_name` is the name of an artifact of that task.
|
||||
"""
|
||||
return self.with_mounts({
|
||||
"file": path or url_basename(url_or_artifact_name),
|
||||
"content": self._mount_content(url_or_artifact_name, task_id, sha256),
|
||||
})
|
||||
|
||||
def with_directory_mount(self, url_or_artifact_name, task_id=None, sha256=None, path=None):
|
||||
"""
|
||||
Make `generic-worker` download an archive before the task starts,
|
||||
and uncompress it at `path` (which is relative to the task’s home directory).
|
||||
|
||||
`url_or_artifact_name` must end in one of `.rar`, `.tar.bz2`, `.tar.gz`, or `.zip`.
|
||||
The archive must be in the corresponding format.
|
||||
|
||||
If `sha256` is provided, `generic-worker` will hash the downloaded archive
|
||||
and check it against the provided signature.
|
||||
|
||||
If `task_id` is provided, this task will depend on that task
|
||||
and `url_or_artifact_name` is the name of an artifact of that task.
|
||||
"""
|
||||
supported_formats = ["rar", "tar.bz2", "tar.gz", "zip"]
|
||||
for fmt in supported_formats:
|
||||
suffix = "." + fmt
|
||||
if url_or_artifact_name.endswith(suffix):
|
||||
return self.with_mounts({
|
||||
"directory": path or url_basename(url_or_artifact_name[:-len(suffix)]),
|
||||
"content": self._mount_content(url_or_artifact_name, task_id, sha256),
|
||||
"format": fmt,
|
||||
})
|
||||
raise ValueError(
|
||||
"%r does not appear to be in one of the supported formats: %r"
|
||||
% (url_or_artifact_name, ", ".join(supported_formats))
|
||||
) # pragma: no cover
|
||||
|
||||
|
||||
class WindowsGenericWorkerTask(GenericWorkerTask):
|
||||
"""
|
||||
Task definition for a `generic-worker` task running on Windows.
|
||||
|
||||
Scripts are written as `.bat` files executed with `cmd.exe`.
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.scripts = []
|
||||
self.rdp_info_artifact_name = None
|
||||
|
||||
with_script = chaining(append_to_attr, "scripts")
|
||||
with_early_script = chaining(prepend_to_attr, "scripts")
|
||||
|
||||
def build_worker_payload(self):
|
||||
if self.rdp_info_artifact_name:
|
||||
rdp_scope = "generic-worker:allow-rdp:%s/%s" % (self.provisioner_id, self.worker_type)
|
||||
self.scopes.append(rdp_scope)
|
||||
self.scopes.append("generic-worker:os-group:proj-servo/win2016/Administrators")
|
||||
self.scopes.append("generic-worker:run-as-administrator:proj-servo/win2016")
|
||||
self.with_features("runAsAdministrator")
|
||||
return dict_update_if_truthy(
|
||||
super().build_worker_payload(),
|
||||
rdpInfo=self.rdp_info_artifact_name,
|
||||
osGroups=["Administrators"]
|
||||
)
|
||||
|
||||
def with_rdp_info(self, *, artifact_name):
|
||||
"""
|
||||
Enable RDP access to this task’s environment.
|
||||
|
||||
See `rdpInfo` in
|
||||
<https://community-tc.services.mozilla.com/docs/reference/workers/generic-worker/multiuser-windows-payload>
|
||||
"""
|
||||
assert not artifact_name.startswith("public/")
|
||||
self.rdp_info_artifact_name = artifact_name
|
||||
|
||||
def build_command(self):
|
||||
return [deindent(s) for s in self.scripts]
|
||||
|
||||
def with_path_from_homedir(self, *paths):
|
||||
"""
|
||||
Interpret each path in `paths` as relative to the task’s home directory,
|
||||
and add it to the `PATH` environment variable.
|
||||
"""
|
||||
for p in paths:
|
||||
self.with_early_script("set PATH=%HOMEDRIVE%%HOMEPATH%\\{};%PATH%".format(p))
|
||||
return self
|
||||
|
||||
def with_repo(self, sparse_checkout=None):
|
||||
"""
|
||||
Make a clone the git repository at the start of the task.
|
||||
This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`,
|
||||
and creates the clone in a `repo` directory in the task’s home directory.
|
||||
|
||||
If `sparse_checkout` is given, it must be a list of path patterns
|
||||
to be used in `.git/info/sparse-checkout`.
|
||||
See <https://git-scm.com/docs/git-read-tree#_sparse_checkout>.
|
||||
"""
|
||||
git = """
|
||||
git init repo
|
||||
cd repo
|
||||
"""
|
||||
if sparse_checkout:
|
||||
self.with_mounts({
|
||||
"file": "sparse-checkout",
|
||||
"content": {"raw": "\n".join(sparse_checkout)},
|
||||
})
|
||||
git += """
|
||||
git config core.sparsecheckout true
|
||||
copy ..\\sparse-checkout .git\\info\\sparse-checkout
|
||||
type .git\\info\\sparse-checkout
|
||||
"""
|
||||
git += """
|
||||
git fetch --no-tags {} {}
|
||||
git reset --hard {}
|
||||
""".format(
|
||||
assert_truthy(self.git_fetch_url),
|
||||
assert_truthy(self.git_fetch_ref),
|
||||
assert_truthy(self.git_checkout_sha),
|
||||
)
|
||||
return self \
|
||||
.with_git() \
|
||||
.with_script(git)
|
||||
|
||||
def with_git(self):
|
||||
"""
|
||||
Make the task download `git-for-windows` and make it available for `git` commands.
|
||||
|
||||
This is implied by `with_repo`.
|
||||
"""
|
||||
return self \
|
||||
.with_path_from_homedir("git\\cmd") \
|
||||
.with_directory_mount(
|
||||
"https://github.com/git-for-windows/git/releases/download/" +
|
||||
"v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip",
|
||||
sha256="c33aec6ae68989103653ca9fb64f12cabccf6c61d0dde30c50da47fc15cf66e2",
|
||||
path="git",
|
||||
)
|
||||
|
||||
def with_curl_script(self, url, file_path):
|
||||
self.with_curl()
|
||||
return super().with_curl_script(url, file_path)
|
||||
|
||||
def with_curl(self):
|
||||
return self \
|
||||
.with_path_from_homedir("curl\\curl-7.73.0-win64-mingw\\bin") \
|
||||
.with_directory_mount(
|
||||
"https://curl.haxx.se/windows/dl-7.73.0/curl-7.73.0-win64-mingw.zip",
|
||||
sha256="2e1ffdb6c25c8648a1243bb3a268120be442399b1c93d7da309bba235ecdab9a",
|
||||
path="curl",
|
||||
)
|
||||
|
||||
def with_rustup(self):
|
||||
"""
|
||||
Download rustup.rs and make it available to task commands,
|
||||
but does not download any default toolchain.
|
||||
"""
|
||||
return self \
|
||||
.with_path_from_homedir(".cargo\\bin") \
|
||||
.with_early_script(
|
||||
"%HOMEDRIVE%%HOMEPATH%\\rustup-init.exe --default-toolchain none --profile=minimal -y"
|
||||
) \
|
||||
.with_file_mount("https://win.rustup.rs/x86_64", path="rustup-init.exe")
|
||||
|
||||
def with_repacked_msi(self, url, sha256, path):
|
||||
"""
|
||||
Download an MSI file from `url`, extract the files in it with `lessmsi`,
|
||||
and make them available in the directory at `path` (relative to the task’s home directory).
|
||||
|
||||
`sha256` is required and the MSI file must have that hash.
|
||||
|
||||
The file extraction (and recompression in a ZIP file) is done in a separate task,
|
||||
wich is indexed based on `sha256` and cached for `CONFIG.repacked_msi_files_expire_in`.
|
||||
|
||||
<https://github.com/activescott/lessmsi>
|
||||
"""
|
||||
repack_task = (
|
||||
WindowsGenericWorkerTask("MSI repack: " + url)
|
||||
.with_worker_type(self.worker_type)
|
||||
.with_max_run_time_minutes(20)
|
||||
.with_file_mount(url, sha256=sha256, path="input.msi")
|
||||
.with_directory_mount(
|
||||
"https://github.com/activescott/lessmsi/releases/download/" +
|
||||
"v1.6.1/lessmsi-v1.6.1.zip",
|
||||
sha256="540b8801e08ec39ba26a100c855898f455410cecbae4991afae7bb2b4df026c7",
|
||||
path="lessmsi"
|
||||
)
|
||||
.with_directory_mount(
|
||||
"https://www.7-zip.org/a/7za920.zip",
|
||||
sha256="2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac",
|
||||
path="7zip",
|
||||
)
|
||||
.with_path_from_homedir("lessmsi", "7zip")
|
||||
.with_script("""
|
||||
lessmsi x input.msi extracted\\
|
||||
cd extracted\\SourceDir
|
||||
7za a repacked.zip *
|
||||
""")
|
||||
.with_artifacts("extracted/SourceDir/repacked.zip")
|
||||
.with_index_and_artifacts_expire_in(CONFIG.repacked_msi_files_expire_in)
|
||||
.find_or_create("repacked-msi." + sha256)
|
||||
)
|
||||
return self \
|
||||
.with_dependencies(repack_task) \
|
||||
.with_directory_mount("public/repacked.zip", task_id=repack_task, path=path)
|
||||
|
||||
def with_python3(self):
|
||||
"""
|
||||
For Python 3, use `with_directory_mount` and the "embeddable zip file" distribution
|
||||
from python.org.
|
||||
You may need to remove `python37._pth` from the ZIP in order to work around
|
||||
<https://bugs.python.org/issue34841>.
|
||||
"""
|
||||
return (
|
||||
self
|
||||
.with_curl_script(
|
||||
"https://www.python.org/ftp/python/3.7.3/python-3.7.3-amd64.exe",
|
||||
"do-the-python.exe"
|
||||
)
|
||||
.with_script("do-the-python.exe /quiet TargetDir=%HOMEDRIVE%%HOMEPATH%\\python3")
|
||||
.with_path_from_homedir("python3", "python3\\Scripts")
|
||||
.with_script("pip install virtualenv==20.2.1")
|
||||
)
|
||||
|
||||
|
||||
class UnixTaskMixin(Task):
|
||||
def with_repo(self, alternate_object_dir=""):
|
||||
"""
|
||||
Make a clone the git repository at the start of the task.
|
||||
This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`
|
||||
|
||||
* generic-worker: creates the clone in a `repo` directory
|
||||
in the task’s directory.
|
||||
|
||||
* docker-worker: creates the clone in a `/repo` directory
|
||||
at the root of the Docker container’s filesystem.
|
||||
`git` and `ca-certificate` need to be installed in the Docker image.
|
||||
|
||||
"""
|
||||
# Not using $GIT_ALTERNATE_OBJECT_DIRECTORIES since it causes
|
||||
# "object not found - no match for id" errors when Cargo fetches git dependencies
|
||||
return self \
|
||||
.with_script("""
|
||||
git init repo
|
||||
cd repo
|
||||
echo "{alternate}" > .git/objects/info/alternates
|
||||
time git fetch --no-tags {} {}
|
||||
time git reset --hard {}
|
||||
""".format(
|
||||
assert_truthy(self.git_fetch_url),
|
||||
assert_truthy(self.git_fetch_ref),
|
||||
assert_truthy(self.git_checkout_sha),
|
||||
alternate=alternate_object_dir,
|
||||
))
|
||||
|
||||
|
||||
class DockerWorkerTask(UnixTaskMixin, Task):
|
||||
"""
|
||||
Task definition for a worker type that runs the `generic-worker` implementation.
|
||||
|
||||
Scripts are interpreted with `bash`.
|
||||
|
||||
<https://github.com/taskcluster/docker-worker>
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.docker_image = "ubuntu:bionic-20180821"
|
||||
self.max_run_time_minutes = 30
|
||||
self.scripts = []
|
||||
self.env = {}
|
||||
self.caches = {}
|
||||
self.features = {}
|
||||
self.capabilities = {}
|
||||
self.artifacts = []
|
||||
|
||||
with_docker_image = chaining(setattr, "docker_image")
|
||||
with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes")
|
||||
with_script = chaining(append_to_attr, "scripts")
|
||||
with_early_script = chaining(prepend_to_attr, "scripts")
|
||||
with_caches = chaining(update_attr, "caches")
|
||||
with_env = chaining(update_attr, "env")
|
||||
with_capabilities = chaining(update_attr, "capabilities")
|
||||
|
||||
def with_artifacts(self, *paths):
|
||||
for path in paths:
|
||||
if path in self.artifacts:
|
||||
raise ValueError("Duplicate artifact: " + path) # pragma: no cover
|
||||
self.artifacts.append(path)
|
||||
return self
|
||||
|
||||
def build_worker_payload(self):
|
||||
"""
|
||||
Return a `docker-worker` worker payload.
|
||||
|
||||
<https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/payload>
|
||||
"""
|
||||
worker_payload = {
|
||||
"image": self.docker_image,
|
||||
"maxRunTime": self.max_run_time_minutes * 60,
|
||||
"command": [
|
||||
"/bin/bash", "--login", "-x", "-e", "-o", "pipefail", "-c",
|
||||
deindent("\n".join(self.scripts))
|
||||
],
|
||||
}
|
||||
return dict_update_if_truthy(
|
||||
worker_payload,
|
||||
env=self.env,
|
||||
cache=self.caches,
|
||||
features=self.features,
|
||||
capabilities=self.capabilities,
|
||||
artifacts={
|
||||
"public/" + url_basename(path): {
|
||||
"type": "file",
|
||||
"path": path,
|
||||
"expires": SHARED.from_now_json(self.index_and_artifacts_expire_in),
|
||||
}
|
||||
for path in self.artifacts
|
||||
},
|
||||
)
|
||||
|
||||
def with_features(self, *names):
|
||||
"""
|
||||
Enable the given `docker-worker` features.
|
||||
|
||||
<https://github.com/taskcluster/docker-worker/blob/master/docs/features.md>
|
||||
"""
|
||||
self.features.update({name: True for name in names})
|
||||
return self
|
||||
|
||||
def with_dockerfile(self, dockerfile):
|
||||
"""
|
||||
Build a Docker image based on the given `Dockerfile`, and use it for this task.
|
||||
|
||||
`dockerfile` is a path in the filesystem where this code is running.
|
||||
Some non-standard syntax is supported, see `expand_dockerfile`.
|
||||
|
||||
The image is indexed based on a hash of the expanded `Dockerfile`,
|
||||
and cached for `CONFIG.docker_images_expire_in`.
|
||||
|
||||
Images are built without any *context*.
|
||||
<https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#understand-build-context>
|
||||
"""
|
||||
basename = os.path.basename(dockerfile)
|
||||
suffix = ".dockerfile"
|
||||
assert basename.endswith(suffix)
|
||||
image_name = basename[:-len(suffix)]
|
||||
|
||||
dockerfile_contents = expand_dockerfile(dockerfile)
|
||||
digest = hashlib.sha256(dockerfile_contents).hexdigest()
|
||||
|
||||
image_build_task = (
|
||||
DockerWorkerTask("Docker image: " + image_name)
|
||||
.with_worker_type(CONFIG.docker_image_build_worker_type or self.worker_type)
|
||||
.with_max_run_time_minutes(30)
|
||||
.with_index_and_artifacts_expire_in(CONFIG.docker_images_expire_in)
|
||||
.with_features("dind")
|
||||
.with_env(DOCKERFILE=dockerfile_contents)
|
||||
.with_artifacts("/image.tar.lz4")
|
||||
.with_script("""
|
||||
echo "$DOCKERFILE" | docker build -t taskcluster-built -
|
||||
docker save taskcluster-built | lz4 > /image.tar.lz4
|
||||
""")
|
||||
.with_docker_image(
|
||||
# https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
|
||||
"servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
|
||||
"0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"
|
||||
)
|
||||
.find_or_create("docker-image." + digest)
|
||||
)
|
||||
|
||||
return self \
|
||||
.with_dependencies(image_build_task) \
|
||||
.with_docker_image({
|
||||
"type": "task-image",
|
||||
"path": "public/image.tar.lz4",
|
||||
"taskId": image_build_task,
|
||||
})
|
||||
|
||||
|
||||
def expand_dockerfile(dockerfile):
|
||||
"""
|
||||
Read the file at path `dockerfile`,
|
||||
and transitively expand the non-standard `% include` header if it is present.
|
||||
"""
|
||||
with open(dockerfile, "rb") as f:
|
||||
dockerfile_contents = f.read()
|
||||
|
||||
include_marker = b"% include"
|
||||
if not dockerfile_contents.startswith(include_marker):
|
||||
return dockerfile_contents
|
||||
|
||||
include_line, _, rest = dockerfile_contents.partition(b"\n")
|
||||
included = include_line[len(include_marker):].strip().decode("utf8")
|
||||
path = os.path.join(os.path.dirname(dockerfile), included)
|
||||
return b"\n".join([expand_dockerfile(path), rest])
|
||||
|
||||
|
||||
def assert_truthy(x):
|
||||
assert x
|
||||
return x
|
||||
|
||||
|
||||
def dict_update_if_truthy(d, **kwargs):
|
||||
for key, value in kwargs.items():
|
||||
if value:
|
||||
d[key] = value
|
||||
return d
|
||||
|
||||
|
||||
def deindent(string):
|
||||
return re.sub("\n +", "\n ", string).strip()
|
||||
|
||||
|
||||
def url_basename(url):
|
||||
return url.rpartition("/")[-1]
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def make_repo_bundle():
|
||||
subprocess.check_call(["git", "config", "user.name", "Decision task"])
|
||||
subprocess.check_call(["git", "config", "user.email", "nobody@mozilla.com"])
|
||||
tree = subprocess.check_output(["git", "show", CONFIG.git_sha, "--pretty=%T", "--no-patch"])
|
||||
message = "Shallow version of commit " + CONFIG.git_sha
|
||||
commit = subprocess.check_output(["git", "commit-tree", tree.strip(), "-m", message])
|
||||
subprocess.check_call(["git", "update-ref", CONFIG.git_bundle_shallow_ref, commit.strip()])
|
||||
subprocess.check_call(["git", "show-ref"])
|
||||
create = ["git", "bundle", "create", "../repo.bundle", CONFIG.git_bundle_shallow_ref]
|
||||
with subprocess.Popen(create) as p:
|
||||
yield
|
||||
exit_code = p.wait()
|
||||
if exit_code:
|
||||
sys.exit(exit_code)
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
FROM ubuntu:20.04
|
||||
|
||||
ENV \
|
||||
#
|
||||
# Some APT packages like 'tzdata' wait for user input on install by default.
|
||||
# https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive
|
||||
DEBIAN_FRONTEND=noninteractive \
|
||||
LANG=C.UTF-8 \
|
||||
LANGUAGE=C.UTF-8 \
|
||||
LC_ALL=C.UTF-8
|
||||
|
||||
RUN \
|
||||
apt-get update -q && \
|
||||
apt-get install -qy --no-install-recommends \
|
||||
#
|
||||
# Cloning the repository
|
||||
git \
|
||||
ca-certificates \
|
||||
#
|
||||
# Running mach with Python 3
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
virtualenv \
|
||||
#
|
||||
# Compiling C modules when installing Python packages in a virtualenv
|
||||
gcc \
|
||||
#
|
||||
# Installing rustup and sccache (build dockerfile) or fetching build artifacts (run tasks)
|
||||
curl \
|
||||
# Setting the default locale
|
||||
locales \
|
||||
locales-all
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
% include base.dockerfile
|
||||
|
||||
RUN \
|
||||
apt-get install -qy --no-install-recommends \
|
||||
#
|
||||
# Testing decisionlib (see etc/taskcluster/mock.py)
|
||||
python3-coverage \
|
||||
#
|
||||
# Multiple C/C++ dependencies built from source
|
||||
g++ \
|
||||
make \
|
||||
cmake \
|
||||
#
|
||||
# Fontconfig
|
||||
gperf \
|
||||
#
|
||||
# ANGLE
|
||||
xorg-dev \
|
||||
#
|
||||
# mozjs (SpiderMonkey)
|
||||
autoconf2.13 \
|
||||
#
|
||||
# Bindgen (for SpiderMonkey bindings)
|
||||
clang \
|
||||
llvm \
|
||||
llvm-dev \
|
||||
#
|
||||
# GStreamer
|
||||
libpcre3-dev \
|
||||
#
|
||||
# OpenSSL
|
||||
libssl-dev \
|
||||
#
|
||||
# blurz
|
||||
libdbus-1-dev \
|
||||
#
|
||||
# sampling profiler
|
||||
libunwind-dev \
|
||||
#
|
||||
# x11 integration
|
||||
libxcb-render-util0-dev \
|
||||
libxcb-shape0-dev \
|
||||
libxcb-xfixes0-dev \
|
||||
#
|
||||
&& \
|
||||
#
|
||||
# Install the version of rustup that is current when this Docker image is being built:
|
||||
# We want at least 1.21 (increment in this comment to force an image rebuild).
|
||||
curl https://sh.rustup.rs -sSf | sh -s -- --profile=minimal -y && \
|
||||
#
|
||||
# There are no sccache binary releases that include this commit, so we install a particular
|
||||
# git commit instead.
|
||||
~/.cargo/bin/cargo install sccache --git https://github.com/mozilla/sccache/ --rev e66c9c15142a7e583d6ab80bd614bdffb2ebcc47
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
% include base.dockerfile
|
||||
|
||||
RUN \
|
||||
apt-get install -qy --no-install-recommends \
|
||||
#
|
||||
# Multiple Android-related tools are in Java
|
||||
openjdk-8-jdk-headless \
|
||||
#
|
||||
# Emulator dependencies
|
||||
libgl1 \
|
||||
libpulse0
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
% include base.dockerfile
|
||||
|
||||
# Servo’s runtime dependencies:
|
||||
RUN apt-get install -qy --no-install-recommends \
|
||||
libgl1 \
|
||||
libssl1.1 \
|
||||
libdbus-1-3 \
|
||||
libxcb-shape0-dev \
|
||||
gstreamer1.0-plugins-good \
|
||||
gstreamer1.0-plugins-bad \
|
||||
gstreamer1.0-libav \
|
||||
gstreamer1.0-gl \
|
||||
libunwind8
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
% include run.dockerfile
|
||||
|
||||
RUN apt-get install -qy --no-install-recommends \
|
||||
python3 \
|
||||
jq
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2018 The Servo Project Developers. See the COPYRIGHT
|
||||
# file at the top-level directory of this distribution.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
# option. This file may not be copied, modified, or distributed
|
||||
# except according to those terms.
|
||||
|
||||
''''set -e
|
||||
python3 -m coverage run $0
|
||||
python3 -m coverage report -m --fail-under 100
|
||||
exit
|
||||
|
||||
Run the decision task with fake Taskcluster APIs, to catch Python errors before pushing.
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
||||
class TaskclusterRestFailure(Exception):
|
||||
status_code = 404
|
||||
|
||||
|
||||
class Index:
|
||||
__init__ = insertTask = lambda *_, **__: None
|
||||
|
||||
def findTask(self, path):
|
||||
if decision_task.CONFIG.git_ref == "refs/heads/master":
|
||||
return {"taskId": "<from index>"}
|
||||
raise TaskclusterRestFailure
|
||||
|
||||
|
||||
stringDate = str
|
||||
slugId = b"<new id>".lower
|
||||
sys.exit = Queue = fromNow = MagicMock()
|
||||
sys.modules["taskcluster"] = sys.modules[__name__]
|
||||
sys.dont_write_bytecode = True
|
||||
os.environ.update(**{k: k for k in "TASK_ID TASK_OWNER TASK_SOURCE GIT_URL GIT_SHA".split()})
|
||||
os.environ["GIT_REF"] = "refs/heads/auto"
|
||||
os.environ["TASKCLUSTER_ROOT_URL"] = "https://community-tc.services.mozilla.com"
|
||||
os.environ["TASKCLUSTER_PROXY_URL"] = "http://taskcluster"
|
||||
os.environ["NEW_AMI_WORKER_TYPE"] = "-"
|
||||
import decision_task # noqa: E402
|
||||
decision_task.decisionlib.subprocess = MagicMock()
|
||||
|
||||
print("\n# Push:")
|
||||
decision_task.main("github-push")
|
||||
|
||||
print("\n# Push with hot caches:")
|
||||
decision_task.main("github-push")
|
||||
|
||||
print("\n# Push to master:")
|
||||
decision_task.CONFIG.git_ref = "refs/heads/master"
|
||||
decision_task.main("github-push")
|
||||
|
||||
print("\n# Daily:")
|
||||
decision_task.main("daily")
|
||||
|
||||
print("\n# Try AMI:")
|
||||
decision_task.main("try-windows-ami")
|
||||
|
||||
print("\n# PR:")
|
||||
decision_task.main("github-pull-request")
|
||||
|
||||
print()
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
''''set -e
|
||||
cd "$(dirname $0)"
|
||||
exec ../../python/_virtualenv/bin/python "$(basename $0)"
|
||||
'''
|
||||
|
||||
try:
|
||||
import jsone
|
||||
except ImportError:
|
||||
import sys
|
||||
sys.exit("pip install git+https://github.com/taskcluster/json-e")
|
||||
|
||||
import yaml
|
||||
import json
|
||||
|
||||
template = yaml.load(open("../../.taskcluster.yml").read().decode("utf8"))
|
||||
repo = dict(
|
||||
repository=dict(
|
||||
clone_url="https://github.com/servo/servo.git",
|
||||
),
|
||||
)
|
||||
contexts = [
|
||||
dict(
|
||||
tasks_for="github-release",
|
||||
event=repo,
|
||||
),
|
||||
dict(
|
||||
tasks_for="github-pull-request",
|
||||
event=dict(
|
||||
action="comment",
|
||||
**repo
|
||||
),
|
||||
),
|
||||
dict(
|
||||
tasks_for="github-push",
|
||||
event=dict(
|
||||
ref="refs/heads/master",
|
||||
compare="https://github.com/servo/servo/compare/1753cda...de09c8f",
|
||||
after="de09c8fb6ef87dec5932d5fab4adcb421d291a54",
|
||||
pusher=dict(
|
||||
name="bors-servo",
|
||||
),
|
||||
**repo
|
||||
),
|
||||
),
|
||||
dict(
|
||||
tasks_for="github-pull-request",
|
||||
event=dict(
|
||||
action="synchronize",
|
||||
pull_request=dict(
|
||||
number=22583,
|
||||
url="https://github.com/servo/servo/pull/22583",
|
||||
head=dict(
|
||||
sha="51a422c9ef47420eb69c802643b7686bdb498652",
|
||||
),
|
||||
merge_commit_sha="876fcf7a5fe971a9ac0a4ce117906c552c08c095",
|
||||
),
|
||||
sender=dict(
|
||||
login="jdm",
|
||||
),
|
||||
**repo
|
||||
),
|
||||
),
|
||||
]
|
||||
for context in contexts:
|
||||
print(context["tasks_for"])
|
||||
print(json.dumps(jsone.render(template, context), indent=2))
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
# Treeherder for Servo
|
||||
|
||||
Treeherder is tool for visualizing the status of “trees”,
|
||||
meaning branches in various source repositories.
|
||||
It shows each push to the repository with the corresponding commits
|
||||
as well as the CI jobs that were started for that push.
|
||||
While it is possible to write other tools that submit job data,
|
||||
CI integration is easiest with Taskcluster.
|
||||
|
||||
* [Production instance](https://treeherder.mozilla.org/)
|
||||
* [Staging instance](https://treeherder.allizom.org/)
|
||||
* [Source code](https://github.com/mozilla/treeherder/)
|
||||
|
||||
|
||||
## Trees / repositories / branches
|
||||
|
||||
Treeherders knows a about a number of *repostories*.
|
||||
Mercurial on Mozilla’s servers and git on GitHub are supported.
|
||||
Despite the name, in the GitHub case
|
||||
each Treeherder repository maps to one branch in a git repository.
|
||||
They are configured in the [`repository.json`] file.
|
||||
As of this writing there are four for `github.com/servo/servo`,
|
||||
named after the corresponding branch:
|
||||
|
||||
[`repository.json`]: https://github.com/mozilla/treeherder/blob/master/treeherder/model/fixtures/repository.json
|
||||
|
||||
* [`servo-master`](https://treeherder.mozilla.org/#/jobs?repo=servo-master)
|
||||
* [`servo-auto`](https://treeherder.mozilla.org/#/jobs?repo=servo-auto)
|
||||
* [`servo-try`](https://treeherder.mozilla.org/#/jobs?repo=servo-try)
|
||||
* [`servo-try-taskcluster`](https://treeherder.mozilla.org/#/jobs?repo=servo-try-taskcluster)
|
||||
|
||||
In the UI, the “Repos” button near the top right corner allows switching.
|
||||
|
||||
`servo-auto` is the relevant one when a pull request is approved with Homu for landing,
|
||||
since the `auto` branch is where it pushes a merge commit for testing.
|
||||
|
||||
|
||||
## Data flow / how it all works
|
||||
|
||||
(This section is mostly useful for future changes or troubleshooting.)
|
||||
|
||||
Changes to the Treeherder repository are deployed to Staging
|
||||
soon (minutes) after they are merged on GitHub,
|
||||
and to Production manually at some point later.
|
||||
See [current deployment status](https://whatsdeployed.io/s-dqv).
|
||||
|
||||
Once a configuration change with a new repository/branch is deployed,
|
||||
Treeherder will show it in its UI and start recording push and job data in its database.
|
||||
This data comes from [Pulse], Mozilla’s shared message queue that coordinates separate services.
|
||||
The [Pulse Inspector] shows messages as they come (though not in the past),
|
||||
which can be useful for debugging.
|
||||
Note that you need to add at least one “Binding”,
|
||||
or the “Start Listening” button won’t do anything.
|
||||
|
||||
[Pulse]: https://wiki.mozilla.org/Auto-tools/Projects/Pulse
|
||||
[Pulse Inspector]: https://community-tc.services.mozilla.com/pulse-messages
|
||||
|
||||
|
||||
### Push data
|
||||
|
||||
When [taskcluster-github] is [enabled] on a repository,
|
||||
it recieves [webhooks] from GitHub for various events
|
||||
such as a push to a branch of the repository.
|
||||
|
||||
In addition to starting Taskcluster tasks based on `.taskcluster.yml` in the repository,
|
||||
in [`api.js`] it creates [Pulse messages] corresponding to those events.
|
||||
Treeherder consumes messages from the `exchange/taskcluster-github/v1/push` exchange
|
||||
(among others) in [`push_loader.py`].
|
||||
In Pulse Inspector, these messages for the Servo repository can be seen
|
||||
by specifying the [`primary.servo.servo`] routing key pattern.
|
||||
|
||||
[taskcluster-github]: https://github.com/taskcluster/taskcluster/tree/master/services/github
|
||||
[enabled]: https://github.com/apps/community-tc-integration/
|
||||
[webhooks]: https://developer.github.com/webhooks/
|
||||
[Pulse messages]: https://community-tc.services.mozilla.com/docs/reference/integrations/github/exchanges
|
||||
[`api.js`]: https://github.com/taskcluster/taskcluster/blob/master/services/github/src/api.js
|
||||
[`push_loader.py`]: https://github.com/mozilla/treeherder/blob/master/treeherder/etl/push_loader.py
|
||||
[`primary.servo.servo`]: https://community-tc.services.mozilla.com/pulse-messages?bindings%5B0%5D%5Bexchange%5D=exchange%2Ftaskcluster-github%2Fv1%2Fpush&bindings%5B0%5D%5BroutingKeyPattern%5D=primary.servo.servo
|
||||
|
||||
|
||||
### (Taskcluster) job data
|
||||
|
||||
The Taskcluster Queue generates a number of [Pulse messages about tasks].
|
||||
Each value of the `routes` array in the task definition, with a `route.` prefix prepended,
|
||||
is additional routing key for those messages.
|
||||
|
||||
Treeherder reads those messages
|
||||
if they have an appropriate route ([see in Pulse inspector][inspector1]),
|
||||
However, it will drop an incoming message
|
||||
if the `extra.treeherder` object in the task definition doesn’t conform to the [schema].
|
||||
Such schema validation errors are logged, but those logs are not easy to access.
|
||||
Ask on IRC on `#taskcluster`.
|
||||
|
||||
Finally, Treeherder reads that latter kind of message in [`job_loader.py`].
|
||||
|
||||
|
||||
|
||||
[Pulse messages about tasks]: https://community-tc.services.mozilla.com/docs/reference/platform/taskcluster-queue/references/events
|
||||
[taskcluster-treeherder]: https://github.com/taskcluster/taskcluster-treeherder/
|
||||
[other messages]: https://community-tc.services.mozilla.com/docs/reference/integrations/taskcluster-treeherder#job-pulse-messages
|
||||
[schema]: https://schemas.taskcluster.net/treeherder/v1/task-treeherder-config.json
|
||||
[`job_loader.py`]: https://github.com/mozilla/treeherder/blob/master/treeherder/etl/job_loader.py
|
||||
[inspector1]: https://tools.taskcluster.net/pulse-inspector?bindings%5B0%5D%5Bexchange%5D=exchange%2Ftaskcluster-queue%2Fv1%2Ftask-defined&bindings%5B0%5D%5BroutingKeyPattern%5D=route.tc-treeherder.%23
|
||||
|
|
@ -21,7 +21,6 @@ import shutil
|
|||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib
|
||||
import xml
|
||||
|
||||
from mach.decorators import (
|
||||
|
|
@ -98,15 +97,6 @@ else:
|
|||
raise e
|
||||
|
||||
|
||||
def get_taskcluster_secret(name):
|
||||
url = (
|
||||
os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster")
|
||||
+ "/api/secrets/v1/secret/project/servo/"
|
||||
+ name
|
||||
)
|
||||
return json.load(urllib.request.urlopen(url))["secret"]
|
||||
|
||||
|
||||
def otool(s):
|
||||
o = subprocess.Popen(['/usr/bin/otool', '-L', s], stdout=subprocess.PIPE)
|
||||
for line in map(lambda s: s.decode('ascii'), o.stdout):
|
||||
|
|
@ -601,23 +591,16 @@ class PackageCommands(CommandBase):
|
|||
@CommandArgument('platform',
|
||||
choices=PACKAGES.keys(),
|
||||
help='Package platform type to upload')
|
||||
@CommandArgument('--secret-from-taskcluster',
|
||||
action='store_true',
|
||||
help='Retrieve the appropriate secrets from taskcluster.')
|
||||
@CommandArgument('--secret-from-environment',
|
||||
action='store_true',
|
||||
help='Retrieve the appropriate secrets from the environment.')
|
||||
def upload_nightly(self, platform, secret_from_taskcluster, secret_from_environment):
|
||||
def upload_nightly(self, platform, secret_from_environment):
|
||||
import boto3
|
||||
|
||||
def get_s3_secret():
|
||||
aws_access_key = None
|
||||
aws_secret_access_key = None
|
||||
if secret_from_taskcluster:
|
||||
secret = get_taskcluster_secret("s3-upload-credentials")
|
||||
aws_access_key = secret["aws_access_key_id"]
|
||||
aws_secret_access_key = secret["aws_secret_access_key"]
|
||||
elif secret_from_environment:
|
||||
if secret_from_environment:
|
||||
secret = json.loads(os.environ['S3_UPLOAD_CREDENTIALS'])
|
||||
aws_access_key = secret["aws_access_key_id"]
|
||||
aws_secret_access_key = secret["aws_secret_access_key"]
|
||||
|
|
@ -758,9 +741,6 @@ class PackageCommands(CommandBase):
|
|||
'--message=Version Bump: {}'.format(brew_version),
|
||||
])
|
||||
|
||||
if secret_from_taskcluster:
|
||||
token = get_taskcluster_secret('github-homebrew-token')["token"]
|
||||
else:
|
||||
token = os.environ['GITHUB_HOMEBREW_TOKEN']
|
||||
|
||||
push_url = 'https://{}@github.com/servo/homebrew-servo.git'
|
||||
|
|
@ -804,8 +784,6 @@ def setup_uwp_signing(ms_app_store, publisher):
|
|||
if ms_app_store:
|
||||
return ["/p:AppxPackageSigningEnabled=false"]
|
||||
|
||||
is_tc = "TASKCLUSTER_PROXY_URL" in os.environ
|
||||
|
||||
def run_powershell_cmd(cmd):
|
||||
try:
|
||||
return (
|
||||
|
|
@ -818,10 +796,7 @@ def setup_uwp_signing(ms_app_store, publisher):
|
|||
exit(1)
|
||||
|
||||
pfx = None
|
||||
if is_tc:
|
||||
print("Packaging on TC. Using secret certificate")
|
||||
pfx = get_taskcluster_secret("windows-codesign-cert/latest")["pfx"]["base64"]
|
||||
elif 'CODESIGN_CERT' in os.environ:
|
||||
if 'CODESIGN_CERT' in os.environ:
|
||||
pfx = os.environ['CODESIGN_CERT']
|
||||
|
||||
if pfx:
|
||||
|
|
@ -832,10 +807,7 @@ def setup_uwp_signing(ms_app_store, publisher):
|
|||
# Powershell command that lists all certificates for publisher
|
||||
cmd = '(dir cert: -Recurse | Where-Object {$_.Issuer -eq "' + publisher + '"}).Thumbprint'
|
||||
certs = list(set(run_powershell_cmd(cmd).splitlines()))
|
||||
if not certs and is_tc:
|
||||
print("Error: No certificate installed for publisher " + publisher)
|
||||
exit(1)
|
||||
if not certs and not is_tc:
|
||||
if not certs:
|
||||
print("No certificate installed for publisher " + publisher)
|
||||
print("Creating and installing a temporary certificate")
|
||||
# PowerShell command that creates and install signing certificate for publisher
|
||||
|
|
|
|||
|
|
@ -123,9 +123,6 @@ files = [
|
|||
"./tests/wpt/mozilla/tests/css/pre_with_tab.html",
|
||||
"./tests/wpt/mozilla/tests/mozilla/textarea_placeholder.html",
|
||||
# Python 3 syntax causes "E901 SyntaxError" when flake8 runs in Python 2
|
||||
"./etc/taskcluster/decision_task.py",
|
||||
"./etc/taskcluster/decisionlib.py",
|
||||
"./tests/wpt/reftests-report/gen.py",
|
||||
"./components/style/properties/build.py",
|
||||
]
|
||||
# Directories that are ignored for the non-WPT tidy check.
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from html import escape as html_escape
|
||||
|
||||
|
||||
TASKCLUSTER_ROOT_URL = "https://community-tc.services.mozilla.com"
|
||||
|
||||
|
||||
def fetch(url):
|
||||
url = TASKCLUSTER_ROOT_URL + "/api/" + url
|
||||
print("Fetching " + url)
|
||||
response = urllib.request.urlopen(url)
|
||||
assert response.getcode() == 200
|
||||
encoding = response.info().get("Content-Encoding")
|
||||
if not encoding:
|
||||
return response
|
||||
elif encoding == "gzip":
|
||||
return gzip.GzipFile(fileobj=response)
|
||||
else:
|
||||
raise ValueError("Unsupported Content-Encoding: %s" % encoding)
|
||||
|
||||
|
||||
def fetch_json(url):
|
||||
with fetch(url) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def task(platform, chunk, key):
|
||||
return "index/v1/task/project.servo.%s_wpt_%s.%s" % (platform, chunk, key)
|
||||
|
||||
|
||||
def failing_reftests(platform, key):
|
||||
chunk_1_task_id = fetch_json(task(platform, 1, key))["taskId"]
|
||||
name = fetch_json("queue/v1/task/" + chunk_1_task_id)["metadata"]["name"]
|
||||
match = re.search("WPT chunk (\d+) / (\d+)", name)
|
||||
assert match.group(1) == "1"
|
||||
total_chunks = int(match.group(2))
|
||||
|
||||
for chunk in range(1, total_chunks + 1):
|
||||
with fetch(task(platform, chunk, key) + "/artifacts/public/test-wpt.log") as response:
|
||||
yield from parse(response)
|
||||
|
||||
|
||||
def parse(file_like):
|
||||
seen = set()
|
||||
for line in file_like:
|
||||
message = json.loads(line)
|
||||
status = message.get("status")
|
||||
if status not in {None, "OK", "PASS"}:
|
||||
screenshots = message.get("extra", {}).get("reftest_screenshots")
|
||||
if screenshots:
|
||||
url = message["test"]
|
||||
assert url.startswith("/")
|
||||
yield url[1:], message.get("expected") == "PASS", screenshots
|
||||
|
||||
|
||||
def main(source, commit_sha=None):
|
||||
failures = Directory()
|
||||
|
||||
if commit_sha:
|
||||
title = "<h1>Layout 2020 regressions in commit <code>%s</code></h1>" % commit_sha
|
||||
failures_2013 = {url for url, _, _ in failing_reftests("linux_x64", source)}
|
||||
for url, _expected_pass, screenshots in failing_reftests("linux_x64_2020", source):
|
||||
if url not in failures_2013:
|
||||
failures.add(url, screenshots)
|
||||
else:
|
||||
title = "Unexpected failures"
|
||||
with open(source, "rb") as file_obj:
|
||||
for url, expected_pass, screenshots in parse(file_obj):
|
||||
if expected_pass:
|
||||
failures.add(url, screenshots)
|
||||
|
||||
here = os.path.dirname(__file__)
|
||||
with open(os.path.join(here, "prism.js")) as f:
|
||||
prism_js = f.read()
|
||||
with open(os.path.join(here, "prism.css")) as f:
|
||||
prism_css = f.read()
|
||||
with open(os.path.join(here, "report.html"), "w", encoding="utf-8") as html:
|
||||
os.chdir(os.path.join(here, ".."))
|
||||
html.write("""
|
||||
<!doctype html>
|
||||
<meta charset=utf-8>
|
||||
<title>WPT reftests failures report</title>
|
||||
<link rel=stylesheet href=prism.css>
|
||||
<style>
|
||||
ul { padding-left: 1em }
|
||||
li { list-style: "⯈ " }
|
||||
li.expanded { list-style: "⯆ " }
|
||||
li:not(.expanded) > ul, li:not(.expanded) > div { display: none }
|
||||
li > div { display: grid; grid-gap: 1em; grid-template-columns: 1fr 1fr }
|
||||
li > div > p { grid-column: span 2 }
|
||||
li > div > img { grid-row: 2; width: 300px; box-shadow: 0 0 10px }
|
||||
li > div > img:hover { transform: scale(3); transform-origin: 0 0 }
|
||||
li > div > pre { grid-row: 3; font-size: 12px !important }
|
||||
pre code { white-space: pre-wrap !important }
|
||||
<h1>%s</h1>
|
||||
</style>
|
||||
%s
|
||||
""" % (prism_css, title))
|
||||
failures.write(html)
|
||||
html.write("""
|
||||
<script>
|
||||
for (let li of document.getElementsByTagName("li")) {
|
||||
li.addEventListener('click', event => {
|
||||
li.classList.toggle("expanded")
|
||||
event.stopPropagation()
|
||||
})
|
||||
}
|
||||
%s
|
||||
</script>
|
||||
""" % prism_js)
|
||||
|
||||
|
||||
class Directory:
|
||||
def __init__(self):
|
||||
self.count = 0
|
||||
self.contents = {}
|
||||
|
||||
def add(self, path, screenshots):
|
||||
self.count += 1
|
||||
first, _, rest = path.partition("/")
|
||||
if rest:
|
||||
self.contents.setdefault(first, Directory()).add(rest, screenshots)
|
||||
else:
|
||||
assert path not in self.contents
|
||||
self.contents[path] = screenshots
|
||||
|
||||
def write(self, html):
|
||||
html.write("<ul>\n")
|
||||
for k, v in self.contents.items():
|
||||
html.write("<li><code>%s</code>\n" % k)
|
||||
if isinstance(v, Directory):
|
||||
html.write("<strong>%s</strong>\n" % v.count)
|
||||
v.write(html)
|
||||
else:
|
||||
a, rel, b = v
|
||||
html.write("<div>\n<p><code>%s</code> %s <code>%s</code></p>\n"
|
||||
% (a["url"], rel, b["url"]))
|
||||
for side in [a, b]:
|
||||
html.write("<img src='data:image/png;base64,%s'>\n" % side["screenshot"])
|
||||
url = side["url"]
|
||||
prefix = "/_mozilla/"
|
||||
if url.startswith(prefix):
|
||||
filename = "mozilla/tests/" + url[len(prefix):]
|
||||
elif url == "about:blank":
|
||||
src = ""
|
||||
filename = None
|
||||
else:
|
||||
filename = "web-platform-tests" + url
|
||||
if filename:
|
||||
with open(filename, encoding="utf-8") as f:
|
||||
src = html_escape(f.read())
|
||||
html.write("<pre><code class=language-html>%s</code></pre>\n" % src)
|
||||
html.write("</li>\n")
|
||||
html.write("</ul>\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(*sys.argv[1:]))
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
/* PrismJS 1.19.0
|
||||
https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */
|
||||
/**
|
||||
* prism.js default theme for JavaScript, CSS and HTML
|
||||
* Based on dabblet (http://dabblet.com)
|
||||
* @author Lea Verou
|
||||
*/
|
||||
|
||||
code[class*="language-"],
|
||||
pre[class*="language-"] {
|
||||
color: black;
|
||||
background: none;
|
||||
text-shadow: 0 1px white;
|
||||
font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
|
||||
font-size: 1em;
|
||||
text-align: left;
|
||||
white-space: pre;
|
||||
word-spacing: normal;
|
||||
word-break: normal;
|
||||
word-wrap: normal;
|
||||
line-height: 1.5;
|
||||
|
||||
-moz-tab-size: 4;
|
||||
-o-tab-size: 4;
|
||||
tab-size: 4;
|
||||
|
||||
-webkit-hyphens: none;
|
||||
-moz-hyphens: none;
|
||||
-ms-hyphens: none;
|
||||
hyphens: none;
|
||||
}
|
||||
|
||||
pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection,
|
||||
code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection {
|
||||
text-shadow: none;
|
||||
background: #b3d4fc;
|
||||
}
|
||||
|
||||
pre[class*="language-"]::selection, pre[class*="language-"] ::selection,
|
||||
code[class*="language-"]::selection, code[class*="language-"] ::selection {
|
||||
text-shadow: none;
|
||||
background: #b3d4fc;
|
||||
}
|
||||
|
||||
@media print {
|
||||
code[class*="language-"],
|
||||
pre[class*="language-"] {
|
||||
text-shadow: none;
|
||||
}
|
||||
}
|
||||
|
||||
/* Code blocks */
|
||||
pre[class*="language-"] {
|
||||
padding: 1em;
|
||||
margin: .5em 0;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
:not(pre) > code[class*="language-"],
|
||||
pre[class*="language-"] {
|
||||
background: #f5f2f0;
|
||||
}
|
||||
|
||||
/* Inline code */
|
||||
:not(pre) > code[class*="language-"] {
|
||||
padding: .1em;
|
||||
border-radius: .3em;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.token.comment,
|
||||
.token.prolog,
|
||||
.token.doctype,
|
||||
.token.cdata {
|
||||
color: slategray;
|
||||
}
|
||||
|
||||
.token.punctuation {
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.token.namespace {
|
||||
opacity: .7;
|
||||
}
|
||||
|
||||
.token.property,
|
||||
.token.tag,
|
||||
.token.boolean,
|
||||
.token.number,
|
||||
.token.constant,
|
||||
.token.symbol,
|
||||
.token.deleted {
|
||||
color: #905;
|
||||
}
|
||||
|
||||
.token.selector,
|
||||
.token.attr-name,
|
||||
.token.string,
|
||||
.token.char,
|
||||
.token.builtin,
|
||||
.token.inserted {
|
||||
color: #690;
|
||||
}
|
||||
|
||||
.token.operator,
|
||||
.token.entity,
|
||||
.token.url,
|
||||
.language-css .token.string,
|
||||
.style .token.string {
|
||||
color: #9a6e3a;
|
||||
background: hsla(0, 0%, 100%, .5);
|
||||
}
|
||||
|
||||
.token.atrule,
|
||||
.token.attr-value,
|
||||
.token.keyword {
|
||||
color: #07a;
|
||||
}
|
||||
|
||||
.token.function,
|
||||
.token.class-name {
|
||||
color: #DD4A68;
|
||||
}
|
||||
|
||||
.token.regex,
|
||||
.token.important,
|
||||
.token.variable {
|
||||
color: #e90;
|
||||
}
|
||||
|
||||
.token.important,
|
||||
.token.bold {
|
||||
font-weight: bold;
|
||||
}
|
||||
.token.italic {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.token.entity {
|
||||
cursor: help;
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -1,99 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import argparse
|
||||
import cStringIO
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import six.moves.urllib as urllib
|
||||
|
||||
treeherder_base = "https://treeherder.mozilla.org/"
|
||||
|
||||
"""Simple script for downloading structured logs from treeherder.
|
||||
|
||||
For the moment this is specialised to work with web-platform-tests
|
||||
logs; in due course it should move somewhere generic and get hooked
|
||||
up to mach or similar"""
|
||||
|
||||
# Interpretation of the "job" list from
|
||||
# https://github.com/mozilla/treeherder-service/blob/master/treeherder/webapp/api/utils.py#L18
|
||||
|
||||
def create_parser():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("branch", action="store",
|
||||
help="Branch on which jobs ran")
|
||||
parser.add_argument("commit",
|
||||
action="store",
|
||||
help="Commit hash for push")
|
||||
|
||||
return parser
|
||||
|
||||
def download(url, prefix, dest, force_suffix=True):
|
||||
if dest is None:
|
||||
dest = "."
|
||||
|
||||
if prefix and not force_suffix:
|
||||
name = os.path.join(dest, prefix + ".log")
|
||||
else:
|
||||
name = None
|
||||
counter = 0
|
||||
|
||||
while not name or os.path.exists(name):
|
||||
counter += 1
|
||||
sep = "" if not prefix else "-"
|
||||
name = os.path.join(dest, prefix + sep + str(counter) + ".log")
|
||||
|
||||
with open(name, "wb") as f:
|
||||
resp = requests.get(url, stream=True)
|
||||
for chunk in resp.iter_content(1024):
|
||||
f.write(chunk)
|
||||
|
||||
def get_blobber_url(branch, job):
|
||||
job_id = job["id"]
|
||||
resp = requests.get(urllib.parse.urljoin(treeherder_base,
|
||||
"/api/project/%s/artifact/?job_id=%i&name=Job%%20Info" % (branch,
|
||||
job_id)))
|
||||
job_data = resp.json()
|
||||
|
||||
if job_data:
|
||||
assert len(job_data) == 1
|
||||
job_data = job_data[0]
|
||||
try:
|
||||
details = job_data["blob"]["job_details"]
|
||||
for item in details:
|
||||
if item["value"] == "wpt_raw.log":
|
||||
return item["url"]
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def get_structured_logs(branch, commit, dest=None):
|
||||
resp = requests.get(urllib.parse.urljoin(treeherder_base, "/api/project/%s/resultset/?revision=%s" % (branch, commit)))
|
||||
|
||||
revision_data = resp.json()
|
||||
|
||||
result_set = revision_data["results"][0]["id"]
|
||||
|
||||
resp = requests.get(urllib.parse.urljoin(treeherder_base, "/api/project/%s/jobs/?result_set_id=%s&count=2000&exclusion_profile=false" % (branch, result_set)))
|
||||
|
||||
job_data = resp.json()
|
||||
|
||||
for result in job_data["results"]:
|
||||
job_type_name = result["job_type_name"]
|
||||
if job_type_name.startswith("W3C Web Platform"):
|
||||
url = get_blobber_url(branch, result)
|
||||
if url:
|
||||
prefix = result["platform"] # platform
|
||||
download(url, prefix, None)
|
||||
|
||||
def main():
|
||||
parser = create_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
get_structured_logs(args.branch, args.commit)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue