From e2c3acf4d58c015470caaf3451934c4bc5a5fd8c Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 7 Dec 2018 19:41:50 +0100 Subject: [PATCH 1/8] Copy from etc/doc.servo.org in `./mach doc` --- etc/ci/upload_docs.sh | 4 ---- python/servo/post_build_commands.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/etc/ci/upload_docs.sh b/etc/ci/upload_docs.sh index 4aab401bdee..d8cc8983087 100755 --- a/etc/ci/upload_docs.sh +++ b/etc/ci/upload_docs.sh @@ -18,10 +18,6 @@ cd "$(dirname ${0})/../.." ./etc/ci/clean_build_artifacts.sh env CC=gcc-5 CXX=g++-5 ./mach doc -# etc/doc.servo.org/index.html overwrites $(mach rust-root)/doc/index.html -# Use recursive copy here to avoid `cp` returning an error code -# when it encounters directories. -cp -r etc/doc.servo.org/* target/doc/ python components/style/properties/build.py servo html regular diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index dc2731c0318..9af82449808 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -263,10 +263,15 @@ class PostBuildCommands(CommandBase): else: copy2(full_name, destination) - return self.call_rustup_run( + returncode = self.call_rustup_run( ["cargo", "doc", "--manifest-path", self.ports_servo_manifest()] + params, - env=self.build_env() - ) + env=self.build_env()) + if returncode: + return returncode + + static = path.join(self.context.topdir, "etc", "doc.servo.org") + for name in os.listdir(static): + copy2(path.join(static, name), path.join(docs, name)) @Command('browse-doc', description='Generate documentation and open it in a web browser', From 98c571013543c99bb3e64406758598f1acb5ace2 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 7 Dec 2018 19:46:35 +0100 Subject: [PATCH 2/8] Generate CSS properties docs in ./mach doc --- etc/ci/upload_docs.sh | 2 -- python/servo/post_build_commands.py | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/ci/upload_docs.sh b/etc/ci/upload_docs.sh index d8cc8983087..36ee8feecfe 100755 --- a/etc/ci/upload_docs.sh +++ b/etc/ci/upload_docs.sh @@ -19,8 +19,6 @@ cd "$(dirname ${0})/../.." env CC=gcc-5 CXX=g++-5 ./mach doc -python components/style/properties/build.py servo html regular - cd components/script cmake . cmake --build . --target supported-apis diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index 9af82449808..8143069ddfb 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -13,6 +13,7 @@ import json import os import os.path as path import subprocess +import sys from shutil import copytree, rmtree, copy2 from mach.decorators import ( @@ -273,6 +274,9 @@ class PostBuildCommands(CommandBase): for name in os.listdir(static): copy2(path.join(static, name), path.join(docs, name)) + build = path.join(self.context.topdir, "components", "style", "properties", "build.py") + subprocess.check_call([sys.executable, build, "servo", "html"]) + @Command('browse-doc', description='Generate documentation and open it in a web browser', category='post-build') From 00a1e9391cab92fa2c16c3b4e6244e7f52309353 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 7 Dec 2018 19:49:43 +0100 Subject: [PATCH 3/8] Generate apis.html in ./mach doc --- etc/ci/upload_docs.sh | 8 -------- python/servo/post_build_commands.py | 5 +++++ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/etc/ci/upload_docs.sh b/etc/ci/upload_docs.sh index 36ee8feecfe..4c49c67b1b5 100755 --- a/etc/ci/upload_docs.sh +++ b/etc/ci/upload_docs.sh @@ -19,14 +19,6 @@ cd "$(dirname ${0})/../.." env CC=gcc-5 CXX=g++-5 ./mach doc -cd components/script -cmake . -cmake --build . --target supported-apis -echo "Copying apis.html." -cp apis.html ../../target/doc/servo/ -echo "Copied apis.html." -cd ../.. - echo "Starting ghp-import." ghp-import -n target/doc echo "Finished ghp-import." diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index 8143069ddfb..4050f33f932 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -277,6 +277,11 @@ class PostBuildCommands(CommandBase): build = path.join(self.context.topdir, "components", "style", "properties", "build.py") subprocess.check_call([sys.executable, build, "servo", "html"]) + script = path.join(self.context.topdir, "components", "script") + subprocess.check_call(["cmake", "."], cwd=script) + subprocess.check_call(["cmake", "--build", ".", "--target", "supported-apis"], cwd=script) + copy2(path.join(script, "apis.html"), path.join(docs, "servo", "apis.html")) + @Command('browse-doc', description='Generate documentation and open it in a web browser', category='post-build') From e5ae3464cc354d96dbb31f8a7ecb4c8040a83fb0 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 7 Dec 2018 17:06:31 +0100 Subject: [PATCH 4/8] Shorter Treeherder "symbols" --- etc/taskcluster/decision_task.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 60434d4c06b..3454f0b3f40 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -71,8 +71,8 @@ windows_sparse_checkout = [ def linux_tidy_unit(): return ( - linux_build_task("Tidy + dev build + unit") - .with_treeherder("Linux x64") + linux_build_task("Tidy + dev build + unit tests") + .with_treeherder("Linux x64", "Tidy+Unit") .with_script(""" ./mach test-tidy --no-progress --all ./mach build --dev @@ -92,7 +92,7 @@ def linux_tidy_unit(): def macos_unit(): return ( macos_build_task("Dev build + unit tests") - .with_treeherder("macOS x64") + .with_treeherder("macOS x64", "Unit") .with_script(""" ./mach build --dev ./mach test-unit @@ -136,7 +136,7 @@ def android_arm32_dev(): def android_arm32_release(): return ( android_build_task("Release build") - .with_treeherder("Android ARMv7") + .with_treeherder("Android ARMv7", "Release") .with_script("./mach build --android --release") .with_artifacts( "/repo/target/android/armv7-linux-androideabi/release/servoapp.apk", @@ -149,7 +149,7 @@ def android_arm32_release(): def android_x86_release(): return ( android_build_task("Release build") - .with_treeherder("Android x86") + .with_treeherder("Android x86", "Release") .with_script("./mach build --target i686-linux-android --release") .with_artifacts( "/repo/target/android/i686-linux-android/release/servoapp.apk", @@ -185,7 +185,7 @@ def android_x86_wpt(): def windows_unit(): return ( windows_build_task("Dev build + unit tests") - .with_treeherder("Windows x64") + .with_treeherder("Windows x64", "Unit") .with_script( # Not necessary as this would be done at the start of `build`, # but this allows timing it separately. @@ -204,7 +204,7 @@ def windows_unit(): def windows_release(): return ( windows_build_task("Release build") - .with_treeherder("Windows x64") + .with_treeherder("Windows x64", "Release") .with_script("mach build --release", "mach package --release") .with_artifacts("repo/target/release/msi/Servo.exe", @@ -224,7 +224,7 @@ def linux_wpt(): def linux_release_build(): return ( linux_build_task("Release build") - .with_treeherder("Linux x64") + .with_treeherder("Linux x64", "Release") .with_script(""" ./mach build --release --with-debug-assertions -p servo ./etc/ci/lockfile_changed.sh @@ -241,7 +241,7 @@ def linux_release_build(): def wpt_chunk(release_build_task, total_chunks, this_chunk): task = ( linux_task("WPT chunk %s / %s" % (this_chunk, total_chunks)) - .with_treeherder("Linux x64", "WPT %s" % this_chunk) + .with_treeherder("Linux x64", "WPT-%s" % this_chunk) .with_dockerfile(dockerfile_path("run")) .with_repo() .with_curl_artifact_script(release_build_task, "target.tar.gz") From a35d16a09c074506770611e142ce8506aec70a70 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 7 Dec 2018 17:19:24 +0100 Subject: [PATCH 5/8] Taskcluster: build docs --- etc/taskcluster/decision_task.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 3454f0b3f40..19c3e435e25 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -13,7 +13,7 @@ def main(task_for, mock=False): if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]: CONFIG.treeherder_repo_name = "servo-" + CONFIG.git_ref.split("/")[-1] - linux_tidy_unit() + linux_tidy_unit_docs() android_arm32_dev() android_arm32_release() android_x86_release() @@ -69,10 +69,10 @@ windows_sparse_checkout = [ ] -def linux_tidy_unit(): +def linux_tidy_unit_docs(): return ( - linux_build_task("Tidy + dev build + unit tests") - .with_treeherder("Linux x64", "Tidy+Unit") + linux_build_task("Tidy + dev build + unit tests + docs") + .with_treeherder("Linux x64", "Tidy+Unit+Doc") .with_script(""" ./mach test-tidy --no-progress --all ./mach build --dev @@ -81,11 +81,22 @@ def linux_tidy_unit(): ./mach build --dev --libsimpleservo ./mach build --dev --no-default-features --features default-except-unstable ./mach test-tidy --no-progress --self-test + ./etc/memory_reports_over_time.py --test ./etc/taskcluster/mock.py ./etc/ci/lockfile_changed.sh ./etc/ci/check_no_panic.sh - """).create() + + ./mach doc + cd target/doc + git init + time git add . + git -c user.name="Taskcluster" -c user.email="" \ + commit -q -m "Rebuild Servo documentation" + git bundle create docs.bundle HEAD + """) + .with_artifacts("/repo/target/doc/docs.bundle") + .find_or_create("docs." + CONFIG.git_sha) ) @@ -98,7 +109,8 @@ def macos_unit(): ./mach test-unit ./mach package --dev ./etc/ci/lockfile_changed.sh - """).create() + """) + .create() ) From eb21ceaf98b9202de9e2dbb6b641b2164748694e Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 7 Dec 2018 19:13:16 +0100 Subject: [PATCH 6/8] Upload to doc.servo.org when merging to master --- .taskcluster.yml | 3 +-- etc/taskcluster/decision_task.py | 36 ++++++++++++++++++++++++++++++-- etc/taskcluster/decisionlib.py | 16 ++++++++------ etc/taskcluster/mock.py | 8 ++++++- 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index ea33b12034f..10559de409c 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -25,8 +25,7 @@ tasks: owner: &task_owner ${event.pusher.name}@users.noreply.github.com source: &task_source ${event.compare} scopes: - # Granted to role "repo:github.com/servo/servo:branch:*" - - "assume:project:servo:decision-task/trusted" + - "assume:repo:github.com/servo/servo:branch:${event.ref[11:]}" routes: # len("refs/heads/") == 11, so event.ref[11:] is the branch name - "tc-treeherder.v2.servo/servo-${event.ref[11:]}.${event.after}" diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 19c3e435e25..a2a284194e6 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -10,9 +10,11 @@ from decisionlib import * def main(task_for, mock=False): if task_for == "github-push": - if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]: - CONFIG.treeherder_repo_name = "servo-" + CONFIG.git_ref.split("/")[-1] + assert CONFIG.git_ref.startswith("refs/heads/") + branch = CONFIG.git_ref[len("refs/heads/"):] + CONFIG.treeherder_repository_names = ["servo-" + branch] + if branch in ["auto", "try", "try-taskcluster"]: linux_tidy_unit_docs() android_arm32_dev() android_arm32_release() @@ -28,6 +30,11 @@ def main(task_for, mock=False): linux_build_task("Indexed by task definition").find_or_create() android_x86_wpt() + if branch == "master": + # Also show these tasks in https://treeherder.mozilla.org/#/jobs?repo=servo-auto + CONFIG.treeherder_repository_names.append("servo-auto") + upload_docs() + # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": daily_tasks_setup() @@ -100,6 +107,31 @@ def linux_tidy_unit_docs(): ) +def upload_docs(): + docs_build_task_id = Task.find("docs." + CONFIG.git_sha) + return ( + linux_task("Upload docs to GitHub Pages") + .with_treeherder("Linux x64", "DocUpload") + .with_dockerfile(dockerfile_path("base")) + .with_curl_artifact_script(docs_build_task_id, "docs.bundle") + .with_features("taskclusterProxy") + .with_scopes("secrets:get:project/servo/doc.servo.org") + .with_env(PY="""if 1: + import urllib, json + url = "http://taskcluster/secrets/v1/secret/project/servo/doc.servo.org" + token = json.load(urllib.urlopen(url))["secret"]["token"] + open("/root/.git-credentials", "w").write("https://git:%s@github.com/" % token) + """) + .with_script(""" + python -c "$PY" + git init --bare + git config credential.helper store + git fetch --quiet docs.bundle + git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages + """) + .create() + ) + def macos_unit(): return ( macos_build_task("Dev build + unit tests") diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 77197b1bd0b..553113db326 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -43,7 +43,7 @@ class Config: self.docker_image_buil_worker_type = None self.docker_images_expire_in = "1 month" self.repacked_msi_files_expire_in = "1 month" - self.treeherder_repo_name = None + self.treeherder_repository_names = [] # Set by docker-worker: # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment @@ -156,9 +156,9 @@ class Task: "symbol": symbol, }) - if CONFIG.treeherder_repo_name: + for repo in CONFIG.treeherder_repository_names: assert CONFIG.git_sha - suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repo_name, CONFIG.git_sha) + suffix = ".v2._/%s.%s" % (repo, CONFIG.git_sha) self.with_routes( "tc-treeherder" + suffix, "tc-treeherder-staging" + suffix, @@ -223,6 +223,11 @@ class Task: print("Scheduled %s" % self.name) return task_id + @staticmethod + def find(index_path): + full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path) + return SHARED.index_service.findTask(full_index_path)["taskId"] + def find_or_create(self, index_path=None): """ Try to find a task in the Index and return its ID. @@ -240,18 +245,17 @@ class Task: worker_type = self.worker_type index_by = json.dumps([worker_type, self.build_worker_payload()]).encode("utf-8") index_path = "by-task-definition." + hashlib.sha256(index_by).hexdigest() - index_path = "%s.%s" % (CONFIG.index_prefix, index_path) task_id = SHARED.found_or_created_indexed_tasks.get(index_path) if task_id is not None: return task_id try: - task_id = SHARED.index_service.findTask(index_path)["taskId"] + task_id = Task.find(index_path) except taskcluster.TaskclusterRestFailure as e: if e.status_code != 404: # pragma: no cover raise - self.routes.append("index." + index_path) + self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) task_id = self.create() SHARED.found_or_created_indexed_tasks[index_path] = task_id diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py index d3e19d88080..43fd0cfe27f 100755 --- a/etc/taskcluster/mock.py +++ b/etc/taskcluster/mock.py @@ -31,7 +31,9 @@ class TaskclusterRestFailure(Exception): class Index: __init__ = insertTask = lambda *_, **__: None - def findTask(self, _): + def findTask(self, path): + if ".docs." in path: + return {"taskId": ""} raise TaskclusterRestFailure @@ -50,5 +52,9 @@ decision_task.main("github-push", mock=True) print("\n# Push with hot caches:") decision_task.main("github-push", mock=True) +print("\n# Push to master:") +decision_task.CONFIG.git_ref = "refs/heads/master" +decision_task.main("github-push", mock=True) + print("\n# Daily:") decision_task.main("daily", mock=True) From 8588e928e748a07c6d67ba3bde22ce3baf3a36c5 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 8 Dec 2018 02:31:19 +0100 Subject: [PATCH 7/8] Remove Buildbot script for docs upload --- etc/ci/upload_docs.sh | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100755 etc/ci/upload_docs.sh diff --git a/etc/ci/upload_docs.sh b/etc/ci/upload_docs.sh deleted file mode 100755 index 4c49c67b1b5..00000000000 --- a/etc/ci/upload_docs.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -# Helper script to upload docs to doc.servo.org. -# Requires ghp-import (from pip) -# GitHub API token must be passed in environment var TOKEN - -set -o errexit -set -o nounset -set -o pipefail - -cd "$(dirname ${0})/../.." - -# Clean up any traces of previous doc builds. -./etc/ci/clean_build_artifacts.sh - -env CC=gcc-5 CXX=g++-5 ./mach doc - -echo "Starting ghp-import." -ghp-import -n target/doc -echo "Finished ghp-import." -git push -qf \ - "https://${TOKEN}@github.com/servo/doc.servo.org.git" gh-pages \ - &>/dev/null -echo "Finished git push." - -# Clean up the traces of the current doc build. -./etc/ci/clean_build_artifacts.sh From e7b493df18b02376c36a9fc94a19c2470c657aa6 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Wed, 12 Dec 2018 12:25:27 +0100 Subject: [PATCH 8/8] Taskcluster: Add support for trychoosers --- etc/taskcluster/decision_task.py | 85 ++++++++++++++++++++++---------- etc/taskcluster/mock.py | 15 ++++-- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index a2a284194e6..04d08b197df 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -8,40 +8,75 @@ import os.path from decisionlib import * -def main(task_for, mock=False): +def main(task_for): if task_for == "github-push": + # FIXME https://github.com/servo/servo/issues/22325 implement these: + macos_wpt = magicleap_dev = linux_arm32_dev = linux_arm64_dev = \ + android_arm32_dev_from_macos = lambda: None + # FIXME still buggy: + linux_wpt = lambda: None # Shadows the existing top-level function + + all_tests = [ + linux_tidy_unit_docs, + windows_unit, + macos_unit, + magicleap_dev, + android_arm32_dev, + android_arm32_release, + android_x86_release, + linux_arm32_dev, + linux_arm64_dev, + linux_wpt, + macos_wpt, + ] + by_branch_name = { + "auto": all_tests, + "try": all_tests, + "try-taskcluster": [ + # Add functions here as needed, in your push to that branch + ], + "master": [ + # Also show these tasks in https://treeherder.mozilla.org/#/jobs?repo=servo-auto + lambda: CONFIG.treeherder_repository_names.append("servo-auto"), + upload_docs, + ], + + # The "try-*" keys match those in `servo_try_choosers` in Homu’s config: + # https://github.com/servo/saltfs/blob/master/homu/map.jinja + + "try-mac": [macos_unit], + "try-linux": [linux_tidy_unit_docs], + "try-windows": [windows_unit], + "try-magicleap": [magicleap_dev], + "try-arm": [linux_arm32_dev, linux_arm64_dev], + "try-wpt": [linux_wpt], + "try-wpt-mac": [macos_wpt], + "try-wpt-android": [android_x86_wpt], + "try-android": [ + android_arm32_dev, + android_arm32_dev_from_macos, + android_x86_wpt + ], + } assert CONFIG.git_ref.startswith("refs/heads/") branch = CONFIG.git_ref[len("refs/heads/"):] - CONFIG.treeherder_repository_names = ["servo-" + branch] - - if branch in ["auto", "try", "try-taskcluster"]: - linux_tidy_unit_docs() - android_arm32_dev() - android_arm32_release() - android_x86_release() - windows_unit() - macos_unit() - - # These are disabled in a "real" decision task, - # but should still run when testing this Python code. (See `mock.py`.) - if mock: - windows_release() - linux_wpt() - linux_build_task("Indexed by task definition").find_or_create() - android_x86_wpt() - - if branch == "master": - # Also show these tasks in https://treeherder.mozilla.org/#/jobs?repo=servo-auto - CONFIG.treeherder_repository_names.append("servo-auto") - upload_docs() + CONFIG.treeherder_repository_names.append("servo-" + branch) + for function in by_branch_name.get(branch, []): + function() # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": daily_tasks_setup() with_rust_nightly() - else: # pragma: no cover - raise ValueError("Unrecognized $TASK_FOR value: %r", task_for) + +# These are disabled in a "real" decision task, +# but should still run when testing this Python code. (See `mock.py`.) +def mocked_only(): + windows_release() + linux_wpt() + android_x86_wpt() + linux_build_task("Indexed by task definition").find_or_create() ping_on_daily_task_failure = "SimonSapin, nox, emilio" diff --git a/etc/taskcluster/mock.py b/etc/taskcluster/mock.py index 43fd0cfe27f..60720f73ddd 100755 --- a/etc/taskcluster/mock.py +++ b/etc/taskcluster/mock.py @@ -32,7 +32,7 @@ class Index: __init__ = insertTask = lambda *_, **__: None def findTask(self, path): - if ".docs." in path: + if decision_task.CONFIG.git_ref == "refs/heads/master": return {"taskId": ""} raise TaskclusterRestFailure @@ -47,14 +47,19 @@ os.environ["GIT_REF"] = "refs/heads/auto" import decision_task print("\n# Push:") -decision_task.main("github-push", mock=True) +decision_task.main("github-push") print("\n# Push with hot caches:") -decision_task.main("github-push", mock=True) +decision_task.main("github-push") + +print("\n# Mocked only:") +decision_task.mocked_only() print("\n# Push to master:") decision_task.CONFIG.git_ref = "refs/heads/master" -decision_task.main("github-push", mock=True) +decision_task.main("github-push") print("\n# Daily:") -decision_task.main("daily", mock=True) +decision_task.main("daily") + +print()