Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-06 16:45:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #22381 - servo:tc-rustdoc, r=nox

Browse source 
Build docs and upload to doc.servo.org on Taskcluster, add support for trychoosers

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22381)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2018-12-12 07:20:54 -05:00 • committed by GitHub
commit a86c23cdca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 152 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.taskcluster.yml
View file

@ -25,8 +25,7 @@ tasks:
owner: &task_owner ${event.pusher.name}@users.noreply.github.com
source: &task_source ${event.compare}
scopes:
# Granted to role "repo:github.com/servo/servo:branch:*"
- "assume:project:servo:decision-task/trusted"
- "assume:repo:github.com/servo/servo:branch:${event.ref[11:]}"
routes:
# len("refs/heads/") == 11, so event.ref[11:] is the branch name
- "tc-treeherder.v2.servo/servo-${event.ref[11:]}.${event.after}"

45
etc/ci/upload_docs.sh
View file

@ -1,45 +0,0 @@
#!/usr/bin/env bash
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# Helper script to upload docs to doc.servo.org.
# Requires ghp-import (from pip)
# GitHub API token must be passed in environment var TOKEN
set -o errexit
set -o nounset
set -o pipefail
cd "$(dirname ${0})/../.."
# Clean up any traces of previous doc builds.
./etc/ci/clean_build_artifacts.sh
env CC=gcc-5 CXX=g++-5 ./mach doc
# etc/doc.servo.org/index.html overwrites $(mach rust-root)/doc/index.html
# Use recursive copy here to avoid `cp` returning an error code
# when it encounters directories.
cp -r etc/doc.servo.org/* target/doc/
python components/style/properties/build.py servo html regular
cd components/script
cmake .
cmake --build . --target supported-apis
echo "Copying apis.html."
cp apis.html ../../target/doc/servo/
echo "Copied apis.html."
cd ../..
echo "Starting ghp-import."
ghp-import -n target/doc
echo "Finished ghp-import."
git push -qf \
"https://${TOKEN}@github.com/servo/doc.servo.org.git" gh-pages \
&>/dev/null
echo "Finished git push."
# Clean up the traces of the current doc build.
./etc/ci/clean_build_artifacts.sh

139
etc/taskcluster/decision_task.py
View file

@ -8,33 +8,75 @@ import os.path
from decisionlib import *
def main(task_for, mock=False):
def main(task_for):
if task_for == "github-push":
if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]:
CONFIG.treeherder_repo_name = "servo-" + CONFIG.git_ref.split("/")[-1]
# FIXME https://github.com/servo/servo/issues/22325 implement these:
macos_wpt = magicleap_dev = linux_arm32_dev = linux_arm64_dev = \
android_arm32_dev_from_macos = lambda: None
# FIXME still buggy:
linux_wpt = lambda: None # Shadows the existing top-level function
linux_tidy_unit()
android_arm32_dev()
android_arm32_release()
android_x86_release()
windows_unit()
macos_unit()
all_tests = [
linux_tidy_unit_docs,
windows_unit,
macos_unit,
magicleap_dev,
android_arm32_dev,
android_arm32_release,
android_x86_release,
linux_arm32_dev,
linux_arm64_dev,
linux_wpt,
macos_wpt,
]
by_branch_name = {
"auto": all_tests,
"try": all_tests,
"try-taskcluster": [
# Add functions here as needed, in your push to that branch
],
"master": [
# Also show these tasks in https://treeherder.mozilla.org/#/jobs?repo=servo-auto
lambda: CONFIG.treeherder_repository_names.append("servo-auto"),
upload_docs,
],
# These are disabled in a "real" decision task,
# but should still run when testing this Python code. (See `mock.py`.)
if mock:
windows_release()
linux_wpt()
linux_build_task("Indexed by task definition").find_or_create()
android_x86_wpt()
# The "try-*" keys match those in `servo_try_choosers` in Homu’s config:
# https://github.com/servo/saltfs/blob/master/homu/map.jinja
"try-mac": [macos_unit],
"try-linux": [linux_tidy_unit_docs],
"try-windows": [windows_unit],
"try-magicleap": [magicleap_dev],
"try-arm": [linux_arm32_dev, linux_arm64_dev],
"try-wpt": [linux_wpt],
"try-wpt-mac": [macos_wpt],
"try-wpt-android": [android_x86_wpt],
"try-android": [
android_arm32_dev,
android_arm32_dev_from_macos,
android_x86_wpt
],
}
assert CONFIG.git_ref.startswith("refs/heads/")
branch = CONFIG.git_ref[len("refs/heads/"):]
CONFIG.treeherder_repository_names.append("servo-" + branch)
for function in by_branch_name.get(branch, []):
function()
# https://tools.taskcluster.net/hooks/project-servo/daily
elif task_for == "daily":
daily_tasks_setup()
with_rust_nightly()
else: # pragma: no cover
raise ValueError("Unrecognized $TASK_FOR value: %r", task_for)
# These are disabled in a "real" decision task,
# but should still run when testing this Python code. (See `mock.py`.)
def mocked_only():
windows_release()
linux_wpt()
android_x86_wpt()
linux_build_task("Indexed by task definition").find_or_create()
ping_on_daily_task_failure = "SimonSapin, nox, emilio"
@ -69,10 +111,10 @@ windows_sparse_checkout = [
]
def linux_tidy_unit():
def linux_tidy_unit_docs():
return (
linux_build_task("Tidy + dev build + unit")
.with_treeherder("Linux x64")
linux_build_task("Tidy + dev build + unit tests + docs")
.with_treeherder("Linux x64", "Tidy+Unit+Doc")
.with_script("""
./mach test-tidy --no-progress --all
./mach build --dev
@ -81,24 +123,61 @@ def linux_tidy_unit():
./mach build --dev --libsimpleservo
./mach build --dev --no-default-features --features default-except-unstable
./mach test-tidy --no-progress --self-test
./etc/memory_reports_over_time.py --test
./etc/taskcluster/mock.py
./etc/ci/lockfile_changed.sh
./etc/ci/check_no_panic.sh
""").create()
./mach doc
cd target/doc
git init
time git add .
git -c user.name="Taskcluster" -c user.email="" \
commit -q -m "Rebuild Servo documentation"
git bundle create docs.bundle HEAD
""")
.with_artifacts("/repo/target/doc/docs.bundle")
.find_or_create("docs." + CONFIG.git_sha)
)
def upload_docs():
docs_build_task_id = Task.find("docs." + CONFIG.git_sha)
return (
linux_task("Upload docs to GitHub Pages")
.with_treeherder("Linux x64", "DocUpload")
.with_dockerfile(dockerfile_path("base"))
.with_curl_artifact_script(docs_build_task_id, "docs.bundle")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/doc.servo.org")
.with_env(PY="""if 1:
import urllib, json
url = "http://taskcluster/secrets/v1/secret/project/servo/doc.servo.org"
token = json.load(urllib.urlopen(url))["secret"]["token"]
open("/root/.git-credentials", "w").write("https://git:%s@github.com/" % token)
""")
.with_script("""
python -c "$PY"
git init --bare
git config credential.helper store
git fetch --quiet docs.bundle
git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages
""")
.create()
)
def macos_unit():
return (
macos_build_task("Dev build + unit tests")
.with_treeherder("macOS x64")
.with_treeherder("macOS x64", "Unit")
.with_script("""
./mach build --dev
./mach test-unit
./mach package --dev
./etc/ci/lockfile_changed.sh
""").create()
""")
.create()
)
@ -136,7 +215,7 @@ def android_arm32_dev():
def android_arm32_release():
return (
android_build_task("Release build")
.with_treeherder("Android ARMv7")
.with_treeherder("Android ARMv7", "Release")
.with_script("./mach build --android --release")
.with_artifacts(
"/repo/target/android/armv7-linux-androideabi/release/servoapp.apk",
@ -149,7 +228,7 @@ def android_arm32_release():
def android_x86_release():
return (
android_build_task("Release build")
.with_treeherder("Android x86")
.with_treeherder("Android x86", "Release")
.with_script("./mach build --target i686-linux-android --release")
.with_artifacts(
"/repo/target/android/i686-linux-android/release/servoapp.apk",
@ -185,7 +264,7 @@ def android_x86_wpt():
def windows_unit():
return (
windows_build_task("Dev build + unit tests")
.with_treeherder("Windows x64")
.with_treeherder("Windows x64", "Unit")
.with_script(
# Not necessary as this would be done at the start of `build`,
# but this allows timing it separately.
@ -204,7 +283,7 @@ def windows_unit():
def windows_release():
return (
windows_build_task("Release build")
.with_treeherder("Windows x64")
.with_treeherder("Windows x64", "Release")
.with_script("mach build --release",
"mach package --release")
.with_artifacts("repo/target/release/msi/Servo.exe",
@ -224,7 +303,7 @@ def linux_wpt():
def linux_release_build():
return (
linux_build_task("Release build")
.with_treeherder("Linux x64")
.with_treeherder("Linux x64", "Release")
.with_script("""
./mach build --release --with-debug-assertions -p servo
./etc/ci/lockfile_changed.sh
@ -241,7 +320,7 @@ def linux_release_build():
def wpt_chunk(release_build_task, total_chunks, this_chunk):
task = (
linux_task("WPT chunk %s / %s" % (this_chunk, total_chunks))
.with_treeherder("Linux x64", "WPT %s" % this_chunk)
.with_treeherder("Linux x64", "WPT-%s" % this_chunk)
.with_dockerfile(dockerfile_path("run"))
.with_repo()
.with_curl_artifact_script(release_build_task, "target.tar.gz")

16
etc/taskcluster/decisionlib.py
View file

@ -43,7 +43,7 @@ class Config:
self.docker_image_buil_worker_type = None
self.docker_images_expire_in = "1 month"
self.repacked_msi_files_expire_in = "1 month"
self.treeherder_repo_name = None
self.treeherder_repository_names = []
# Set by docker-worker:
# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
@ -156,9 +156,9 @@ class Task:
"symbol": symbol,
})
if CONFIG.treeherder_repo_name:
for repo in CONFIG.treeherder_repository_names:
assert CONFIG.git_sha
suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repo_name, CONFIG.git_sha)
suffix = ".v2._/%s.%s" % (repo, CONFIG.git_sha)
self.with_routes(
"tc-treeherder" + suffix,
"tc-treeherder-staging" + suffix,
@ -223,6 +223,11 @@ class Task:
print("Scheduled %s" % self.name)
return task_id
@staticmethod
def find(index_path):
full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path)
return SHARED.index_service.findTask(full_index_path)["taskId"]
def find_or_create(self, index_path=None):
"""
Try to find a task in the Index and return its ID.
@ -240,18 +245,17 @@ class Task:
worker_type = self.worker_type
index_by = json.dumps([worker_type, self.build_worker_payload()]).encode("utf-8")
index_path = "by-task-definition." + hashlib.sha256(index_by).hexdigest()
index_path = "%s.%s" % (CONFIG.index_prefix, index_path)
task_id = SHARED.found_or_created_indexed_tasks.get(index_path)
if task_id is not None:
return task_id
try:
task_id = SHARED.index_service.findTask(index_path)["taskId"]
task_id = Task.find(index_path)
except taskcluster.TaskclusterRestFailure as e:
if e.status_code != 404: # pragma: no cover
raise
self.routes.append("index." + index_path)
self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path))
task_id = self.create()
SHARED.found_or_created_indexed_tasks[index_path] = task_id

19
etc/taskcluster/mock.py
View file

@ -31,7 +31,9 @@ class TaskclusterRestFailure(Exception):
class Index:
__init__ = insertTask = lambda *_, **__: None
def findTask(self, _):
def findTask(self, path):
if decision_task.CONFIG.git_ref == "refs/heads/master":
return {"taskId": ""}
raise TaskclusterRestFailure
@ -45,10 +47,19 @@ os.environ["GIT_REF"] = "refs/heads/auto"
import decision_task
print("\n# Push:")
decision_task.main("github-push", mock=True)
decision_task.main("github-push")
print("\n# Push with hot caches:")
decision_task.main("github-push", mock=True)
decision_task.main("github-push")
print("\n# Mocked only:")
decision_task.mocked_only()
print("\n# Push to master:")
decision_task.CONFIG.git_ref = "refs/heads/master"
decision_task.main("github-push")
print("\n# Daily:")
decision_task.main("daily", mock=True)
decision_task.main("daily")
print()

20
python/servo/post_build_commands.py
View file

@ -13,6 +13,7 @@ import json
import os
import os.path as path
import subprocess
import sys
from shutil import copytree, rmtree, copy2
from mach.decorators import (
@ -263,10 +264,23 @@ class PostBuildCommands(CommandBase):
else:
copy2(full_name, destination)
return self.call_rustup_run(
returncode = self.call_rustup_run(
["cargo", "doc", "--manifest-path", self.ports_servo_manifest()] + params,
env=self.build_env()
)
env=self.build_env())
if returncode:
return returncode
static = path.join(self.context.topdir, "etc", "doc.servo.org")
for name in os.listdir(static):
copy2(path.join(static, name), path.join(docs, name))
build = path.join(self.context.topdir, "components", "style", "properties", "build.py")
subprocess.check_call([sys.executable, build, "servo", "html"])
script = path.join(self.context.topdir, "components", "script")
subprocess.check_call(["cmake", "."], cwd=script)
subprocess.check_call(["cmake", "--build", ".", "--target", "supported-apis"], cwd=script)
copy2(path.join(script, "apis.html"), path.join(docs, "servo", "apis.html"))
@Command('browse-doc',
description='Generate documentation and open it in a web browser',