Auto merge of #22381 - servo:tc-rustdoc, r=nox

Build docs and upload to doc.servo.org on Taskcluster, add support for trychoosers

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22381)
<!-- Reviewable:end -->

.taskcluster.yml

@@ -25,8 +25,7 @@ tasks:
           owner: &task_owner ${event.pusher.name}@users.noreply.github.com
           source: &task_source ${event.compare}
         scopes:
-          # Granted to role "repo:github.com/servo/servo:branch:*"
+          - "assume:repo:github.com/servo/servo:branch:${event.ref[11:]}"
-          - "assume:project:servo:decision-task/trusted"
         routes:
           # len("refs/heads/") == 11, so event.ref[11:] is the branch name
           - "tc-treeherder.v2.servo/servo-${event.ref[11:]}.${event.after}"

etc/ci/upload_docs.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-# Helper script to upload docs to doc.servo.org.
-# Requires ghp-import (from pip)
-# GitHub API token must be passed in environment var TOKEN
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-cd "$(dirname ${0})/../.."
-
-# Clean up any traces of previous doc builds.
-./etc/ci/clean_build_artifacts.sh
-
-env CC=gcc-5 CXX=g++-5 ./mach doc
-# etc/doc.servo.org/index.html overwrites $(mach rust-root)/doc/index.html
-# Use recursive copy here to avoid `cp` returning an error code
-# when it encounters directories.
-cp -r etc/doc.servo.org/* target/doc/
-
-python components/style/properties/build.py servo html regular
-
-cd components/script
-cmake .
-cmake --build . --target supported-apis
-echo "Copying apis.html."
-cp apis.html ../../target/doc/servo/
-echo "Copied apis.html."
-cd ../..
-
-echo "Starting ghp-import."
-ghp-import -n target/doc
-echo "Finished ghp-import."
-git push -qf \
-    "https://${TOKEN}@github.com/servo/doc.servo.org.git" gh-pages \
-    &>/dev/null
-echo "Finished git push."
-
-# Clean up the traces of the current doc build.
-./etc/ci/clean_build_artifacts.sh

etc/taskcluster/decision_task.py

@@ -8,33 +8,75 @@ import os.path
 from decisionlib import *
 
 
-def main(task_for, mock=False):
+def main(task_for):
     if task_for == "github-push":
-        if CONFIG.git_ref in ["refs/heads/auto", "refs/heads/try", "refs/heads/try-taskcluster"]:
+        # FIXME https://github.com/servo/servo/issues/22325 implement these:
-            CONFIG.treeherder_repo_name = "servo-" + CONFIG.git_ref.split("/")[-1]
+        macos_wpt = magicleap_dev = linux_arm32_dev = linux_arm64_dev = \
+            android_arm32_dev_from_macos = lambda: None
+        # FIXME still buggy:
+        linux_wpt = lambda: None  # Shadows the existing top-level function
 
-            linux_tidy_unit()
+        all_tests = [
-            android_arm32_dev()
+            linux_tidy_unit_docs,
-            android_arm32_release()
+            windows_unit,
-            android_x86_release()
+            macos_unit,
-            windows_unit()
+            magicleap_dev,
-            macos_unit()
+            android_arm32_dev,
+            android_arm32_release,
+            android_x86_release,
+            linux_arm32_dev,
+            linux_arm64_dev,
+            linux_wpt,
+            macos_wpt,
+        ]
+        by_branch_name = {
+            "auto": all_tests,
+            "try": all_tests,
+            "try-taskcluster": [
+                # Add functions here as needed, in your push to that branch
+            ],
+            "master": [
+                # Also show these tasks in https://treeherder.mozilla.org/#/jobs?repo=servo-auto
+                lambda: CONFIG.treeherder_repository_names.append("servo-auto"),
+                upload_docs,
+            ],
 
-            # These are disabled in a "real" decision task,
+            # The "try-*" keys match those in `servo_try_choosers` in Homu’s config:
-            # but should still run when testing this Python code. (See `mock.py`.)
+            # https://github.com/servo/saltfs/blob/master/homu/map.jinja
-            if mock:
+
-                windows_release()
+            "try-mac": [macos_unit],
-                linux_wpt()
+            "try-linux": [linux_tidy_unit_docs],
-                linux_build_task("Indexed by task definition").find_or_create()
+            "try-windows": [windows_unit],
-                android_x86_wpt()
+            "try-magicleap": [magicleap_dev],
+            "try-arm": [linux_arm32_dev, linux_arm64_dev],
+            "try-wpt": [linux_wpt],
+            "try-wpt-mac": [macos_wpt],
+            "try-wpt-android": [android_x86_wpt],
+            "try-android": [
+                android_arm32_dev,
+                android_arm32_dev_from_macos,
+                android_x86_wpt
+            ],
+        }
+        assert CONFIG.git_ref.startswith("refs/heads/")
+        branch = CONFIG.git_ref[len("refs/heads/"):]
+        CONFIG.treeherder_repository_names.append("servo-" + branch)
+        for function in by_branch_name.get(branch, []):
+            function()
 
     # https://tools.taskcluster.net/hooks/project-servo/daily
     elif task_for == "daily":
         daily_tasks_setup()
         with_rust_nightly()
 
-    else:  # pragma: no cover
+
-        raise ValueError("Unrecognized $TASK_FOR value: %r", task_for)
+# These are disabled in a "real" decision task,
+# but should still run when testing this Python code. (See `mock.py`.)
+def mocked_only():
+    windows_release()
+    linux_wpt()
+    android_x86_wpt()
+    linux_build_task("Indexed by task definition").find_or_create()
 
 
 ping_on_daily_task_failure = "SimonSapin, nox, emilio"
@@ -69,10 +111,10 @@ windows_sparse_checkout = [
 ]
 
 
-def linux_tidy_unit():
+def linux_tidy_unit_docs():
     return (
-        linux_build_task("Tidy + dev build + unit")
+        linux_build_task("Tidy + dev build + unit tests + docs")
-        .with_treeherder("Linux x64")
+        .with_treeherder("Linux x64", "Tidy+Unit+Doc")
         .with_script("""
             ./mach test-tidy --no-progress --all
             ./mach build --dev
@@ -81,24 +123,61 @@ def linux_tidy_unit():
             ./mach build --dev --libsimpleservo
             ./mach build --dev --no-default-features --features default-except-unstable
             ./mach test-tidy --no-progress --self-test
+
             ./etc/memory_reports_over_time.py --test
             ./etc/taskcluster/mock.py
             ./etc/ci/lockfile_changed.sh
             ./etc/ci/check_no_panic.sh
-        """).create()
+
+            ./mach doc
+            cd target/doc
+            git init
+            time git add .
+            git -c user.name="Taskcluster" -c user.email="" \
+                commit -q -m "Rebuild Servo documentation"
+            git bundle create docs.bundle HEAD
+        """)
+        .with_artifacts("/repo/target/doc/docs.bundle")
+        .find_or_create("docs." + CONFIG.git_sha)
     )
 
 
+def upload_docs():
+    docs_build_task_id = Task.find("docs." + CONFIG.git_sha)
+    return (
+        linux_task("Upload docs to GitHub Pages")
+        .with_treeherder("Linux x64", "DocUpload")
+        .with_dockerfile(dockerfile_path("base"))
+        .with_curl_artifact_script(docs_build_task_id, "docs.bundle")
+        .with_features("taskclusterProxy")
+        .with_scopes("secrets:get:project/servo/doc.servo.org")
+        .with_env(PY="""if 1:
+            import urllib, json
+            url = "http://taskcluster/secrets/v1/secret/project/servo/doc.servo.org"
+            token = json.load(urllib.urlopen(url))["secret"]["token"]
+            open("/root/.git-credentials", "w").write("https://git:%s@github.com/" % token)
+        """)
+        .with_script("""
+            python -c "$PY"
+            git init --bare
+            git config credential.helper store
+            git fetch --quiet docs.bundle
+            git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages
+        """)
+        .create()
+    )
+
 def macos_unit():
     return (
         macos_build_task("Dev build + unit tests")
-        .with_treeherder("macOS x64")
+        .with_treeherder("macOS x64", "Unit")
         .with_script("""
             ./mach build --dev
             ./mach test-unit
             ./mach package --dev
             ./etc/ci/lockfile_changed.sh
-        """).create()
+        """)
+        .create()
     )
 
 
@@ -136,7 +215,7 @@ def android_arm32_dev():
 def android_arm32_release():
     return (
         android_build_task("Release build")
-        .with_treeherder("Android ARMv7")
+        .with_treeherder("Android ARMv7", "Release")
         .with_script("./mach build --android --release")
         .with_artifacts(
             "/repo/target/android/armv7-linux-androideabi/release/servoapp.apk",
@@ -149,7 +228,7 @@ def android_arm32_release():
 def android_x86_release():
     return (
         android_build_task("Release build")
-        .with_treeherder("Android x86")
+        .with_treeherder("Android x86", "Release")
         .with_script("./mach build --target i686-linux-android --release")
         .with_artifacts(
             "/repo/target/android/i686-linux-android/release/servoapp.apk",
@@ -185,7 +264,7 @@ def android_x86_wpt():
 def windows_unit():
     return (
         windows_build_task("Dev build + unit tests")
-        .with_treeherder("Windows x64")
+        .with_treeherder("Windows x64", "Unit")
         .with_script(
             # Not necessary as this would be done at the start of `build`,
             # but this allows timing it separately.
@@ -204,7 +283,7 @@ def windows_unit():
 def windows_release():
     return (
         windows_build_task("Release build")
-        .with_treeherder("Windows x64")
+        .with_treeherder("Windows x64", "Release")
         .with_script("mach build --release",
                      "mach package --release")
         .with_artifacts("repo/target/release/msi/Servo.exe",
@@ -224,7 +303,7 @@ def linux_wpt():
 def linux_release_build():
     return (
         linux_build_task("Release build")
-        .with_treeherder("Linux x64")
+        .with_treeherder("Linux x64", "Release")
         .with_script("""
             ./mach build --release --with-debug-assertions -p servo
             ./etc/ci/lockfile_changed.sh
@@ -241,7 +320,7 @@ def linux_release_build():
 def wpt_chunk(release_build_task, total_chunks, this_chunk):
     task = (
         linux_task("WPT chunk %s / %s" % (this_chunk, total_chunks))
-        .with_treeherder("Linux x64", "WPT %s" % this_chunk)
+        .with_treeherder("Linux x64", "WPT-%s" % this_chunk)
         .with_dockerfile(dockerfile_path("run"))
         .with_repo()
         .with_curl_artifact_script(release_build_task, "target.tar.gz")

etc/taskcluster/decisionlib.py

@@ -43,7 +43,7 @@ class Config:
         self.docker_image_buil_worker_type = None
         self.docker_images_expire_in = "1 month"
         self.repacked_msi_files_expire_in = "1 month"
-        self.treeherder_repo_name = None
+        self.treeherder_repository_names = []
 
         # Set by docker-worker:
         # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/environment
@@ -156,9 +156,9 @@ class Task:
             "symbol": symbol,
         })
 
-        if CONFIG.treeherder_repo_name:
+        for repo in CONFIG.treeherder_repository_names:
             assert CONFIG.git_sha
-            suffix = ".v2._/%s.%s" % (CONFIG.treeherder_repo_name, CONFIG.git_sha)
+            suffix = ".v2._/%s.%s" % (repo, CONFIG.git_sha)
             self.with_routes(
                 "tc-treeherder" + suffix,
                 "tc-treeherder-staging" + suffix,
@@ -223,6 +223,11 @@ class Task:
         print("Scheduled %s" % self.name)
         return task_id
 
+    @staticmethod
+    def find(index_path):
+        full_index_path = "%s.%s" % (CONFIG.index_prefix, index_path)
+        return SHARED.index_service.findTask(full_index_path)["taskId"]
+
     def find_or_create(self, index_path=None):
         """
         Try to find a task in the Index and return its ID.
@@ -240,18 +245,17 @@ class Task:
             worker_type = self.worker_type
             index_by = json.dumps([worker_type, self.build_worker_payload()]).encode("utf-8")
             index_path = "by-task-definition." + hashlib.sha256(index_by).hexdigest()
-        index_path = "%s.%s" % (CONFIG.index_prefix, index_path)
 
         task_id = SHARED.found_or_created_indexed_tasks.get(index_path)
         if task_id is not None:
             return task_id
 
         try:
-            task_id = SHARED.index_service.findTask(index_path)["taskId"]
+            task_id = Task.find(index_path)
         except taskcluster.TaskclusterRestFailure as e:
             if e.status_code != 404:  # pragma: no cover
                 raise
-            self.routes.append("index." + index_path)
+            self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path))
             task_id = self.create()
 
         SHARED.found_or_created_indexed_tasks[index_path] = task_id

etc/taskcluster/mock.py

@@ -31,7 +31,9 @@ class TaskclusterRestFailure(Exception):
 class Index:
     __init__ = insertTask = lambda *_, **__: None
 
-    def findTask(self, _):
+    def findTask(self, path):
+        if decision_task.CONFIG.git_ref == "refs/heads/master":
+            return {"taskId": ""}
         raise TaskclusterRestFailure
 
 
@@ -45,10 +47,19 @@ os.environ["GIT_REF"] = "refs/heads/auto"
 import decision_task
 
 print("\n# Push:")
-decision_task.main("github-push", mock=True)
+decision_task.main("github-push")
 
 print("\n# Push with hot caches:")
-decision_task.main("github-push", mock=True)
+decision_task.main("github-push")
+
+print("\n# Mocked only:")
+decision_task.mocked_only()
+
+print("\n# Push to master:")
+decision_task.CONFIG.git_ref = "refs/heads/master"
+decision_task.main("github-push")
 
 print("\n# Daily:")
-decision_task.main("daily", mock=True)
+decision_task.main("daily")
+
+print()

python/servo/post_build_commands.py

@@ -13,6 +13,7 @@ import json
 import os
 import os.path as path
 import subprocess
+import sys
 from shutil import copytree, rmtree, copy2
 
 from mach.decorators import (
@@ -263,10 +264,23 @@ class PostBuildCommands(CommandBase):
                     else:
                         copy2(full_name, destination)
 
-        return self.call_rustup_run(
+        returncode = self.call_rustup_run(
             ["cargo", "doc", "--manifest-path", self.ports_servo_manifest()] + params,
-            env=self.build_env()
+            env=self.build_env())
-        )
+        if returncode:
+            return returncode
+
+        static = path.join(self.context.topdir, "etc", "doc.servo.org")
+        for name in os.listdir(static):
+            copy2(path.join(static, name), path.join(docs, name))
+
+        build = path.join(self.context.topdir, "components", "style", "properties", "build.py")
+        subprocess.check_call([sys.executable, build, "servo", "html"])
+
+        script = path.join(self.context.topdir, "components", "script")
+        subprocess.check_call(["cmake", "."], cwd=script)
+        subprocess.check_call(["cmake", "--build", ".", "--target", "supported-apis"], cwd=script)
+        copy2(path.join(script, "apis.html"), path.join(docs, "servo", "apis.html"))
 
     @Command('browse-doc',
              description='Generate documentation and open it in a web browser',