Auto merge of #28424 - jdm:nightlies, r=jdm

Replace more taskcluster jobs

Fixes #28283. Fixes #28282. Fixes #28279.

.github/workflows/docs.yml

@@ -0,0 +1,30 @@
+name: Docs upload
+
+on:
+  push:
+    branches: ["master"]
+  workflow_dispatch:
+
+jobs:
+  upload-docs:
+    name: Upload docs to GitHub Pages
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Compile docs
+        run: python3 ./mach doc
+        env:
+          RUSTDOCFLAGS: --disable-minification
+      - name: Upload docs
+        run: |
+          cd target/doc
+          git init
+          git add .
+          git -c user.name="Workflow" -c user.email="" \
+              commit -q -m "Rebuild Servo documentation"
+          git remote set-url origin https://git:${DOC_SERVO_ORG}@github.com/servo/doc.servo.org
+          git push --force origin FETCH_HEAD:gh-pages
+        env:
+          DOC_SERVO_ORG: ${{ secrets.DOC_SERVO_ORG }}

.github/workflows/nightly.yml

@@ -0,0 +1,110 @@
+name: Nightly builds
+
+on:
+  schedule:
+    # Run at 5:30 am, daily.
+    - cron: '15 5 * * *'
+  workflow_dispatch:
+
+env:
+  RUST_BACKTRACE: 1
+  SHELL: /bin/bash
+
+jobs:
+  build-linux-with-rust-nightly:
+    name: Build (Linux) + rustc nightly
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Update rustc
+        run: echo nightly > rust-toolchain
+      - name: Bootstrap
+        run: |
+          python3 -m pip install --upgrade pip virtualenv
+          sudo apt update
+          python3 ./mach bootstrap
+      - name: Release build
+        run: python3 ./mach build --release
+      - name: Unit tests
+        run: python3 ./mach test-unit --release
+
+  upload-linux:
+    name: Upload nightly (Linux)
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Bootstrap
+        run: |
+          python3 -m pip install --upgrade pip virtualenv
+          sudo apt update
+          python3 ./mach bootstrap
+      - name: Release build
+        run: python3 ./mach build --release
+      - name: Package
+        run: python3 ./mach package --release
+      - name: Upload
+        run: python3 ./mach upload-nightly linux --secret-from-environment
+        env:
+          S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}
+
+  upload-mac:
+    name: Upload nightly (macOS)
+    runs-on: macos-10.15
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Bootstrap
+        run: |
+          python3 -m pip install --upgrade pip virtualenv
+          brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile
+          brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile-build
+          rm -rf /usr/local/etc/openssl
+          rm -rf /usr/local/etc/openssl@1.1
+          brew install openssl@1.1 gnu-tar
+      - name: Release build
+        run: |
+          export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include"
+          export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib"
+          export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/"
+          export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH"
+          python3 ./mach build --release
+      - name: Package
+        run: python3 ./mach package --release
+      - name: Smoketest
+        run: ./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg
+      - name: Upload
+        run: python3 ./mach upload-nightly mac --secret-from-environment
+        env:
+          S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}
+          GITHUB_HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_TOKEN }}
+
+  upload-win:
+    name: Upload nightly (Windows)
+    runs-on: windows-2019
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      - name: Copy to C drive
+        run: cp D:\a C:\ -Recurse
+      - name: Bootstrap
+        working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
+        run: |
+          python -m pip install --upgrade pip virtualenv
+          python mach fetch
+      - name: Release build
+        working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
+        run: python mach build --release --media-stack=dummy
+      - name: Package
+        working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
+        run: python mach package --release
+      - name: Upload
+        working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }"
+        run: python mach upload-nightly windows-msvc --secret-from-environment
+        env:
+          S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }}

etc/ci/workflow.mako

@@ -1,11 +1,9 @@
 name: CI
 
 on:
-  # Triggers the workflow on push or pull request events but only for the master branch
+  # Triggers the workflow on push events but only for the master branch
   push:
-    branches: [ "master", "github-actions-dev", "auto", "try", "try-linux", "try-mac", "try-windows", "try-wpt", "linux-wpt-tests"]
+    branches: [ "auto", "try", "try-linux", "try-mac", "try-windows", "try-wpt"]
-  pull_request:
-    branches: [ "master", "github-actions-dev" ]
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -63,6 +61,12 @@ jobs:
           python3 ./mach build --release
       - name: Smoketest
         run: python3 ./mach smoketest
+      - name: Unit tests
+        run: python3 ./mach test-unit --release
+      - name: Test package
+        run: python3 ./mach package --release
+      - name: Package smoketest
+        run: ./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg
       - name: Package binary
         run: gtar -czf target.tar.gz target/release/servo target/release/*.dylib resources
       - name: Archive binary

etc/taskcluster/decision_task.py

@@ -36,8 +36,6 @@ def tasks(task_for):
             windows_unit,
             windows_arm64,
             windows_uwp_x64,
-            macos_unit,
-            linux_release,
         ]
         by_branch_name = {
             "auto": all_tests,
@@ -46,15 +44,14 @@ def tasks(task_for):
                 # Add functions here as needed, in your push to that branch
             ],
             "master": [
-                upload_docs,
                 layout_2020_regressions_report,
             ],
 
             # The "try-*" keys match those in `servo_try_choosers` in Homu’s config:
             # https://github.com/servo/saltfs/blob/master/homu/map.jinja
 
-            "try-mac": [macos_unit],
+            "try-mac": [],
-            "try-linux": [linux_tidy_unit, linux_docs_check, linux_release],
+            "try-linux": [linux_tidy_unit, linux_docs_check],
             "try-windows": [windows_unit, windows_arm64, windows_uwp_x64],
             "try-arm": [windows_arm64],
             "try-wpt": [],
@@ -89,10 +86,6 @@ def tasks(task_for):
     # https://tools.taskcluster.net/hooks/project-servo/daily
     elif task_for == "daily":
         daily_tasks_setup()
-        with_rust_nightly()
-        linux_nightly()
-        windows_nightly()
-        macos_nightly()
         update_wpt()
         uwp_nightly()
 
@@ -119,7 +112,6 @@ linux_build_env = {
     # https://github.com/servo/servo/issues/24714#issuecomment-552951519
     "SCCACHE_MAX_FRAME_LENGTH": str(100 * 1024 * 1024),  # 100 MiB
 }
-macos_build_env = {}
 windows_build_env = {
     "x86_64": {
         "GSTREAMER_1_0_ROOT_X86_64": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\",
@@ -198,58 +190,17 @@ def linux_tidy_unit():
 
 def linux_docs_check():
     return (
-        linux_build_task("Docs + check")
+        linux_build_task("Check")
-        .with_treeherder("Linux x64", "Doc+Check")
+        .with_treeherder("Linux x64", "Check")
-        .with_script("""
+        .with_script('RUSTDOCFLAGS="--disable-minification" python3 ./mach doc')
-            RUSTDOCFLAGS="--disable-minification" python3 ./mach doc
-            (
-                cd target/doc
-                git init
-                git add .
-                git -c user.name="Taskcluster" -c user.email="" \
-                    commit -q -m "Rebuild Servo documentation"
-                git bundle create docs.bundle HEAD
-            )
-
-        """
         # Because `rustdoc` needs metadata of dependency crates,
         # `cargo doc` does almost all of the work that `cargo check` does.
         # Therefore, when running them in this order the second command does very little
         # and should finish quickly.
         # The reverse order would not increase the total amount of work to do,
         # but would reduce the amount of parallelism available.
-        """
+        .with_script("python3 ./mach check")
-            python3 ./mach check
+        .find_or_create("check." + CONFIG.tree_hash())
-        """)
-        .with_artifacts("/repo/target/doc/docs.bundle")
-        .find_or_create("docs." + CONFIG.tree_hash())
-    )
-
-
-def upload_docs():
-    docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.tree_hash())
-    return (
-        linux_task("Upload docs to GitHub Pages")
-        .with_treeherder("Linux x64", "DocUpload")
-        .with_dockerfile(dockerfile_path("base"))
-        .with_curl_artifact_script(docs_build_task_id, "docs.bundle")
-        .with_features("taskclusterProxy")
-        .with_scopes("secrets:get:project/servo/doc.servo.org")
-        .with_env(PY="""if 1:
-            import urllib.request, json, os
-            root_url = os.environ["TASKCLUSTER_PROXY_URL"]
-            url = root_url + "/api/secrets/v1/secret/project/servo/doc.servo.org"
-            token = json.load(urllib.request.urlopen(url))["secret"]["token"]
-            open("/root/.git-credentials", "w").write("https://git:%s@github.com/" % token)
-        """)
-        .with_script("""
-            python3 -c "$PY"
-            git init --bare
-            git config credential.helper store
-            git fetch --quiet docs.bundle
-            git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages
-        """)
-        .create()
     )
 
 
@@ -269,39 +220,6 @@ def layout_2020_regressions_report():
         .create()
     )
 
-def macos_unit():
-    return (
-        macos_build_task("Dev build + unit tests")
-        .with_treeherder("macOS x64", "Unit")
-        .with_script("""
-            python3 ./mach build --dev --verbose
-            python3 ./mach test-unit
-            python3 ./mach package --dev
-            ./etc/ci/macos_package_smoketest.sh target/debug/servo-tech-demo.dmg
-            ./etc/ci/lockfile_changed.sh
-        """)
-        .find_or_create("macos_unit." + CONFIG.tree_hash())
-    )
-
-
-def with_rust_nightly():
-    modified_build_env = dict(build_env)
-    flags = modified_build_env.pop("RUSTFLAGS").split(" ")
-    flags.remove("-Dwarnings")
-    if flags:  # pragma: no cover
-        modified_build_env["RUSTFLAGS"] = " ".join(flags)
-
-    return (
-        linux_build_task("with Rust Nightly", build_env=modified_build_env)
-        .with_treeherder("Linux x64", "RustNightly")
-        .with_script("""
-            echo "nightly" > rust-toolchain
-            python3 ./mach build --dev
-            python3 ./mach test-unit
-        """)
-        .create()
-    )
-
 
 appx_artifact = '/'.join([
     'repo',
@@ -397,71 +315,6 @@ def windows_unit(cached=True, rdp=False):
         return task.create()
 
 
-def windows_nightly(rdp=False):
-    return (
-        windows_build_task("Nightly build and upload", rdp=rdp)
-        .with_treeherder("Windows x64", "Nightly")
-        .with_features("taskclusterProxy")
-        .with_scopes("secrets:get:project/servo/s3-upload-credentials")
-        .with_script("python mach fetch",
-                     "python mach build --release",
-                     "python mach package --release",
-                     "python mach upload-nightly windows-msvc --secret-from-taskcluster")
-        .with_artifacts("repo/target/release/msi/Servo.exe",
-                        "repo/target/release/msi/Servo.zip")
-        .find_or_create("build.windows_x64_nightly." + CONFIG.tree_hash())
-    )
-
-
-def linux_nightly():
-    return (
-        linux_build_task("Nightly build and upload")
-        .with_treeherder("Linux x64", "Nightly")
-        .with_features("taskclusterProxy")
-        .with_scopes("secrets:get:project/servo/s3-upload-credentials")
-        # Not reusing the build made for WPT because it has debug assertions
-        .with_script(
-            "python3 ./mach build --release",
-            "python3 ./mach package --release",
-            "python3 ./mach upload-nightly linux --secret-from-taskcluster",
-        )
-        .with_artifacts("/repo/target/release/servo-tech-demo.tar.gz")
-        .find_or_create("build.linux_x64_nightly" + CONFIG.tree_hash())
-    )
-
-
-def linux_release():
-    return (
-        linux_build_task("Release build")
-        .with_treeherder("Linux x64", "Release")
-        .with_script(
-            "python3 ./mach build --release",
-            "python3 ./mach package --release",
-        )
-        .find_or_create("build.linux_x64_release" + CONFIG.tree_hash())
-    )
-
-
-def macos_nightly():
-    return (
-        macos_build_task("Nightly build and upload")
-        .with_treeherder("macOS x64", "Nightly")
-        .with_features("taskclusterProxy")
-        .with_scopes(
-            "secrets:get:project/servo/s3-upload-credentials",
-            "secrets:get:project/servo/github-homebrew-token",
-        )
-        .with_script(
-            "python3 ./mach build --release",
-            "python3 ./mach package --release",
-            "./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg",
-            "python3 ./mach upload-nightly mac --secret-from-taskcluster",
-        )
-        .with_artifacts("repo/target/release/servo-tech-demo.dmg")
-        .find_or_create("build.mac_x64_nightly." + CONFIG.tree_hash())
-    )
-
-
 def update_wpt():
     build_task = linux_release_build_with_debug_assertions(layout_2020=False)
     return (
@@ -486,24 +339,6 @@ def update_wpt():
     )
 
 
-def macos_release_build_with_debug_assertions(priority=None):
-    return (
-        macos_build_task("Release build, with debug assertions")
-        .with_treeherder("macOS x64", "Release+A")
-        .with_priority(priority)
-        .with_script("\n".join([
-            "python3 ./mach build --release --verbose --with-debug-assertions",
-            "./etc/ci/lockfile_changed.sh",
-            "tar -czf target.tar.gz" +
-            " target/release/servo" +
-            " target/release/*.dylib" +
-            " resources",
-        ]))
-        .with_artifacts("repo/target.tar.gz")
-        .find_or_create("build.macos_x64_release_w_assertions." + CONFIG.tree_hash())
-    ) # pragma: no cover
-
-
 def linux_release_build_with_debug_assertions(layout_2020):
     if layout_2020:
         name_prefix = "Layout 2020 "      # pragma: no cover
@@ -568,14 +403,6 @@ def windows_task(name):
     )
 
 
-def macos_task(name):
-    return (
-        decisionlib.MacOsGenericWorkerTask(name)
-        .with_worker_type(CONFIG.macos_worker_type)
-        .with_treeherder_required()
-    )
-
-
 def linux_build_task(name, *, build_env=build_env):
     task = (
         linux_task(name)
@@ -648,58 +475,6 @@ def windows_build_task(name, package=True, arch="x86_64", rdp=False):
     return task
 
 
-def with_homebrew(task, brewfiles):
-    for brewfile in brewfiles:
-        task.with_script("time brew bundle install --verbose --no-upgrade --file=" + brewfile)
-    return task
-
-
-def macos_build_task(name):
-    build_task = (
-        macos_task(name)
-        # Stray processes eating CPU can slow things down:
-        # https://github.com/servo/servo/issues/24735
-        .with_max_run_time_minutes(60 * 2)
-        .with_env(**build_env, **unix_build_env, **macos_build_env)
-        .with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects")
-        .with_python3()
-        .with_rustup()
-        .with_index_and_artifacts_expire_in(build_artifacts_expire_in)
-        # Debugging for surprising generic-worker behaviour
-        .with_early_script("ls")
-        .with_script("ls target || true")
-    )
-    return (
-        with_homebrew(build_task, [
-            "etc/taskcluster/macos/Brewfile",
-            "etc/taskcluster/macos/Brewfile-build",
-        ])
-        .with_script("""
-            export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include"
-            export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib"
-            export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/"
-            export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH"
-        """)
-
-        .with_directory_mount(
-            "https://github.com/mozilla/sccache/releases/download/"
-                "0.2.7/sccache-0.2.7-x86_64-apple-darwin.tar.gz",
-            sha256="f86412abbbcce2d3f23e7d33305469198949f5cf807e6c3258c9e1885b4cb57f",
-            path="sccache",
-        )
-        # Early script in order to run with the initial $PWD
-        .with_early_script("""
-            export PATH="$PWD/sccache/sccache-0.2.7-x86_64-apple-darwin:$PATH"
-        """)
-        # sccache binaries requires OpenSSL 1.1 and are not compatible with 1.0.
-        # "Late" script in order to run after Homebrew is installed.
-        .with_script("""
-            time brew install openssl@1.1
-            export DYLD_LIBRARY_PATH="$HOME/homebrew/opt/openssl@1.1/lib"
-        """)
-    )
-
-
 CONFIG.task_name_template = "Servo: %s"
 CONFIG.docker_images_expire_in = build_dependencies_artifacts_expire_in
 CONFIG.repacked_msi_files_expire_in = build_dependencies_artifacts_expire_in
@@ -708,7 +483,6 @@ CONFIG.default_provisioner_id = "proj-servo"
 CONFIG.docker_image_build_worker_type = "docker"
 
 CONFIG.windows_worker_type = "win2016"
-CONFIG.macos_worker_type = "macos"
 
 if __name__ == "__main__":  # pragma: no cover
     main(task_for=os.environ["TASK_FOR"])

etc/taskcluster/decisionlib.py

@@ -28,7 +28,7 @@ import taskcluster
 # Public API
 __all__ = [
     "CONFIG", "SHARED", "Task", "DockerWorkerTask",
-    "GenericWorkerTask", "WindowsGenericWorkerTask", "MacOsGenericWorkerTask",
+    "GenericWorkerTask", "WindowsGenericWorkerTask",
     "make_repo_bundle",
 ]
 
@@ -673,45 +673,6 @@ class UnixTaskMixin(Task):
         ))
 
 
-class MacOsGenericWorkerTask(UnixTaskMixin, GenericWorkerTask):
-    """
-    Task definition for a `generic-worker` task running on macOS.
-
-    Scripts are interpreted with `bash`.
-    """
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.scripts = []
-
-    with_script = chaining(append_to_attr, "scripts")
-    with_early_script = chaining(prepend_to_attr, "scripts")
-
-    def build_command(self):
-        # generic-worker accepts multiple commands, but unlike on Windows
-        # the current directory and environment variables
-        # are not preserved across commands on macOS.
-        # So concatenate scripts and use a single `bash` command instead.
-        return [
-            [
-                "/bin/bash", "--login", "-x", "-e", "-o", "pipefail", "-c",
-                deindent("\n".join(self.scripts))
-            ]
-        ] # pragma: no cover
-
-    def with_python3(self):
-        return self.with_early_script("""
-            python3 -m ensurepip --user
-            python3 -m pip install --user virtualenv
-        """)
-
-    def with_rustup(self):
-        return self.with_early_script("""
-            export PATH="$HOME/.cargo/bin:$PATH"
-            which rustup || curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y
-            rustup self update
-        """)
-
-
 class DockerWorkerTask(UnixTaskMixin, Task):
     """
     Task definition for a worker type that runs the `generic-worker` implementation.

python/servo/package_commands.py

@@ -601,7 +601,10 @@ class PackageCommands(CommandBase):
     @CommandArgument('--secret-from-taskcluster',
                      action='store_true',
                      help='Retrieve the appropriate secrets from taskcluster.')
-    def upload_nightly(self, platform, secret_from_taskcluster):
+    @CommandArgument('--secret-from-environment',
+                     action='store_true',
+                     help='Retrieve the appropriate secrets from the environment.')
+    def upload_nightly(self, platform, secret_from_taskcluster, secret_from_environment):
         import boto3
 
         def get_s3_secret():
@@ -611,6 +614,10 @@ class PackageCommands(CommandBase):
                 secret = get_taskcluster_secret("s3-upload-credentials")
                 aws_access_key = secret["aws_access_key_id"]
                 aws_secret_access_key = secret["aws_secret_access_key"]
+            elif secret_from_environment:
+                secret = json.loads(os.environ['S3_UPLOAD_CREDENTIALS'])
+                aws_access_key = secret["aws_access_key_id"]
+                aws_secret_access_key = secret["aws_secret_access_key"]
             return (aws_access_key, aws_secret_access_key)
 
         def nightly_filename(package, timestamp):