diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000000..4c92e6c2fa6 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,30 @@ +name: Docs upload + +on: + push: + branches: ["master"] + workflow_dispatch: + +jobs: + upload-docs: + name: Upload docs to GitHub Pages + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - name: Compile docs + run: python3 ./mach doc + env: + RUSTDOCFLAGS: --disable-minification + - name: Upload docs + run: | + cd target/doc + git init + git add . + git -c user.name="Workflow" -c user.email="" \ + commit -q -m "Rebuild Servo documentation" + git remote set-url origin https://git:${DOC_SERVO_ORG}@github.com/servo/doc.servo.org + git push --force origin FETCH_HEAD:gh-pages + env: + DOC_SERVO_ORG: ${{ secrets.DOC_SERVO_ORG }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..2748ce5534d --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,110 @@ +name: Nightly builds + +on: + schedule: + # Run at 5:30 am, daily. + - cron: '15 5 * * *' + workflow_dispatch: + +env: + RUST_BACKTRACE: 1 + SHELL: /bin/bash + +jobs: + build-linux-with-rust-nightly: + name: Build (Linux) + rustc nightly + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - name: Update rustc + run: echo nightly > rust-toolchain + - name: Bootstrap + run: | + python3 -m pip install --upgrade pip virtualenv + sudo apt update + python3 ./mach bootstrap + - name: Release build + run: python3 ./mach build --release + - name: Unit tests + run: python3 ./mach test-unit --release + + upload-linux: + name: Upload nightly (Linux) + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - name: Bootstrap + run: | + python3 -m pip install --upgrade pip virtualenv + sudo apt update + python3 ./mach bootstrap + - name: Release build + run: python3 ./mach build --release + - name: Package + run: python3 ./mach package --release + - name: Upload + run: python3 ./mach upload-nightly linux --secret-from-environment + env: + S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} + + upload-mac: + name: Upload nightly (macOS) + runs-on: macos-10.15 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - name: Bootstrap + run: | + python3 -m pip install --upgrade pip virtualenv + brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile + brew bundle install --verbose --no-upgrade --file=etc/taskcluster/macos/Brewfile-build + rm -rf /usr/local/etc/openssl + rm -rf /usr/local/etc/openssl@1.1 + brew install openssl@1.1 gnu-tar + - name: Release build + run: | + export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include" + export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib" + export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/" + export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH" + python3 ./mach build --release + - name: Package + run: python3 ./mach package --release + - name: Smoketest + run: ./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg + - name: Upload + run: python3 ./mach upload-nightly mac --secret-from-environment + env: + S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} + GITHUB_HOMEBREW_TOKEN: ${{ secrets.HOMEBREW_TOKEN }} + + upload-win: + name: Upload nightly (Windows) + runs-on: windows-2019 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - name: Copy to C drive + run: cp D:\a C:\ -Recurse + - name: Bootstrap + working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }" + run: | + python -m pip install --upgrade pip virtualenv + python mach fetch + - name: Release build + working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }" + run: python mach build --release --media-stack=dummy + - name: Package + working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }" + run: python mach package --release + - name: Upload + working-directory: "C:\\a\\${ REPOSITORY_NAME }\\${ REPOSITORY_NAME }" + run: python mach upload-nightly windows-msvc --secret-from-environment + env: + S3_UPLOAD_CREDENTIALS: ${{ secrets.S3_UPLOAD_CREDENTIALS }} diff --git a/etc/ci/workflow.mako b/etc/ci/workflow.mako index 8c2e34e720c..650df49c131 100644 --- a/etc/ci/workflow.mako +++ b/etc/ci/workflow.mako @@ -1,11 +1,9 @@ name: CI on: - # Triggers the workflow on push or pull request events but only for the master branch + # Triggers the workflow on push events but only for the master branch push: - branches: [ "master", "github-actions-dev", "auto", "try", "try-linux", "try-mac", "try-windows", "try-wpt", "linux-wpt-tests"] - pull_request: - branches: [ "master", "github-actions-dev" ] + branches: [ "auto", "try", "try-linux", "try-mac", "try-windows", "try-wpt"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -63,6 +61,12 @@ jobs: python3 ./mach build --release - name: Smoketest run: python3 ./mach smoketest + - name: Unit tests + run: python3 ./mach test-unit --release + - name: Test package + run: python3 ./mach package --release + - name: Package smoketest + run: ./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg - name: Package binary run: gtar -czf target.tar.gz target/release/servo target/release/*.dylib resources - name: Archive binary diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 5ee2a4ddfb1..3b9ff3a5fff 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -36,8 +36,6 @@ def tasks(task_for): windows_unit, windows_arm64, windows_uwp_x64, - macos_unit, - linux_release, ] by_branch_name = { "auto": all_tests, @@ -46,15 +44,14 @@ def tasks(task_for): # Add functions here as needed, in your push to that branch ], "master": [ - upload_docs, layout_2020_regressions_report, ], # The "try-*" keys match those in `servo_try_choosers` in Homu’s config: # https://github.com/servo/saltfs/blob/master/homu/map.jinja - "try-mac": [macos_unit], - "try-linux": [linux_tidy_unit, linux_docs_check, linux_release], + "try-mac": [], + "try-linux": [linux_tidy_unit, linux_docs_check], "try-windows": [windows_unit, windows_arm64, windows_uwp_x64], "try-arm": [windows_arm64], "try-wpt": [], @@ -89,10 +86,6 @@ def tasks(task_for): # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": daily_tasks_setup() - with_rust_nightly() - linux_nightly() - windows_nightly() - macos_nightly() update_wpt() uwp_nightly() @@ -119,7 +112,6 @@ linux_build_env = { # https://github.com/servo/servo/issues/24714#issuecomment-552951519 "SCCACHE_MAX_FRAME_LENGTH": str(100 * 1024 * 1024), # 100 MiB } -macos_build_env = {} windows_build_env = { "x86_64": { "GSTREAMER_1_0_ROOT_X86_64": "%HOMEDRIVE%%HOMEPATH%\\gst\\gstreamer\\1.0\\x86_64\\", @@ -198,58 +190,17 @@ def linux_tidy_unit(): def linux_docs_check(): return ( - linux_build_task("Docs + check") - .with_treeherder("Linux x64", "Doc+Check") - .with_script(""" - RUSTDOCFLAGS="--disable-minification" python3 ./mach doc - ( - cd target/doc - git init - git add . - git -c user.name="Taskcluster" -c user.email="" \ - commit -q -m "Rebuild Servo documentation" - git bundle create docs.bundle HEAD - ) - - """ + linux_build_task("Check") + .with_treeherder("Linux x64", "Check") + .with_script('RUSTDOCFLAGS="--disable-minification" python3 ./mach doc') # Because `rustdoc` needs metadata of dependency crates, # `cargo doc` does almost all of the work that `cargo check` does. # Therefore, when running them in this order the second command does very little # and should finish quickly. # The reverse order would not increase the total amount of work to do, # but would reduce the amount of parallelism available. - """ - python3 ./mach check - """) - .with_artifacts("/repo/target/doc/docs.bundle") - .find_or_create("docs." + CONFIG.tree_hash()) - ) - - -def upload_docs(): - docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.tree_hash()) - return ( - linux_task("Upload docs to GitHub Pages") - .with_treeherder("Linux x64", "DocUpload") - .with_dockerfile(dockerfile_path("base")) - .with_curl_artifact_script(docs_build_task_id, "docs.bundle") - .with_features("taskclusterProxy") - .with_scopes("secrets:get:project/servo/doc.servo.org") - .with_env(PY="""if 1: - import urllib.request, json, os - root_url = os.environ["TASKCLUSTER_PROXY_URL"] - url = root_url + "/api/secrets/v1/secret/project/servo/doc.servo.org" - token = json.load(urllib.request.urlopen(url))["secret"]["token"] - open("/root/.git-credentials", "w").write("https://git:%s@github.com/" % token) - """) - .with_script(""" - python3 -c "$PY" - git init --bare - git config credential.helper store - git fetch --quiet docs.bundle - git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages - """) - .create() + .with_script("python3 ./mach check") + .find_or_create("check." + CONFIG.tree_hash()) ) @@ -269,39 +220,6 @@ def layout_2020_regressions_report(): .create() ) -def macos_unit(): - return ( - macos_build_task("Dev build + unit tests") - .with_treeherder("macOS x64", "Unit") - .with_script(""" - python3 ./mach build --dev --verbose - python3 ./mach test-unit - python3 ./mach package --dev - ./etc/ci/macos_package_smoketest.sh target/debug/servo-tech-demo.dmg - ./etc/ci/lockfile_changed.sh - """) - .find_or_create("macos_unit." + CONFIG.tree_hash()) - ) - - -def with_rust_nightly(): - modified_build_env = dict(build_env) - flags = modified_build_env.pop("RUSTFLAGS").split(" ") - flags.remove("-Dwarnings") - if flags: # pragma: no cover - modified_build_env["RUSTFLAGS"] = " ".join(flags) - - return ( - linux_build_task("with Rust Nightly", build_env=modified_build_env) - .with_treeherder("Linux x64", "RustNightly") - .with_script(""" - echo "nightly" > rust-toolchain - python3 ./mach build --dev - python3 ./mach test-unit - """) - .create() - ) - appx_artifact = '/'.join([ 'repo', @@ -397,71 +315,6 @@ def windows_unit(cached=True, rdp=False): return task.create() -def windows_nightly(rdp=False): - return ( - windows_build_task("Nightly build and upload", rdp=rdp) - .with_treeherder("Windows x64", "Nightly") - .with_features("taskclusterProxy") - .with_scopes("secrets:get:project/servo/s3-upload-credentials") - .with_script("python mach fetch", - "python mach build --release", - "python mach package --release", - "python mach upload-nightly windows-msvc --secret-from-taskcluster") - .with_artifacts("repo/target/release/msi/Servo.exe", - "repo/target/release/msi/Servo.zip") - .find_or_create("build.windows_x64_nightly." + CONFIG.tree_hash()) - ) - - -def linux_nightly(): - return ( - linux_build_task("Nightly build and upload") - .with_treeherder("Linux x64", "Nightly") - .with_features("taskclusterProxy") - .with_scopes("secrets:get:project/servo/s3-upload-credentials") - # Not reusing the build made for WPT because it has debug assertions - .with_script( - "python3 ./mach build --release", - "python3 ./mach package --release", - "python3 ./mach upload-nightly linux --secret-from-taskcluster", - ) - .with_artifacts("/repo/target/release/servo-tech-demo.tar.gz") - .find_or_create("build.linux_x64_nightly" + CONFIG.tree_hash()) - ) - - -def linux_release(): - return ( - linux_build_task("Release build") - .with_treeherder("Linux x64", "Release") - .with_script( - "python3 ./mach build --release", - "python3 ./mach package --release", - ) - .find_or_create("build.linux_x64_release" + CONFIG.tree_hash()) - ) - - -def macos_nightly(): - return ( - macos_build_task("Nightly build and upload") - .with_treeherder("macOS x64", "Nightly") - .with_features("taskclusterProxy") - .with_scopes( - "secrets:get:project/servo/s3-upload-credentials", - "secrets:get:project/servo/github-homebrew-token", - ) - .with_script( - "python3 ./mach build --release", - "python3 ./mach package --release", - "./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg", - "python3 ./mach upload-nightly mac --secret-from-taskcluster", - ) - .with_artifacts("repo/target/release/servo-tech-demo.dmg") - .find_or_create("build.mac_x64_nightly." + CONFIG.tree_hash()) - ) - - def update_wpt(): build_task = linux_release_build_with_debug_assertions(layout_2020=False) return ( @@ -486,24 +339,6 @@ def update_wpt(): ) -def macos_release_build_with_debug_assertions(priority=None): - return ( - macos_build_task("Release build, with debug assertions") - .with_treeherder("macOS x64", "Release+A") - .with_priority(priority) - .with_script("\n".join([ - "python3 ./mach build --release --verbose --with-debug-assertions", - "./etc/ci/lockfile_changed.sh", - "tar -czf target.tar.gz" + - " target/release/servo" + - " target/release/*.dylib" + - " resources", - ])) - .with_artifacts("repo/target.tar.gz") - .find_or_create("build.macos_x64_release_w_assertions." + CONFIG.tree_hash()) - ) # pragma: no cover - - def linux_release_build_with_debug_assertions(layout_2020): if layout_2020: name_prefix = "Layout 2020 " # pragma: no cover @@ -568,14 +403,6 @@ def windows_task(name): ) -def macos_task(name): - return ( - decisionlib.MacOsGenericWorkerTask(name) - .with_worker_type(CONFIG.macos_worker_type) - .with_treeherder_required() - ) - - def linux_build_task(name, *, build_env=build_env): task = ( linux_task(name) @@ -648,58 +475,6 @@ def windows_build_task(name, package=True, arch="x86_64", rdp=False): return task -def with_homebrew(task, brewfiles): - for brewfile in brewfiles: - task.with_script("time brew bundle install --verbose --no-upgrade --file=" + brewfile) - return task - - -def macos_build_task(name): - build_task = ( - macos_task(name) - # Stray processes eating CPU can slow things down: - # https://github.com/servo/servo/issues/24735 - .with_max_run_time_minutes(60 * 2) - .with_env(**build_env, **unix_build_env, **macos_build_env) - .with_repo_bundle(alternate_object_dir="/var/cache/servo.git/objects") - .with_python3() - .with_rustup() - .with_index_and_artifacts_expire_in(build_artifacts_expire_in) - # Debugging for surprising generic-worker behaviour - .with_early_script("ls") - .with_script("ls target || true") - ) - return ( - with_homebrew(build_task, [ - "etc/taskcluster/macos/Brewfile", - "etc/taskcluster/macos/Brewfile-build", - ]) - .with_script(""" - export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include" - export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib" - export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/" - export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH" - """) - - .with_directory_mount( - "https://github.com/mozilla/sccache/releases/download/" - "0.2.7/sccache-0.2.7-x86_64-apple-darwin.tar.gz", - sha256="f86412abbbcce2d3f23e7d33305469198949f5cf807e6c3258c9e1885b4cb57f", - path="sccache", - ) - # Early script in order to run with the initial $PWD - .with_early_script(""" - export PATH="$PWD/sccache/sccache-0.2.7-x86_64-apple-darwin:$PATH" - """) - # sccache binaries requires OpenSSL 1.1 and are not compatible with 1.0. - # "Late" script in order to run after Homebrew is installed. - .with_script(""" - time brew install openssl@1.1 - export DYLD_LIBRARY_PATH="$HOME/homebrew/opt/openssl@1.1/lib" - """) - ) - - CONFIG.task_name_template = "Servo: %s" CONFIG.docker_images_expire_in = build_dependencies_artifacts_expire_in CONFIG.repacked_msi_files_expire_in = build_dependencies_artifacts_expire_in @@ -708,7 +483,6 @@ CONFIG.default_provisioner_id = "proj-servo" CONFIG.docker_image_build_worker_type = "docker" CONFIG.windows_worker_type = "win2016" -CONFIG.macos_worker_type = "macos" if __name__ == "__main__": # pragma: no cover main(task_for=os.environ["TASK_FOR"]) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index dbcf11536ce..22d9a2f22a0 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -28,7 +28,7 @@ import taskcluster # Public API __all__ = [ "CONFIG", "SHARED", "Task", "DockerWorkerTask", - "GenericWorkerTask", "WindowsGenericWorkerTask", "MacOsGenericWorkerTask", + "GenericWorkerTask", "WindowsGenericWorkerTask", "make_repo_bundle", ] @@ -673,45 +673,6 @@ class UnixTaskMixin(Task): )) -class MacOsGenericWorkerTask(UnixTaskMixin, GenericWorkerTask): - """ - Task definition for a `generic-worker` task running on macOS. - - Scripts are interpreted with `bash`. - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.scripts = [] - - with_script = chaining(append_to_attr, "scripts") - with_early_script = chaining(prepend_to_attr, "scripts") - - def build_command(self): - # generic-worker accepts multiple commands, but unlike on Windows - # the current directory and environment variables - # are not preserved across commands on macOS. - # So concatenate scripts and use a single `bash` command instead. - return [ - [ - "/bin/bash", "--login", "-x", "-e", "-o", "pipefail", "-c", - deindent("\n".join(self.scripts)) - ] - ] # pragma: no cover - - def with_python3(self): - return self.with_early_script(""" - python3 -m ensurepip --user - python3 -m pip install --user virtualenv - """) - - def with_rustup(self): - return self.with_early_script(""" - export PATH="$HOME/.cargo/bin:$PATH" - which rustup || curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none -y - rustup self update - """) - - class DockerWorkerTask(UnixTaskMixin, Task): """ Task definition for a worker type that runs the `generic-worker` implementation. diff --git a/python/servo/package_commands.py b/python/servo/package_commands.py index 02c29ba4d23..280fe782481 100644 --- a/python/servo/package_commands.py +++ b/python/servo/package_commands.py @@ -601,7 +601,10 @@ class PackageCommands(CommandBase): @CommandArgument('--secret-from-taskcluster', action='store_true', help='Retrieve the appropriate secrets from taskcluster.') - def upload_nightly(self, platform, secret_from_taskcluster): + @CommandArgument('--secret-from-environment', + action='store_true', + help='Retrieve the appropriate secrets from the environment.') + def upload_nightly(self, platform, secret_from_taskcluster, secret_from_environment): import boto3 def get_s3_secret(): @@ -611,6 +614,10 @@ class PackageCommands(CommandBase): secret = get_taskcluster_secret("s3-upload-credentials") aws_access_key = secret["aws_access_key_id"] aws_secret_access_key = secret["aws_secret_access_key"] + elif secret_from_environment: + secret = json.loads(os.environ['S3_UPLOAD_CREDENTIALS']) + aws_access_key = secret["aws_access_key_id"] + aws_secret_access_key = secret["aws_secret_access_key"] return (aws_access_key, aws_secret_access_key) def nightly_filename(package, timestamp):