From c292b1aaf03e47a09983eff086d46ffc1673bec7 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 15 Nov 2019 15:41:20 +0100 Subject: [PATCH 01/10] Expect Homebrew to already be installed and in `$PATH` --- etc/taskcluster/decision_task.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 93a508e0c13..e2288f45366 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -876,17 +876,9 @@ def windows_build_task(name, package=True, arch="x86_64", worker_type=None): def with_homebrew(task, brewfiles): - task = task.with_script(""" - mkdir -p "$HOME/homebrew" - export PATH="$HOME/homebrew/bin:$PATH" - which brew || curl -L https://github.com/Homebrew/brew/tarball/master \ - | tar xz --strip 1 -C "$HOME/homebrew" - """) - for brewfile in brewfiles: - task = task.with_script(""" - time brew bundle install --no-upgrade --file={brewfile} - """.format(brewfile=brewfile)) - return task + for brewfile in brewfiles: + task.with_script("time brew bundle install --no-upgrade --file=" + brewfile) + return task def macos_build_task(name): From 9b17deb11e3657ae205d874d36a0a91ff71d8dd5 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 15 Nov 2019 18:20:28 +0100 Subject: [PATCH 02/10] Install zlib on macOS https://community-tc.services.mozilla.com/tasks/fcbLrz33RHeshgRZGvSAjg/runs/0/logs/https%3A%2F%2Fcommunity-tc.services.mozilla.com%2Fapi%2Fqueue%2Fv1%2Ftask%2FfcbLrz33RHeshgRZGvSAjg%2Fruns%2F0%2Fartifacts%2Fpublic%2Flogs%2Flive.log#L1359 Note that the above is on macOS 10.15. Maybe previous versions provided zlib system-wide? --- etc/taskcluster/decision_task.py | 1 + etc/taskcluster/macos/Brewfile | 1 + 2 files changed, 2 insertions(+) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index e2288f45366..72e62985356 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -909,6 +909,7 @@ def macos_build_task(name): export OPENSSL_INCLUDE_DIR="$(brew --prefix openssl)/include" export OPENSSL_LIB_DIR="$(brew --prefix openssl)/lib" export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/" + export PKG_CONFIG_PATH="$(brew --prefix zlib)/lib/pkgconfig/:$PKG_CONFIG_PATH" """) .with_directory_mount( diff --git a/etc/taskcluster/macos/Brewfile b/etc/taskcluster/macos/Brewfile index d13b3cfa678..ab177ef32d6 100644 --- a/etc/taskcluster/macos/Brewfile +++ b/etc/taskcluster/macos/Brewfile @@ -5,6 +5,7 @@ brew "openssl" brew "pkg-config" brew "llvm" brew "yasm" +brew "zlib" # For sccache brew "openssl@1.1" From d04a2745c666d722b4f9ee0eccff3e411ec2a841 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Fri, 15 Nov 2019 19:08:09 +0100 Subject: [PATCH 03/10] Make the macOS worker type configurable --- etc/taskcluster/decision_task.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 72e62985356..e688d43694e 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -89,7 +89,8 @@ def main(task_for): elif task_for == "try-windows-ami": CONFIG.git_sha_is_current_head() - windows_unit(os.environ["NEW_AMI_WORKER_TYPE"], cached=False) + CONFIG.windows_worker_type = os.environ["NEW_AMI_WORKER_TYPE"] + windows_unit(cached=False) # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": @@ -457,9 +458,9 @@ def uwp_nightly(): ) -def windows_unit(worker_type=None, cached=True): +def windows_unit(cached=True): task = ( - windows_build_task("Dev build + unit tests", worker_type=worker_type) + windows_build_task("Dev build + unit tests") .with_treeherder("Windows x64", "Unit") .with_script( # Not necessary as this would be done at the start of `build`, @@ -757,12 +758,10 @@ def linux_task(name): ) -def windows_task(name, worker_type=None): - if worker_type is None: - worker_type = "win2016" +def windows_task(name): return ( decisionlib.WindowsGenericWorkerTask(name) - .with_worker_type(worker_type) + .with_worker_type(CONFIG.windows_worker_type) .with_treeherder_required() ) @@ -771,7 +770,7 @@ def macos_task(name): return ( decisionlib.MacOsGenericWorkerTask(name) .with_provisioner_id("proj-servo") - .with_worker_type("macos") + .with_worker_type(CONFIG.macos_worker_type) .with_treeherder_required() ) @@ -814,7 +813,7 @@ def android_build_task(name): ) -def windows_build_task(name, package=True, arch="x86_64", worker_type=None): +def windows_build_task(name, package=True, arch="x86_64"): hashes = { "devel": { "x86_64": "c136cbfb0330041d52fe6ec4e3e468563176333c857f6ed71191ebc37fc9d605", @@ -828,7 +827,7 @@ def windows_build_task(name, package=True, arch="x86_64", worker_type=None): } version = "1.16.0" task = ( - windows_task(name, worker_type=worker_type) + windows_task(name) .with_max_run_time_minutes(90) .with_env( **build_env, @@ -992,6 +991,8 @@ CONFIG.index_prefix = "project.servo" CONFIG.default_provisioner_id = "proj-servo" CONFIG.docker_image_build_worker_type = "docker" +CONFIG.windows_worker_type = "win2016" +CONFIG.macos_worker_type = "macos" if __name__ == "__main__": # pragma: no cover main(task_for=os.environ["TASK_FOR"]) From 8b39fd3c7b846d9292b9ac1ce43d70fead97a496 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 16 Nov 2019 10:10:13 +0100 Subject: [PATCH 04/10] Verbose Homebrew --- etc/taskcluster/decision_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index e688d43694e..396236d0500 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -876,7 +876,7 @@ def windows_build_task(name, package=True, arch="x86_64"): def with_homebrew(task, brewfiles): for brewfile in brewfiles: - task.with_script("time brew bundle install --no-upgrade --file=" + brewfile) + task.with_script("time brew bundle install --verbose --no-upgrade --file=" + brewfile) return task From 50d5bac736fcff3882bcc8d43d7843f62a8d3c9a Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 16 Nov 2019 10:11:36 +0100 Subject: [PATCH 05/10] Reduce macOS build task timeout --- etc/taskcluster/decision_task.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 396236d0500..43f8efb35a6 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -883,8 +883,9 @@ def with_homebrew(task, brewfiles): def macos_build_task(name): build_task = ( macos_task(name) - # Allow long runtime in case the cache expired for all those Homebrew dependencies - .with_max_run_time_minutes(60 * 4) + # Stray processes eating CPU can slow things down: + # https://github.com/servo/servo/issues/24735 + .with_max_run_time_minutes(60 * 2) .with_env(**build_env, **unix_build_env, **macos_build_env) .with_repo() .with_python2() From 591d7ad6c18875ed2f5cc3dc4f54e0b2b53c50fc Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 16 Nov 2019 10:46:08 +0100 Subject: [PATCH 06/10] Make Cargo print timing information to stdout This adds lines such as ``` Completed cssparser v0.27.1 custom-build in 2.4s Completed cssparser v0.27.1 custom-build (run) in 0.6s Completed cssparser v0.27.1 in 1.1s ``` --- python/servo/build_commands.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index aa99557c7b7..3abd74d81c9 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -627,6 +627,9 @@ class MachCommands(CommandBase): '-C', env["GSTREAMER_DIR"], ]) + # https://internals.rust-lang.org/t/exploring-crate-graph-build-times-with-cargo-build-ztimings/10975 + opts += ["-Ztimings=info"] + if very_verbose: print (["Calling", "cargo", "build"] + opts) for key in env: From a74046a3b66814ea807f9ada9f6bdc35664765eb Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 16 Nov 2019 14:15:15 +0100 Subject: [PATCH 07/10] Print timing of git operations --- etc/taskcluster/decisionlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 5009e939825..d89e944b6b3 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -597,8 +597,8 @@ class UnixTaskMixin(Task): .with_early_script(""" git init repo cd repo - git fetch {depth} "$GIT_URL" "$GIT_REF" - git reset --hard "$GIT_SHA" + time git fetch {depth} "$GIT_URL" "$GIT_REF" + time git reset --hard "$GIT_SHA" """.format(depth="--depth 100" if shallow else "")) def with_curl_script(self, url, file_path): From 188dd9e7abce4956ec7ac053588a938cd10580ed Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 16 Nov 2019 14:15:52 +0100 Subject: [PATCH 08/10] Reduce git fetch depth. 100 still takes quite some time! ``` (git init servo.git && cd servo.git && time git fetch https://github.com/servo/servo master $ARGS && ); rm -rf servo.git Full: 724.75 MiB 57s home fiber in Paris 1m25s AWS us-west-2 Oregon 3m23s Macstadium DC1 Atlanta 4m22s Macstadium DC2 Las Vegas --depth 100: 129.00 MiB 1m21s home 1m18s AWS 1m30s Macstadium 1 1m24s Macstadium 2 --depth 50: 97.62 MiB 30s home 30s AWS 41s Macstadium 1 40s Macstadium 2 --depth 30: 92.47 MiB 17s home 18s AWS 27s Macstadium 1 26s Macstadium 2 --depth 10: 88.25 MiB 11s home 12s AWS 26s Macstadium 1 25s Macstadium 2 --depth 1: 87.53 MiB 10s home 10s AWS 22s Macstadium 1 28s Macstadium 2 ``` --- etc/taskcluster/decisionlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index d89e944b6b3..93ecae9ca4c 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -477,7 +477,7 @@ class WindowsGenericWorkerTask(GenericWorkerTask): git += """ git fetch {depth} %GIT_URL% %GIT_REF% git reset --hard %GIT_SHA% - """.format(depth="--depth 100" if shallow else "") + """.format(depth="--depth 30" if shallow else "") return self \ .with_git() \ .with_script(git) \ @@ -599,7 +599,7 @@ class UnixTaskMixin(Task): cd repo time git fetch {depth} "$GIT_URL" "$GIT_REF" time git reset --hard "$GIT_SHA" - """.format(depth="--depth 100" if shallow else "")) + """.format(depth="--depth 30" if shallow else "")) def with_curl_script(self, url, file_path): self.curl_scripts_count += 1 From 657fead0cea272cc955a6f8eb427db4952beab73 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 16 Nov 2019 14:26:37 +0100 Subject: [PATCH 09/10] =?UTF-8?q?Don=E2=80=99t=20fetch=20git=20tags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- etc/taskcluster/decisionlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 93ecae9ca4c..5df33970cd4 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -475,7 +475,7 @@ class WindowsGenericWorkerTask(GenericWorkerTask): type .git\\info\\sparse-checkout """ git += """ - git fetch {depth} %GIT_URL% %GIT_REF% + git fetch --no-tags {depth} %GIT_URL% %GIT_REF% git reset --hard %GIT_SHA% """.format(depth="--depth 30" if shallow else "") return self \ @@ -597,7 +597,7 @@ class UnixTaskMixin(Task): .with_early_script(""" git init repo cd repo - time git fetch {depth} "$GIT_URL" "$GIT_REF" + time git fetch --no-tags {depth} "$GIT_URL" "$GIT_REF" time git reset --hard "$GIT_SHA" """.format(depth="--depth 30" if shallow else "")) From 0dad48f54f6fdbb105175f7ad37e28ba4802e455 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sat, 16 Nov 2019 15:46:22 +0100 Subject: [PATCH 10/10] macOS CI: use a cache of github.com/servo/servo Relies on https://github.com/servo/taskcluster-config/pull/6 --- etc/taskcluster/decision_task.py | 19 +++++++++++++------ etc/taskcluster/decisionlib.py | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 43f8efb35a6..4adedd4505d 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -595,7 +595,7 @@ def update_wpt(): "etc/taskcluster/macos/Brewfile-gstreamer", ]) # Pushing the new changes to the git remote requires a full repo clone. - .with_repo(shallow=False) + .with_repo(shallow=False, alternate_object_dir="/var/cache/servo.git/objects") .with_curl_artifact_script(build_task, "target.tar.gz") .with_script(""" export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/" @@ -631,19 +631,26 @@ def macos_wpt(): def macos_run_task(name): task = macos_task(name).with_python2() return with_homebrew(task, ["etc/taskcluster/macos/Brewfile-gstreamer"]) - wpt_chunks("macOS x64", macos_run_task, build_task, repo_dir="repo", - total_chunks=6, processes=4) + wpt_chunks( + "macOS x64", + macos_run_task, + build_task, + repo_dir="repo", + repo_kwargs=dict(alternate_object_dir="/var/cache/servo.git/objects"), + total_chunks=6, + processes=4, + ) def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, - repo_dir, chunks="all"): + repo_dir, chunks="all", repo_kwargs={}): if chunks == "all": chunks = [n + 1 for n in range(total_chunks)] for this_chunk in chunks: task = ( make_chunk_task("WPT chunk %s / %s" % (this_chunk, total_chunks)) .with_treeherder(platform, "WPT-%s" % this_chunk) - .with_repo() + .with_repo(**repo_kwargs) .with_curl_artifact_script(build_task, "target.tar.gz") .with_script("tar -xzf target.tar.gz") .with_index_and_artifacts_expire_in(log_artifacts_expire_in) @@ -887,7 +894,7 @@ def macos_build_task(name): # https://github.com/servo/servo/issues/24735 .with_max_run_time_minutes(60 * 2) .with_env(**build_env, **unix_build_env, **macos_build_env) - .with_repo() + .with_repo(alternate_object_dir="/var/cache/servo.git/objects") .with_python2() .with_rustup() # Since macOS workers are long-lived and ~/.rustup kept across tasks: diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 5df33970cd4..488ae989f5c 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -579,7 +579,7 @@ class UnixTaskMixin(Task): super().__init__(*args, **kwargs) self.curl_scripts_count = 0 - def with_repo(self, shallow=True): + def with_repo(self, shallow=True, alternate_object_dir=None): """ Make a shallow clone the git repository at the start of the task. This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha` @@ -592,14 +592,25 @@ class UnixTaskMixin(Task): `git` and `ca-certificate` need to be installed in the Docker image. """ + # Not using $GIT_ALTERNATE_OBJECT_DIRECTORIES since it causes + # "object not found - no match for id" errors when Cargo fetches git dependencies + if alternate_object_dir: + self.with_env(ALTERNATE_OBJDIR=alternate_object_dir) return self \ .with_env(**git_env()) \ .with_early_script(""" git init repo cd repo + {alternate} time git fetch --no-tags {depth} "$GIT_URL" "$GIT_REF" time git reset --hard "$GIT_SHA" - """.format(depth="--depth 30" if shallow else "")) + """.format( + depth="--depth 30" if shallow else "", + alternate=( + """echo "$ALTERNATE_OBJDIR" > .git/objects/info/alternates""" + if alternate_object_dir else "" + ) + )) def with_curl_script(self, url, file_path): self.curl_scripts_count += 1