From 0edf865debcd6d7b86f3e13238eb5daf997975fb Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Mon, 13 May 2019 17:50:10 -0400 Subject: [PATCH 1/3] Index taskcluster tasks by merge parents if available. --- .taskcluster.yml | 4 +++- etc/taskcluster/decision_task.py | 40 +++++++++++++++++--------------- etc/taskcluster/decisionlib.py | 12 ++++++++++ 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 360af84ad3a..87989caba1e 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -30,10 +30,12 @@ tasks: - '--login' - '-e' - '-c' + # A depth of 25 is used to work around `git show` ignoring + # any provided format with a depth of 1. - >- git init repo && cd repo && - git fetch --depth 1 "$GIT_URL" "$GIT_REF" && + git fetch --depth 25 "$GIT_URL" "$GIT_REF" && git reset --hard "$GIT_SHA" && python3 etc/taskcluster/decision_task.py in: diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index c3424fa01d8..82ec88a2572 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -17,6 +17,8 @@ def main(task_for): ) if task_for == "github-push": + CONFIG.init_task_id() + # FIXME https://github.com/servo/servo/issues/22325 implement these: linux_arm32_dev = linux_arm64_dev = lambda: None @@ -193,12 +195,12 @@ def linux_tidy_unit_docs(): git bundle create docs.bundle HEAD """) .with_artifacts("/repo/target/doc/docs.bundle") - .find_or_create("docs." + CONFIG.git_sha) + .find_or_create("docs." + CONFIG.task_id) ) def upload_docs(): - docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.git_sha) + docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.task_id) return ( linux_task("Upload docs to GitHub Pages") .with_treeherder("Linux x64", "DocUpload") @@ -233,7 +235,7 @@ def macos_unit(): ./mach package --dev ./etc/ci/lockfile_changed.sh """) - .find_or_create("macos_unit." + CONFIG.git_sha) + .find_or_create("macos_unit." + CONFIG.task_id) ) @@ -266,7 +268,7 @@ def android_arm32_dev_from_macos(): ./mach bootstrap-android --accept-all-licences --build ./mach build --android --dev --verbose """) - .find_or_create("android_arm32_dev.macos." + CONFIG.git_sha) + .find_or_create("android_arm32_dev.macos." + CONFIG.task_id) ) @@ -279,7 +281,7 @@ def android_arm32_dev(): ./etc/ci/lockfile_changed.sh python ./etc/ci/check_dynamic_symbols.py """) - .find_or_create("android_arm32_dev." + CONFIG.git_sha) + .find_or_create("android_arm32_dev." + CONFIG.task_id) ) @@ -303,7 +305,7 @@ def android_nightly(): "/repo/target/android/i686-linux-android/release/servoapp.apk", "/repo/target/android/i686-linux-android/release/servoview.aar", ) - .find_or_create("build.android_nightlies." + CONFIG.git_sha) + .find_or_create("build.android_nightlies." + CONFIG.task_id) ) @@ -316,7 +318,7 @@ def android_arm32_release(): "/repo/target/android/armv7-linux-androideabi/release/servoapp.apk", "/repo/target/android/armv7-linux-androideabi/release/servoview.aar", ) - .find_or_create("build.android_armv7_release." + CONFIG.git_sha) + .find_or_create("build.android_armv7_release." + CONFIG.task_id) ) @@ -329,7 +331,7 @@ def android_x86_release(): "/repo/target/android/i686-linux-android/release/servoapp.apk", "/repo/target/android/i686-linux-android/release/servoview.aar", ) - .find_or_create("build.android_x86_release." + CONFIG.git_sha) + .find_or_create("build.android_x86_release." + CONFIG.task_id) ) @@ -352,7 +354,7 @@ def android_x86_wpt(): /_mozilla/mozilla/DOMParser.html \ /_mozilla/mozilla/webgl/context_creation_error.html """) - .find_or_create("android_x86_release." + CONFIG.git_sha) + .find_or_create("android_x86_release." + CONFIG.task_id) ) @@ -366,7 +368,7 @@ def windows_x86(): .with_script( "python mach build --dev --target i686-pc-windows-msvc", ) - .find_or_create("build.windows_x86_dev." + CONFIG.git_sha) + .find_or_create("build.windows_x86_dev." + CONFIG.task_id) ) @@ -385,7 +387,7 @@ def windows_unit(): ) .with_artifacts("repo/target/debug/msi/Servo.exe", "repo/target/debug/msi/Servo.zip") - .find_or_create("build.windows_x64_dev." + CONFIG.git_sha) + .find_or_create("build.windows_x64_dev." + CONFIG.task_id) ) @@ -397,7 +399,7 @@ def windows_release(): "mach package --release") .with_artifacts("repo/target/release/msi/Servo.exe", "repo/target/release/msi/Servo.zip") - .find_or_create("build.windows_x64_release." + CONFIG.git_sha) + .find_or_create("build.windows_x64_release." + CONFIG.task_id) ) @@ -413,7 +415,7 @@ def windows_nightly(): "mach upload-nightly windows-msvc --secret-from-taskcluster") .with_artifacts("repo/target/release/msi/Servo.exe", "repo/target/release/msi/Servo.zip") - .find_or_create("build.windows_x64_nightly." + CONFIG.git_sha) + .find_or_create("build.windows_x64_nightly." + CONFIG.task_id) ) @@ -430,7 +432,7 @@ def linux_nightly(): "./mach upload-nightly linux --secret-from-taskcluster", ) .with_artifacts("/repo/target/release/servo-tech-demo.tar.gz") - .find_or_create("build.linux_x64_nightly" + CONFIG.git_sha) + .find_or_create("build.linux_x64_nightly" + CONFIG.task_id) ) @@ -447,7 +449,7 @@ def linux_wpt(): target/release/build/osmesa-src-*/out/lib/gallium """) .with_artifacts("/target.tar.gz") - .find_or_create("build.linux_x64_release~assertions" + CONFIG.git_sha) + .find_or_create("build.linux_x64_release~assertions" + CONFIG.task_id) ) def linux_run_task(name): return linux_task(name).with_dockerfile(dockerfile_path("run")) @@ -470,7 +472,7 @@ def macos_nightly(): "./mach upload-nightly mac --secret-from-taskcluster", ) .with_artifacts("repo/target/release/servo-tech-demo.dmg") - .find_or_create("build.mac_x64_nightly." + CONFIG.git_sha) + .find_or_create("build.mac_x64_nightly." + CONFIG.task_id) ) @@ -500,7 +502,7 @@ def update_wpt(): ./etc/ci/update-wpt-checkout open-pr ./etc/ci/update-wpt-checkout cleanup """) - .find_or_create("wpt_update." + CONFIG.git_sha) + .find_or_create("wpt_update." + CONFIG.task_id) ) @@ -518,7 +520,7 @@ def macos_release_build(): target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs """) .with_artifacts("repo/target.tar.gz") - .find_or_create("build.macos_x64_release." + CONFIG.git_sha) + .find_or_create("build.macos_x64_release." + CONFIG.task_id) ) @@ -606,7 +608,7 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, if word.endswith(".log") ]) platform_id = platform.replace(" ", "_").lower() - task.find_or_create("%s_wpt_%s.%s" % (platform_id, this_chunk, CONFIG.git_sha)) + task.find_or_create("%s_wpt_%s.%s" % (platform_id, this_chunk, CONFIG.task_id)) def daily_tasks_setup(): diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 0a6d9b99e1c..cee90ba469b 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -57,9 +57,21 @@ class Config: self.git_ref = os.environ.get("GIT_REF") self.git_sha = os.environ.get("GIT_SHA") + def init_task_id(self): + # If the head commit is a merge, we want to generate a unique task id which incorporates + # the merge parents rather that the actual sha of the merge commit. This ensures that tasks + # can be reused if the tree is in an identical state. Otherwise, if the head commit is + # not a merge, we can rely on the head commit sha for that purpose. + merge_parents_output = subprocess.check_output(["git", "show", "--format=%P", "HEAD"]) + merge_parents = merge_parents_output.decode("utf8").strip().split(' ') + self.task_id = self.git_sha + if len(merge_parents) > 1: + self.task_id = '-'.join(merge_parents) + def git_sha_is_current_head(self): output = subprocess.check_output(["git", "rev-parse", "HEAD"]) self.git_sha = output.decode("utf8").strip() + self.init_task_id() class Shared: From 0ed6cdbb4256d40c15df2058c66098658f997373 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 23 May 2019 18:40:51 +0200 Subject: [PATCH 2/3] Compute the "task ID" only when needed --- etc/taskcluster/decision_task.py | 44 +++++++++++++++----------------- etc/taskcluster/decisionlib.py | 10 +++++--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 82ec88a2572..a732a61f705 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -17,8 +17,6 @@ def main(task_for): ) if task_for == "github-push": - CONFIG.init_task_id() - # FIXME https://github.com/servo/servo/issues/22325 implement these: linux_arm32_dev = linux_arm64_dev = lambda: None @@ -195,12 +193,12 @@ def linux_tidy_unit_docs(): git bundle create docs.bundle HEAD """) .with_artifacts("/repo/target/doc/docs.bundle") - .find_or_create("docs." + CONFIG.task_id) + .find_or_create("docs." + CONFIG.task_id()) ) def upload_docs(): - docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.task_id) + docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.task_id()) return ( linux_task("Upload docs to GitHub Pages") .with_treeherder("Linux x64", "DocUpload") @@ -235,7 +233,7 @@ def macos_unit(): ./mach package --dev ./etc/ci/lockfile_changed.sh """) - .find_or_create("macos_unit." + CONFIG.task_id) + .find_or_create("macos_unit." + CONFIG.task_id()) ) @@ -268,7 +266,7 @@ def android_arm32_dev_from_macos(): ./mach bootstrap-android --accept-all-licences --build ./mach build --android --dev --verbose """) - .find_or_create("android_arm32_dev.macos." + CONFIG.task_id) + .find_or_create("android_arm32_dev.macos." + CONFIG.task_id()) ) @@ -281,7 +279,7 @@ def android_arm32_dev(): ./etc/ci/lockfile_changed.sh python ./etc/ci/check_dynamic_symbols.py """) - .find_or_create("android_arm32_dev." + CONFIG.task_id) + .find_or_create("android_arm32_dev." + CONFIG.task_id()) ) @@ -305,7 +303,7 @@ def android_nightly(): "/repo/target/android/i686-linux-android/release/servoapp.apk", "/repo/target/android/i686-linux-android/release/servoview.aar", ) - .find_or_create("build.android_nightlies." + CONFIG.task_id) + .find_or_create("build.android_nightlies." + CONFIG.task_id()) ) @@ -318,7 +316,7 @@ def android_arm32_release(): "/repo/target/android/armv7-linux-androideabi/release/servoapp.apk", "/repo/target/android/armv7-linux-androideabi/release/servoview.aar", ) - .find_or_create("build.android_armv7_release." + CONFIG.task_id) + .find_or_create("build.android_armv7_release." + CONFIG.task_id()) ) @@ -331,7 +329,7 @@ def android_x86_release(): "/repo/target/android/i686-linux-android/release/servoapp.apk", "/repo/target/android/i686-linux-android/release/servoview.aar", ) - .find_or_create("build.android_x86_release." + CONFIG.task_id) + .find_or_create("build.android_x86_release." + CONFIG.task_id()) ) @@ -354,7 +352,7 @@ def android_x86_wpt(): /_mozilla/mozilla/DOMParser.html \ /_mozilla/mozilla/webgl/context_creation_error.html """) - .find_or_create("android_x86_release." + CONFIG.task_id) + .find_or_create("android_x86_release." + CONFIG.task_id()) ) @@ -368,7 +366,7 @@ def windows_x86(): .with_script( "python mach build --dev --target i686-pc-windows-msvc", ) - .find_or_create("build.windows_x86_dev." + CONFIG.task_id) + .find_or_create("build.windows_x86_dev." + CONFIG.task_id()) ) @@ -387,7 +385,7 @@ def windows_unit(): ) .with_artifacts("repo/target/debug/msi/Servo.exe", "repo/target/debug/msi/Servo.zip") - .find_or_create("build.windows_x64_dev." + CONFIG.task_id) + .find_or_create("build.windows_x64_dev." + CONFIG.task_id()) ) @@ -399,7 +397,7 @@ def windows_release(): "mach package --release") .with_artifacts("repo/target/release/msi/Servo.exe", "repo/target/release/msi/Servo.zip") - .find_or_create("build.windows_x64_release." + CONFIG.task_id) + .find_or_create("build.windows_x64_release." + CONFIG.task_id()) ) @@ -415,7 +413,7 @@ def windows_nightly(): "mach upload-nightly windows-msvc --secret-from-taskcluster") .with_artifacts("repo/target/release/msi/Servo.exe", "repo/target/release/msi/Servo.zip") - .find_or_create("build.windows_x64_nightly." + CONFIG.task_id) + .find_or_create("build.windows_x64_nightly." + CONFIG.task_id()) ) @@ -432,7 +430,7 @@ def linux_nightly(): "./mach upload-nightly linux --secret-from-taskcluster", ) .with_artifacts("/repo/target/release/servo-tech-demo.tar.gz") - .find_or_create("build.linux_x64_nightly" + CONFIG.task_id) + .find_or_create("build.linux_x64_nightly" + CONFIG.task_id()) ) @@ -449,7 +447,7 @@ def linux_wpt(): target/release/build/osmesa-src-*/out/lib/gallium """) .with_artifacts("/target.tar.gz") - .find_or_create("build.linux_x64_release~assertions" + CONFIG.task_id) + .find_or_create("build.linux_x64_release~assertions" + CONFIG.task_id()) ) def linux_run_task(name): return linux_task(name).with_dockerfile(dockerfile_path("run")) @@ -472,7 +470,7 @@ def macos_nightly(): "./mach upload-nightly mac --secret-from-taskcluster", ) .with_artifacts("repo/target/release/servo-tech-demo.dmg") - .find_or_create("build.mac_x64_nightly." + CONFIG.task_id) + .find_or_create("build.mac_x64_nightly." + CONFIG.task_id()) ) @@ -502,7 +500,7 @@ def update_wpt(): ./etc/ci/update-wpt-checkout open-pr ./etc/ci/update-wpt-checkout cleanup """) - .find_or_create("wpt_update." + CONFIG.task_id) + .find_or_create("wpt_update." + CONFIG.task_id()) ) @@ -520,7 +518,7 @@ def macos_release_build(): target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs """) .with_artifacts("repo/target.tar.gz") - .find_or_create("build.macos_x64_release." + CONFIG.task_id) + .find_or_create("build.macos_x64_release." + CONFIG.task_id()) ) @@ -608,7 +606,7 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, if word.endswith(".log") ]) platform_id = platform.replace(" ", "_").lower() - task.find_or_create("%s_wpt_%s.%s" % (platform_id, this_chunk, CONFIG.task_id)) + task.find_or_create("%s_wpt_%s.%s" % (platform_id, this_chunk, CONFIG.task_id())) def daily_tasks_setup(): @@ -847,7 +845,7 @@ def magicleap_dev(): ./mach build --magicleap --dev env -u DYLD_LIBRARY_PATH ./mach package --magicleap --dev """) - .find_or_create("build.magicleap_dev." + CONFIG.git_sha) + .find_or_create("build.magicleap_dev." + CONFIG.task_id()) ) @@ -862,7 +860,7 @@ def magicleap_nightly(): ./mach upload-nightly magicleap --secret-from-taskcluster """) .with_artifacts("repo/target/magicleap/aarch64-linux-android/release/Servo2D.mpk") - .find_or_create("build.magicleap_nightly." + CONFIG.git_sha) + .find_or_create("build.magicleap_nightly." + CONFIG.task_id()) ) diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index cee90ba469b..9e9a0070848 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -57,21 +57,23 @@ class Config: self.git_ref = os.environ.get("GIT_REF") self.git_sha = os.environ.get("GIT_SHA") - def init_task_id(self): + def task_id(self): + if hasattr(self, "_task_id"): + return self._task_id # If the head commit is a merge, we want to generate a unique task id which incorporates # the merge parents rather that the actual sha of the merge commit. This ensures that tasks # can be reused if the tree is in an identical state. Otherwise, if the head commit is # not a merge, we can rely on the head commit sha for that purpose. merge_parents_output = subprocess.check_output(["git", "show", "--format=%P", "HEAD"]) merge_parents = merge_parents_output.decode("utf8").strip().split(' ') - self.task_id = self.git_sha + self._task_id = self.git_sha if len(merge_parents) > 1: - self.task_id = '-'.join(merge_parents) + self._task_id = '-'.join(merge_parents) + return self._task_id def git_sha_is_current_head(self): output = subprocess.check_output(["git", "rev-parse", "HEAD"]) self.git_sha = output.decode("utf8").strip() - self.init_task_id() class Shared: From 3d0be552d82f11c5358c3e35298eda18223688ad Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 23 May 2019 18:50:07 +0200 Subject: [PATCH 3/3] `git show --pretty=%P` sometimes prints a diff. Use `git cat-file` instead. --- .taskcluster.yml | 4 +--- etc/taskcluster/decisionlib.py | 16 +++++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index 87989caba1e..360af84ad3a 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -30,12 +30,10 @@ tasks: - '--login' - '-e' - '-c' - # A depth of 25 is used to work around `git show` ignoring - # any provided format with a depth of 1. - >- git init repo && cd repo && - git fetch --depth 25 "$GIT_URL" "$GIT_REF" && + git fetch --depth 1 "$GIT_URL" "$GIT_REF" && git reset --hard "$GIT_SHA" && python3 etc/taskcluster/decision_task.py in: diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 9e9a0070848..4df1dd57039 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -64,11 +64,17 @@ class Config: # the merge parents rather that the actual sha of the merge commit. This ensures that tasks # can be reused if the tree is in an identical state. Otherwise, if the head commit is # not a merge, we can rely on the head commit sha for that purpose. - merge_parents_output = subprocess.check_output(["git", "show", "--format=%P", "HEAD"]) - merge_parents = merge_parents_output.decode("utf8").strip().split(' ') - self._task_id = self.git_sha - if len(merge_parents) > 1: - self._task_id = '-'.join(merge_parents) + raw_commit = subprocess.check_output(["git", "cat-file", "commit", "HEAD"]) + parent_commits = [ + value.decode("utf8") + for line in raw_commit.split(b"\n") + for key, _, value in [line.partition(b" ")] + if key == b"parent" + ] + if len(parent_commits) > 1: + self._task_id = "-".join(parent_commits) # pragma: no cover + else: + self._task_id = self.git_sha # pragma: no cover return self._task_id def git_sha_is_current_head(self):