Split decision task code into more functions

etc/ci/taskcluster/decision-task.py

@@ -10,42 +10,25 @@ from decisionlib import DecisionTask
 
 
 def main():
-    decision = DecisionTask(
+    linux_tidy_unit()
-        project_name="Servo",  # Used in task names
+    linux_wpt()
-        route_prefix="project.servo.servo",
-        worker_type="servo-docker-worker",
-    )
 
-    build_artifacts_expiry = "1 week"
-    log_artifacts_expiry = "1 year"
 
-    # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
+build_artifacts_expiry = "1 week"
-    cache_scopes = [
+log_artifacts_expiry = "1 year"
-        "docker-worker:cache:cargo-*",
-    ]
-    build_caches = {
-        "cargo-registry-cache": "/root/.cargo/registry",
-        "cargo-git-cache": "/root/.cargo/git",
-        "cargo-rustup": "/root/.rustup",
-        "cargo-sccache": "/root/.cache/sccache",
-    }
-    build_env = {
-        "RUST_BACKTRACE": "1",
-        "RUSTFLAGS": "-Dwarnings",
-        "CARGO_INCREMENTAL": "0",
-        "SCCACHE_IDLE_TIMEOUT": "1200",
-        "CCACHE": "sccache",
-        "RUSTC_WRAPPER": "sccache",
-    }
-    build_kwargs = {
-        "max_run_time_minutes": 60,
-        "dockerfile": dockerfile_path("build"),
-        "env": build_env,
-        "scopes": cache_scopes,
-        "cache": build_caches,
-    }
 
-    decision.create_task(
+build_env = {
+    "RUST_BACKTRACE": "1",
+    "RUSTFLAGS": "-Dwarnings",
+    "CARGO_INCREMENTAL": "0",
+    "SCCACHE_IDLE_TIMEOUT": "1200",
+    "CCACHE": "sccache",
+    "RUSTC_WRAPPER": "sccache",
+}
+
+
+def linux_tidy_unit():
+    return decision.create_task(
         task_name="Linux x86_64: tidy + dev build + unit tests",
         script="""
             ./mach test-tidy --no-progress --all
@@ -60,9 +43,19 @@ def main():
         **build_kwargs
     )
 
-    release_build_task = decision.find_or_create_task(
+
+def linux_wpt():
+    release_build_task = linux_release_build()
+    total_chunks = 2
+    for i in range(total_chunks):
+        this_chunk = i + 1
+        wpt_chunk(release_build_task, total_chunks, this_chunk, extra=(this_chunk == 1))
+
+
+def linux_release_build():
+    return decision.find_or_create_task(
         route_bucket="build.linux_x86-64_release",
-        route_key=os.environ["GIT_SHA"],
+        route_key=os.environ["GIT_SHA"],  # Set in .taskcluster.yml
         route_expiry=build_artifacts_expiry,
 
         task_name="Linux x86_64: release build",
@@ -80,70 +73,96 @@ def main():
         **build_kwargs
     )
 
-    def create_run_task(*, script, env=None, **kwargs):
-        fetch_build = """
-            ./etc/ci/taskcluster/curl-artifact.sh ${BUILD_TASK_ID} target.tar.gz | tar -xz
-        """
-        kwargs.setdefault("artifacts", []).extend(
-            ("/repo/" + word, log_artifacts_expiry)
-            for word in script.split() if word.endswith(".log")
-        )
-        decision.create_task(
-            script=fetch_build + script,
-            env=dict(**env or {}, BUILD_TASK_ID=release_build_task),
-            dependencies=[release_build_task],
-            max_run_time_minutes=60,
-            dockerfile=dockerfile_path("run"),
-            **kwargs
-        )
 
-    total_chunks = 2
+def wpt_chunk(release_build_task, total_chunks, this_chunk, extra):
-    for i in range(total_chunks):
+    if extra:
-        chunk = i + 1
+        name_extra = " + extra"
-        if chunk == 1:
+        script_extra = """
-            name_extra = " + extra"
+            ./mach test-wpt-failure
-            script_extra = """
+            ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \
-                ./mach test-wpt-failure
+                --log-raw test-wpt-mp.log \
-                ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \
+                --log-errorsummary wpt-mp-errorsummary.log \
-                    --log-raw test-wpt-mp.log \
+                eventsource
-                    --log-errorsummary wpt-mp-errorsummary.log \
-                    eventsource
-            """
-        else:
-            name_extra = ""
-            script_extra = ""
-        script = """
-            ./mach test-wpt \
-                --release \
-                --processes 24 \
-                --total-chunks "$TOTAL_CHUNKS" \
-                --this-chunk "$THIS_CHUNK" \
-                --log-raw test-wpt.log \
-                --log-errorsummary wpt-errorsummary.log \
-                --always-succeed
-            ./mach filter-intermittents\
-                wpt-errorsummary.log \
-                --log-intermittents intermittents.log \
-                --log-filteredsummary filtered-wpt-errorsummary.log \
-                --tracker-api default
         """
-        # FIXME: --reporter-api default
+    else:
-        # IndexError: list index out of range
+        name_extra = ""
-        # File "/repo/python/servo/testing_commands.py", line 533, in filter_intermittents
+        script_extra = ""
-        #   pull_request = int(last_merge.split(' ')[4][1:])
+    script = """
-        create_run_task(
+        ./mach test-wpt \
-            task_name="Linux x86_64: WPT chunk %s / %s%s" % (chunk, total_chunks, name_extra),
+            --release \
-            script=script_extra + script,
+            --processes 24 \
-            env={
+            --total-chunks "$TOTAL_CHUNKS" \
-                "TOTAL_CHUNKS": total_chunks,
+            --this-chunk "$THIS_CHUNK" \
-                "THIS_CHUNK": chunk,
+            --log-raw test-wpt.log \
-            },
+            --log-errorsummary wpt-errorsummary.log \
-        )
+            --always-succeed
+        ./mach filter-intermittents\
+            wpt-errorsummary.log \
+            --log-intermittents intermittents.log \
+            --log-filteredsummary filtered-wpt-errorsummary.log \
+            --tracker-api default
+    """
+    # FIXME: --reporter-api default
+    # IndexError: list index out of range
+    # File "/repo/python/servo/testing_commands.py", line 533, in filter_intermittents
+    #   pull_request = int(last_merge.split(' ')[4][1:])
+    create_run_task(
+        build_task=release_build_task,
+        task_name="Linux x86_64: WPT chunk %s / %s%s" % (this_chunk, total_chunks, name_extra),
+        script=script_extra + script,
+        env={
+            "TOTAL_CHUNKS": total_chunks,
+            "THIS_CHUNK": this_chunk,
+        },
+    )
+
+
+def create_run_task(*, build_task, script, **kwargs):
+    fetch_build = """
+        ./etc/ci/taskcluster/curl-artifact.sh ${BUILD_TASK_ID} target.tar.gz | tar -xz
+    """
+    kwargs.setdefault("env", {})["BUILD_TASK_ID"] = build_task
+    kwargs.setdefault("dependencies", []).append(build_task)
+    kwargs.setdefault("artifacts", []).extend(
+        ("/repo/" + word, log_artifacts_expiry)
+        for word in script.split() if word.endswith(".log")
+    )
+    return decision.create_task(
+        script=fetch_build + script,
+        max_run_time_minutes=60,
+        dockerfile=dockerfile_path("run"),
+        **kwargs
+    )
 
 
 def dockerfile_path(name):
     return os.path.join(os.path.dirname(__file__), "docker", name + ".dockerfile")
 
 
+decision = DecisionTask(
+    project_name="Servo",  # Used in task names
+    route_prefix="project.servo.servo",
+    worker_type="servo-docker-worker",
+)
+
+# https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
+cache_scopes = [
+    "docker-worker:cache:cargo-*",
+]
+build_caches = {
+    "cargo-registry-cache": "/root/.cargo/registry",
+    "cargo-git-cache": "/root/.cargo/git",
+    "cargo-rustup": "/root/.rustup",
+    "cargo-sccache": "/root/.cache/sccache",
+}
+build_kwargs = {
+    "max_run_time_minutes": 60,
+    "dockerfile": dockerfile_path("build"),
+    "env": build_env,
+    "scopes": cache_scopes,
+    "cache": build_caches,
+}
+
+
 if __name__ == "__main__":
     main()