Taskcluster: move curl’ing artifacts into decisionlib

etc/taskcluster/curl-artifact.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-task_id="${1}"
-artifact="${2}"
-shift 2
-queue="https://queue.taskcluster.net/v1"
-url="${queue}/task/${task_id}/artifacts/public/${artifact}"
-echo "Fetching ${url}" >&2
-curl \
-    --retry 5 \
-    --connect-timeout 10 \
-    --location \
-    --fail \
-    "${url}" \
-    "${@}"

etc/taskcluster/decision_task.py

@@ -110,14 +110,9 @@ def android_x86():
         .with_capabilities(privileged=True)
         .with_scopes("project:servo:docker-worker-kvm:capability:privileged")
         .with_dockerfile(dockerfile_path("run-android-emulator"))
-        .with_dependencies(build_task)
-        .with_env(BUILD_TASK_ID=build_task)
         .with_repo()
+        .with_curl_artifact_script(build_task, "servoapp.apk", "target/i686-linux-android/release")
         .with_script("""
-            mkdir -p target/i686-linux-android/release/
-            ./etc/taskcluster/curl-artifact.sh ${BUILD_TASK_ID} servoapp.apk \
-                -o target/i686-linux-android/release/servoapp.apk
-
             ./mach bootstrap-android --accept-all-licences --emulator-x86
             ./mach test-android-startup --release
             ./mach test-wpt-android --release \
@@ -128,7 +123,6 @@ def android_x86():
     )
 
 
-
 def windows_dev():
     return (
         windows_build_task("Windows x64: dev build + unit tests")
@@ -224,11 +218,8 @@ def linux_run_task(name, build_task, script):
         linux_task(name)
         .with_dockerfile(dockerfile_path("run"))
         .with_repo()
-        .with_early_script("""
-            ./etc/taskcluster/curl-artifact.sh ${BUILD_TASK_ID} target.tar.gz | tar -xz
-        """)
-        .with_env(BUILD_TASK_ID=build_task)
-        .with_dependencies(build_task)
+        .with_curl_artifact_script(build_task, "target.tar.gz")
+        .with_script("tar -xzf target.tar.gz")
         .with_script(script)
         .with_index_and_artifacts_expire_in(log_artifacts_expire_in)
         .with_artifacts(*[

etc/taskcluster/decisionlib.py

@@ -514,6 +514,7 @@ class DockerWorkerTask(Task):
         self.features = {}
         self.capabilities = {}
         self.artifacts = []
+        self.curl_scripts_count = 0
 
     with_docker_image = chaining(setattr, "docker_image")
     with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes")
@@ -563,6 +564,28 @@ class DockerWorkerTask(Task):
         self.features.update({name: True for name in names})
         return self
 
+    def with_curl_script(self, url, file_path):
+        self.curl_scripts_count += 1
+        n = self.curl_scripts_count
+        return self \
+        .with_env(**{
+            "CURL_%s_URL" % n: url,
+            "CURL_%s_PATH" % n: file_path,
+        }) \
+        .with_script("""
+            mkdir -p $(dirname "$CURL_{n}_PATH")
+            curl --retry 5 --connect-timeout 10 -Lf "$CURL_{n}_URL" -o "$CURL_{n}_PATH"
+        """.format(n=n))
+
+    def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
+        return self \
+        .with_dependencies(task_id) \
+        .with_curl_script(
+            "https://queue.taskcluster.net/v1/task/%s/artifacts/public/%s"
+                % (task_id, artifact_name),
+            os.path.join(out_directory, url_basename(artifact_name)),
+        )
+
     def with_repo(self):
         """
         Make a shallow clone the git repository at the start of the task.