Cache repackaged Python 2 for Windows

etc/taskcluster/decision-task.py

@@ -4,6 +4,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import hashlib
+import json
 import os.path
 import subprocess
 from decisionlib import DecisionTask
@@ -97,29 +99,16 @@ def android_arm32():
 
 
 def windows_dev():
-    def extract_msi(*names):
+    python2_task_definition = dict(
-        return "".join(
+        task_name="Windows x86_64: repackage Python 2",
-            "lessmsi x %HOMEDRIVE%%HOMEPATH%\\{name}.msi %HOMEDRIVE%%HOMEPATH%\\{name}\\\n"
-            .format(name=name)
-            for name in names
-        )
-
-    return decision.create_task(
-        task_name="Windows x86_64: clone only (for now)",
         worker_type="servo-win2016",
-        script=extract_msi("python2") + """
+        with_repo=False,
-            python -c "import os; print(os.listdir('.'))"
+        script="""
+            lessmsi x python2.msi python2\\
+            cd python2\\SourceDir
+            7za a python2.zip *
         """,
         mounts=[
-            {
-                "directory": "git",
-                "format": "zip",
-                "content": {
-                    "url": "https://github.com/git-for-windows/git/releases/download/" +
-                           "v2.19.0.windows.1/MinGit-2.19.0-64-bit.zip",
-                    "sha256": "424d24b5fc185a9c5488d7872262464f2facab4f1d4693ea8008196f14a3c19b",
-                }
-            },
             {
                 "file": "python2.msi",
                 "content": {
@@ -136,13 +125,73 @@ def windows_dev():
                     "sha256": "540b8801e08ec39ba26a100c855898f455410cecbae4991afae7bb2b4df026c7",
                 }
             },
+            {
+                "directory": "7zip",
+                "format": "zip",
+                "content": {
+                    "url": "https://www.7-zip.org/a/7za920.zip",
+                    "sha256": "2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac",
+                }
+            }
+        ],
+        homedir_path=[
+            "lessmsi",
+            "7zip",
+        ],
+        artifacts=[
+            "python2/SourceDir/python2.zip",
+        ],
+        max_run_time_minutes=20,
+    )
+    index_by = json.dumps(python2_task_definition).encode("utf-8")
+    python2_task = decision.find_or_create_task(
+        index_bucket="by-task-definition",
+        index_key=hashlib.sha256(index_by).hexdigest(),
+        index_expiry=build_artifacts_expiry,
+        **python2_task_definition
+    )
+
+    return decision.create_task(
+        task_name="Windows x86_64: clone only (for now)",
+        worker_type="servo-win2016",
+        script="""
+            python -m ensurepip
+            pip install virtualenv==16.0.0
+            python mach --help
+        """,
+        mounts=[
+            {
+                "directory": "git",
+                "format": "zip",
+                "content": {
+                    "url": "https://github.com/git-for-windows/git/releases/download/" +
+                           "v2.19.0.windows.1/MinGit-2.19.0-64-bit.zip",
+                    "sha256": "424d24b5fc185a9c5488d7872262464f2facab4f1d4693ea8008196f14a3c19b",
+                }
+            },
+            {
+                "directory": "python2",
+                "format": "zip",
+                "content": {
+                    "taskId": python2_task,
+                    "artifact": "public/python2.zip",
+                }
+            },
         ],
         homedir_path=[
             "git\\cmd",
-            "lessmsi",
+            "python2",
-            "python2\\SourceDir",
+            "python2\\Scripts",
+        ],
+        dependencies=[python2_task],
+        sparse_checkout=[
+            "/*",
+            "!/tests/wpt/metadata",
+            "!/tests/wpt/mozilla",
+            "!/tests/wpt/webgl",
+            "!/tests/wpt/web-platform-tests",
+            "/tests/wpt/web-platform-tests/tools",
         ],
-        sparse_checkout_exclude=["tests/wpt"],
         **build_kwargs
     )
 

etc/taskcluster/decisionlib.py

@@ -135,7 +135,7 @@ class DecisionTask:
                     docker_image=None, dockerfile=None,  # One of these is required
                     artifacts=None, dependencies=None, env=None, cache=None, scopes=None,
                     routes=None, extra=None, features=None, mounts=None, homedir_path=None,
-                    worker_type=None, with_repo=True, sparse_checkout_exclude=None):
+                    worker_type=None, with_repo=True, sparse_checkout=None):
         """
         Schedule a new task. Returns the new task ID.
 
@@ -196,7 +196,7 @@ class DecisionTask:
                         git init repo
                         cd repo
                     """
-                    if sparse_checkout_exclude:
+                    if sparse_checkout:
                         git += """
                             git config core.sparsecheckout true
                             echo %SPARSE_CHECKOUT_BASE64% > .git\\info\\sparse.b64
@@ -204,9 +204,7 @@ class DecisionTask:
                             type .git\\info\\sparse-checkout
                         """
                         env["SPARSE_CHECKOUT_BASE64"] = base64.b64encode(
-                            b"/*" +
+                            "\n".join(sparse_checkout).encode("utf-8"))
-                            "".join("\n!" + p for p in sparse_checkout_exclude).encode("utf-8")
-                        )
                 command.append(deindent(git + """
                     git fetch --depth 1 %GIT_URL% %GIT_REF%
                     git reset --hard %GIT_SHA%
@@ -227,14 +225,25 @@ class DecisionTask:
         if mounts:
             worker_payload["mounts"] = mounts
         if artifacts:
-            worker_payload["artifacts"] = {
+            if "docker" in worker_type:
-                "public/" + os.path.basename(path): {
+                worker_payload["artifacts"] = {
-                    "type": "file",
+                    "public/" + os.path.basename(path): {
-                    "path": path,
+                        "type": "file",
-                    "expires": self.from_now_json(expires),
+                        "path": path,
+                        "expires": self.from_now_json(expires),
+                    }
+                    for path, expires in artifacts
                 }
-                for path, expires in artifacts
+            else:
-            }
+                worker_payload["artifacts"] = [
+                    {
+                        "type": "file",
+                        "name": "public/" + os.path.basename(path),
+                        "path": path,
+                        "expires": self.from_now_json(expires),
+                    }
+                    for path, expires in artifacts
+                ]
         payload = {
             "taskGroupId": decision_task_id,
             "dependencies": dependencies or [],