Auto merge of #22459 - servo:tc-macos-wpt, r=jdm

Move macOS WPT chunk 1 (only) from Buildbot to Taskcluster

This is only one chunk at a time so that we don’t need to double the number of worker nor the cycle time during the migration.

A release build takes ~40 minutes which seems worrying, but this also happens (sometimes) on Buildbot. For example: https://build.servo.org/builders/mac-rel-wpt1/builds/10401

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/22459)
<!-- Reviewable:end -->

etc/ci/buildbot_steps.yml

@@ -5,18 +5,6 @@ env:
   SCCACHE_IDLE_TIMEOUT: "1200"
   GST_DEBUG: '3'
 
-mac-rel-wpt1:
-  - ./mach clean-nightlies --keep 3 --force
-  - ./mach clean-cargo-cache --keep 3 --force
-  - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release
-  - ./mach run -r -o output.png
-  - ./mach test-wpt-failure
-  - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
-  - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default
-  - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
-  - ./mach test-wpt --release --product=servodriver --headless tests/wpt/mozilla/tests/mozilla/DOMParser.html tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html tests/wpt/mozilla/tests/css/img_simple.html tests/wpt/mozilla/tests/mozilla/secure.https.html
-  - bash ./etc/ci/lockfile_changed.sh
-
 mac-rel-wpt2:
   - ./mach clean-nightlies --keep 3 --force
   - ./mach clean-cargo-cache --keep 3 --force
@@ -243,3 +231,4 @@ android: []
 android-x86: []
 linux-rel-wpt: []
 linux-rel-css: []
+mac-rel-wpt1: []

etc/taskcluster/decision_task.py

@@ -18,7 +18,7 @@ def main(task_for):
 
     if task_for == "github-push":
         # FIXME https://github.com/servo/servo/issues/22325 implement these:
-        macos_wpt = magicleap_dev = linux_arm32_dev = linux_arm64_dev = \
+        magicleap_dev = linux_arm32_dev = linux_arm64_dev = \
             android_arm32_dev_from_macos = lambda: None
 
         # FIXME https://github.com/servo/servo/issues/22187
@@ -80,7 +80,6 @@ def main(task_for):
 # but should still run when testing this Python code. (See `mock.py`.)
 def mocked_only():
     windows_release()
-    linux_wpt()
     android_x86_wpt()
     decisionlib.DockerWorkerTask("Indexed by task definition").find_or_create()
 
@@ -173,6 +172,7 @@ def upload_docs():
         .create()
     )
 
+
 def macos_unit():
     return (
         macos_build_task("Dev build + unit tests")
@@ -325,21 +325,11 @@ def linux_nightly():
 
 
 def linux_wpt():
-    release_build_task = linux_release_build(with_debug_assertions=True)
+    release_build_task = (
-    total_chunks = 2
+        linux_build_task("Release build, with debug assertions")
-    for i in range(total_chunks):
+        .with_treeherder("Linux x64", "Release+A")
-        this_chunk = i + 1
-        wpt_chunk(release_build_task, total_chunks, this_chunk)
-
-
-def linux_release_build(with_debug_assertions=False):
-    a = with_debug_assertions
-    return (
-        linux_build_task("Release build" + ", with debug assertions" if a else "")
-        .with_treeherder("Linux x64", "Release" + "+A" if a else "")
-        .with_env(BUILD_FLAGS="--with-debug-assertions" if a else "")
         .with_script("""
-            ./mach build --release $BUILD_FLAGS -p servo
+            ./mach build --release --with-debug-assertions -p servo
             ./etc/ci/lockfile_changed.sh
             tar -czf /target.tar.gz \
                 target/release/servo \
@@ -347,68 +337,101 @@ def linux_release_build(with_debug_assertions=False):
                 target/release/build/osmesa-src-*/out/lib/gallium
         """)
         .with_artifacts("/target.tar.gz")
-        .find_or_create(
+        .find_or_create("build.linux_x64_release~assertions" + CONFIG.git_sha)
-            "build.linux_x64_release%s.%s" % ("_assertions" if a else "", CONFIG.git_sha)
-        )
     )
+    def linux_run_task(name):
+        return linux_task(name).with_dockerfile(dockerfile_path("run"))
+    wpt_chunks("Linux x64", linux_run_task, release_build_task, repo_dir="/repo",
+               total_chunks=2, processes=24)
 
 
-def wpt_chunk(release_build_task, total_chunks, this_chunk):
+def macos_wpt():
-    task = (
+    build_task = (
-        linux_task("WPT chunk %s / %s" % (this_chunk, total_chunks))
+        macos_build_task("Release build")
-        .with_treeherder("Linux x64", "WPT-%s" % this_chunk)
+        .with_treeherder("macOS x64", "Release")
-        .with_dockerfile(dockerfile_path("run"))
+        .with_script("""
-        .with_repo()
+            ./mach build --release
-        .with_curl_artifact_script(release_build_task, "target.tar.gz")
+            ./etc/ci/lockfile_changed.sh
-        .with_script("tar -xzf target.tar.gz")
+            tar -czf target.tar.gz \
-        .with_index_and_artifacts_expire_in(log_artifacts_expire_in)
+                target/release/servo \
-        .with_max_run_time_minutes(60)
+                target/release/build/osmesa-src-*/output \
-        .with_env(TOTAL_CHUNKS=total_chunks, THIS_CHUNK=this_chunk)
+                target/release/build/osmesa-src-*/out/src/gallium/targets/osmesa/.libs \
-    )
+                target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs
-    if this_chunk == 1:
-        task.name += " + extra"
-        task.extra["treeherder"]["symbol"] += "+"
-        task.with_script("""
-            ./mach test-wpt-failure
-            ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \
-                --log-raw test-wpt-mp.log \
-                --log-errorsummary wpt-mp-errorsummary.log \
-                eventsource \
-                | cat
-            time ./mach test-wpt --release --product=servodriver --headless  \
-                tests/wpt/mozilla/tests/mozilla/DOMParser.html \
-                tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \
-                tests/wpt/mozilla/tests/css/img_simple.html \
-                tests/wpt/mozilla/tests/mozilla/secure.https.html \
-                | cat
         """)
-    # `test-wpt` is piped into `cat` so that stdout is not a TTY
+        .with_artifacts("repo/target.tar.gz")
-    # and wptrunner does not use "interactive mode" formatting:
+        .find_or_create("build.macos_x64_release." + CONFIG.git_sha)
-    # https://github.com/servo/servo/issues/22438
+    )
-    task.with_script("""
+    def macos_run_task(name):
-        ./mach test-wpt \
+        return macos_task(name).with_python2()
-            --release \
+    wpt_chunks("macOS x64", macos_run_task, build_task, repo_dir="repo",
-            --processes 24 \
+               total_chunks=6, processes=4, chunks=[1])
-            --total-chunks "$TOTAL_CHUNKS" \
+
-            --this-chunk "$THIS_CHUNK" \
+
-            --log-raw test-wpt.log \
+def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes,
-            --log-errorsummary wpt-errorsummary.log \
+               repo_dir, chunks="all"):
-            --always-succeed \
+    if chunks == "all":
-            | cat
+        chunks = [n + 1 for n in range(total_chunks)]
-        ./mach filter-intermittents\
+    for this_chunk in chunks:
-            wpt-errorsummary.log \
+        task = (
-            --log-intermittents intermittents.log \
+            make_chunk_task("WPT chunk %s / %s" % (this_chunk, total_chunks))
-            --log-filteredsummary filtered-wpt-errorsummary.log \
+            .with_treeherder(platform, "WPT-%s" % this_chunk)
-            --tracker-api default \
+            .with_repo()
-            --reporter-api default
+            .with_curl_artifact_script(build_task, "target.tar.gz")
-    """)
+            .with_script("tar -xzf target.tar.gz")
-    task.with_artifacts(*[
+            .with_index_and_artifacts_expire_in(log_artifacts_expire_in)
-        "/repo/" + word
+            .with_max_run_time_minutes(60)
-        for script in task.scripts
+            .with_env(
-        for word in script.split()
+                TOTAL_CHUNKS=str(total_chunks),
-        if word.endswith(".log")
+                THIS_CHUNK=str(this_chunk),
-    ])
+                PROCESSES=str(processes),
-    return task.find_or_create("linux_wpt_%s.%s" % (this_chunk, CONFIG.git_sha))
+            )
+        )
+        if this_chunk == chunks[-1]:
+            task.name += " + extra"
+            task.extra["treeherder"]["symbol"] += "+"
+            task.with_script("""
+                ./mach test-wpt-failure
+                time ./mach test-wpt --release --binary-arg=--multiprocess \
+                    --processes $PROCESSES \
+                    --log-raw test-wpt-mp.log \
+                    --log-errorsummary wpt-mp-errorsummary.log \
+                    eventsource \
+                    | cat
+                time ./mach test-wpt --release --product=servodriver --headless  \
+                    tests/wpt/mozilla/tests/mozilla/DOMParser.html \
+                    tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \
+                    tests/wpt/mozilla/tests/css/img_simple.html \
+                    tests/wpt/mozilla/tests/mozilla/secure.https.html \
+                    | cat
+            """)
+        # `test-wpt` is piped into `cat` so that stdout is not a TTY
+        # and wptrunner does not use "interactive mode" formatting:
+        # https://github.com/servo/servo/issues/22438
+        task.with_script("""
+            ./mach test-wpt \
+                --release \
+                --processes $PROCESSES \
+                --total-chunks "$TOTAL_CHUNKS" \
+                --this-chunk "$THIS_CHUNK" \
+                --log-raw test-wpt.log \
+                --log-errorsummary wpt-errorsummary.log \
+                --always-succeed \
+                | cat
+            ./mach filter-intermittents \
+                wpt-errorsummary.log \
+                --log-intermittents intermittents.log \
+                --log-filteredsummary filtered-wpt-errorsummary.log \
+                --tracker-api default \
+                --reporter-api default
+        """)
+        task.with_artifacts(*[
+            "%s/%s" % (repo_dir, word)
+            for script in task.scripts
+            for word in script.split()
+            if word.endswith(".log")
+        ])
+        platform_id = platform.replace(" ", "_").lower()
+        task.find_or_create("%s_wpt_%s.%s" % (platform_id, this_chunk, CONFIG.git_sha))
 
 
 def daily_tasks_setup():

etc/taskcluster/decisionlib.py

@@ -541,7 +541,57 @@ class WindowsGenericWorkerTask(GenericWorkerTask):
         .with_path_from_homedir("python2", "python2\\Scripts")
 
 
-class MacOsGenericWorkerTask(GenericWorkerTask):
+class UnixTaskMixin(Task):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.curl_scripts_count = 0
+
+    def with_repo(self):
+        """
+        Make a shallow clone the git repository at the start of the task.
+        This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`
+
+        * generic-worker: creates the clone in a `repo` directory
+          in the task’s directory.
+
+        * docker-worker: creates the clone in a `/repo` directory
+          at the root of the Docker container’s filesystem.
+          `git` and `ca-certificate` need to be installed in the Docker image.
+
+        """
+        return self \
+        .with_env(**git_env()) \
+        .with_early_script("""
+            git init repo
+            cd repo
+            git fetch --depth 1 "$GIT_URL" "$GIT_REF"
+            git reset --hard "$GIT_SHA"
+        """)
+
+    def with_curl_script(self, url, file_path):
+        self.curl_scripts_count += 1
+        n = self.curl_scripts_count
+        return self \
+        .with_env(**{
+            "CURL_%s_URL" % n: url,
+            "CURL_%s_PATH" % n: file_path,
+        }) \
+        .with_script("""
+            mkdir -p $(dirname "$CURL_{n}_PATH")
+            curl --retry 5 --connect-timeout 10 -Lf "$CURL_{n}_URL" -o "$CURL_{n}_PATH"
+        """.format(n=n))
+
+    def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
+        return self \
+        .with_dependencies(task_id) \
+        .with_curl_script(
+            "https://queue.taskcluster.net/v1/task/%s/artifacts/public/%s"
+                % (task_id, artifact_name),
+            os.path.join(out_directory, url_basename(artifact_name)),
+        )
+
+
+class MacOsGenericWorkerTask(UnixTaskMixin, GenericWorkerTask):
     """
     Task definition for a `generic-worker` task running on macOS.
 
@@ -566,21 +616,6 @@ class MacOsGenericWorkerTask(GenericWorkerTask):
             ]
         ]
 
-    def with_repo(self):
-        """
-        Make a shallow clone the git repository at the start of the task.
-        This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`,
-        and creates the clone in a `repo` directory in the task’s directory.
-        """
-        return self \
-        .with_env(**git_env()) \
-        .with_early_script("""
-            git init repo
-            cd repo
-            git fetch --depth 1 "$GIT_URL" "$GIT_REF"
-            git reset --hard "$GIT_SHA"
-        """)
-
     def with_python2(self):
         return self.with_early_script("""
             export PATH="$HOME/Library/Python/2.7/bin:$PATH"
@@ -595,7 +630,7 @@ class MacOsGenericWorkerTask(GenericWorkerTask):
         """)
 
 
-class DockerWorkerTask(Task):
+class DockerWorkerTask(UnixTaskMixin, Task):
     """
     Task definition for a worker type that runs the `generic-worker` implementation.
 
@@ -613,7 +648,6 @@ class DockerWorkerTask(Task):
         self.features = {}
         self.capabilities = {}
         self.artifacts = []
-        self.curl_scripts_count = 0
 
     with_docker_image = chaining(setattr, "docker_image")
     with_max_run_time_minutes = chaining(setattr, "max_run_time_minutes")
@@ -663,46 +697,6 @@ class DockerWorkerTask(Task):
         self.features.update({name: True for name in names})
         return self
 
-    def with_curl_script(self, url, file_path):
-        self.curl_scripts_count += 1
-        n = self.curl_scripts_count
-        return self \
-        .with_env(**{
-            "CURL_%s_URL" % n: url,
-            "CURL_%s_PATH" % n: file_path,
-        }) \
-        .with_script("""
-            mkdir -p $(dirname "$CURL_{n}_PATH")
-            curl --retry 5 --connect-timeout 10 -Lf "$CURL_{n}_URL" -o "$CURL_{n}_PATH"
-        """.format(n=n))
-
-    def with_curl_artifact_script(self, task_id, artifact_name, out_directory=""):
-        return self \
-        .with_dependencies(task_id) \
-        .with_curl_script(
-            "https://queue.taskcluster.net/v1/task/%s/artifacts/public/%s"
-                % (task_id, artifact_name),
-            os.path.join(out_directory, url_basename(artifact_name)),
-        )
-
-    def with_repo(self):
-        """
-        Make a shallow clone the git repository at the start of the task.
-        This uses `CONFIG.git_url`, `CONFIG.git_ref`, and `CONFIG.git_sha`,
-        and creates the clone in a `/repo` directory
-        at the root of the Docker container’s filesystem.
-
-        `git` and `ca-certificate` need to be installed in the Docker image.
-        """
-        return self \
-        .with_env(**git_env()) \
-        .with_early_script("""
-            git init repo
-            cd repo
-            git fetch --depth 1 "$GIT_URL" "$GIT_REF"
-            git reset --hard "$GIT_SHA"
-        """)
-
     def with_dockerfile(self, dockerfile):
         """
         Build a Docker image based on the given `Dockerfile`, and use it for this task.

etc/taskcluster/macos/Brewfile

@@ -9,7 +9,8 @@ brew "gst-libav"
 brew "gst-plugins-bad"
 brew "gst-plugins-good"
 brew "gst-rtsp-server"
-brew "htop"
 brew "llvm"
-brew "openssl@1.1"
 brew "yasm"
+
+# For sccache
+brew "openssl@1.1"

etc/taskcluster/macos/config/roster

@@ -1,2 +1,4 @@
 mac1:
     host: servo-tc-mac1.servo.org
+mac2:
+    host: servo-tc-mac2.servo.org

etc/taskcluster/macos/states/generic-worker.plist.jinja

@@ -7,7 +7,7 @@
     <key>StandardOutPath</key>   <string>stdout.log</string>
     <key>StandardErrorPath</key> <string>stderr.log</string>
     <key>WorkingDirectory</key>  <string>{{ home }}</string>
-    <key>UserName</key>          <string>{{ user }}</string>
+    <key>UserName</key>          <string>{{ username }}</string>
     <key>ProgramArguments</key>  <array>
         <string>{{ bin }}/generic-worker</string>
         <string>run</string>

etc/taskcluster/macos/states/generic-worker.sls

@@ -46,7 +46,7 @@ GMT:
         provisionerId: proj-servo
         workerType: macos
         workerGroup: servo-macos
-        workerId: mac1
+        workerId: {{ grains["id"] }}
         tasksDir: {{ home }}/tasks
         publicIP: {{ salt.network.ip_addrs()[0] }}
         signingKeyLocation: {{ home }}/key
@@ -85,7 +85,7 @@ GMT:
       bin: {{ bin }}
       etc: {{ etc }}
       home: {{ home }}
-      user: {{ user }}
+      username: {{ user }}
 
 net.generic.worker:
   service.running:

python/servo/testing_commands.py

@@ -574,9 +574,6 @@ class MachCommands(CommandBase):
                     json.dump(intermittent, intermittents_file, indent=4)
                     print("\n", end='', file=intermittents_file)
 
-        if len(actual_failures) == 0:
-            return 0
-
         output = open(log_filteredsummary, "w") if log_filteredsummary else sys.stdout
         for failure in actual_failures:
             json.dump(failure, output, indent=4)
@@ -584,6 +581,9 @@ class MachCommands(CommandBase):
 
         if output is not sys.stdout:
             output.close()
+
+        if len(actual_failures) == 0:
+            return 0
         return 1
 
     @Command('test-android-startup',