Auto merge of #26437 - servo:reftest-report, r=jdm

Generalize the 2020 regression report to show local unexpected failures

Example usage:

```
./mach test-wpt --release --layout-2020 --log-raw /tmp/servo.log
./tests/wpt/reftests-report/gen.py /tmp/servo.log
firefox ./tests/wpt/reftests-report/report.html
```

Produces a report similar https://community-tc.services.mozilla.com/api/index/v1/task/project.servo.layout-2020-regressions-report/artifacts/public/regressions.html, but showing unexpected reftest failures. The CI-generated one shows Layout 2020 failures (expected or not) that succeed in Layout 2013.

.gitignore

@@ -6,7 +6,7 @@
 /.vs
 /android-toolchains
 /target
-/etc/layout-2020-regressions/regressions.html
+/tests/wpt/reftests-report/report.html
 /ports/android/bin
 /ports/android/libs
 /ports/android/local.properties

etc/taskcluster/decision_task.py

@@ -260,12 +260,13 @@ def layout_2020_regressions_report():
         .with_dockerfile(dockerfile_path("base"))
         .with_repo_bundle()
         .with_script(
-            "python3 etc/layout-2020-regressions/gen.py %s %s"
+            "python3 tests/wpt/reftests-report/gen.py %s %s"
             % (CONFIG.tree_hash(), CONFIG.git_sha)
         )
         .with_index_and_artifacts_expire_in(log_artifacts_expire_in)
-        .with_artifacts("/repo/etc/layout-2020-regressions/regressions.html")
-        .find_or_create("layout-2020-regressions-report")
+        .with_artifacts("/repo/tests/wpt/reftests-report/report.html")
+        .with_index_at("layout-2020-regressions-report")
+        .create()
     )
 
 def macos_unit():

etc/taskcluster/decisionlib.py

@@ -157,6 +157,10 @@ class Task:
 
     with_extra = chaining(update_attr, "extra")
 
+    def with_index_at(self, index_path):
+        self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path))
+        return self
+
     def with_treeherder_required(self):
         self.treeherder_required = True
         return self
@@ -291,7 +295,7 @@ class Task:
             if e.status_code != 404:  # pragma: no cover
                 raise
             if not CONFIG.index_read_only:
-                self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path))
+                self.with_index_at(index_path)
             task_id = self.create()
 
         SHARED.found_or_created_indexed_tasks[index_path] = task_id

servo-tidy.toml

@@ -84,6 +84,7 @@ files = [
   # Python 3 syntax causes "E901 SyntaxError" when flake8 runs in Python 2
   "./etc/taskcluster/decision_task.py",
   "./etc/taskcluster/decisionlib.py",
+  "./tests/wpt/reftests-report/gen.py",
   "./components/style/properties/build.py",
 ]
 # Directories that are ignored for the non-WPT tidy check.

tests/wpt/reftests-report/gen.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -41,33 +41,49 @@ def failing_reftests(platform, key):
 
     for chunk in range(1, total_chunks + 1):
         with fetch(task(platform, chunk, key) + "/artifacts/public/test-wpt.log") as response:
-            for line in response:
-                message = json.loads(line)
-                if message.get("status") not in {None, "OK", "PASS"}:
-                    screenshots = message.get("extra", {}).get("reftest_screenshots")
-                    if screenshots:
-                        yield message["test"], screenshots
+            yield from parse(response)
 
 
-def main(index_key, commit_sha):
-    failures_2013 = {url for url, _ in failing_reftests("linux_x64", index_key)}
-    failures_2020 = Directory()
-    for url, screenshots in failing_reftests("linux_x64_2020", index_key):
-        if url not in failures_2013:
-            assert url.startswith("/")
-            failures_2020.add(url[1:], screenshots)
+def parse(file_like):
+    seen = set()
+    for line in file_like:
+        message = json.loads(line)
+        status = message.get("status")
+        if status not in {None, "OK", "PASS"}:
+            screenshots = message.get("extra", {}).get("reftest_screenshots")
+            if screenshots:
+                url = message["test"]
+                assert url.startswith("/")
+                yield url[1:], message.get("expected") == "PASS", screenshots
+
+
+def main(source, commit_sha=None):
+    failures = Directory()
+
+    if commit_sha:
+        title = "<h1>Layout 2020 regressions in commit <code>%s</code></h1>" % commit_sha
+        failures_2013 = {url for url, _ in failing_reftests("linux_x64", source)}
+        for url, _expected_pass, screenshots in failing_reftests("linux_x64_2020", source):
+            if url not in failures_2013:
+                failures.add(url, screenshots)
+    else:
+        title = "Unexpected failures"
+        with open(source, "rb") as file_obj:
+            for url, expected_pass, screenshots in parse(file_obj):
+                if expected_pass:
+                    failures.add(url, screenshots)
 
     here = os.path.dirname(__file__)
     with open(os.path.join(here, "prism.js")) as f:
         prism_js = f.read()
     with open(os.path.join(here, "prism.css")) as f:
         prism_css = f.read()
-    with open(os.path.join(here, "regressions.html"), "w", encoding="utf-8") as html:
-        os.chdir(os.path.join(here, "../../tests/wpt"))
+    with open(os.path.join(here, "report.html"), "w", encoding="utf-8") as html:
+        os.chdir(os.path.join(here, ".."))
         html.write("""
             <!doctype html>
             <meta charset=utf-8>
-            <title>Layout 2020 regressions</title>
+            <title>WPT reftests failures report</title>
             <link rel=stylesheet href=prism.css>
             <style>
                 ul { padding-left: 1em }
@@ -80,11 +96,11 @@ def main(index_key, commit_sha):
                 li > div > img:hover { transform: scale(3); transform-origin: 0 0 }
                 li > div > pre { grid-row: 3; font-size: 12px !important }
                 pre code { white-space: pre-wrap !important }
-                %s
+                <h1>%s</h1>
             </style>
-            <h1>Layout 2020 regressions in tree <code>%s</code></h1>
-        """ % (prism_css, commit_sha))
-        failures_2020.write(html)
+            %s
+        """ % (prism_css, title))
+        failures.write(html)
         html.write("""
             <script>
                 for (let li of document.getElementsByTagName("li")) {
@@ -129,11 +145,15 @@ class Directory:
                     prefix = "/_mozilla/"
                     if url.startswith(prefix):
                         filename = "mozilla/tests/" + url[len(prefix):]
+                    elif url == "about:blank":
+                        src = ""
+                        filename = None
                     else:
                         filename = "web-platform-tests" + url
-                    with open(filename, encoding="utf-8") as f:
-                        src = html_escape(f.read())
-                        html.write("<pre><code class=language-html>%s</code></pre>\n" % src)
+                    if filename:
+                        with open(filename, encoding="utf-8") as f:
+                            src = html_escape(f.read())
+                    html.write("<pre><code class=language-html>%s</code></pre>\n" % src)
             html.write("</li>\n")
         html.write("</ul>\n")