diff --git a/.gitignore b/.gitignore index 25a90bcb750..7d17b468937 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ /.vs /android-toolchains /target -/etc/layout-2020-regressions/regressions.html +/tests/wpt/reftests-report/report.html /ports/android/bin /ports/android/libs /ports/android/local.properties diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 227a5b103a8..3a5ce27070e 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -260,12 +260,13 @@ def layout_2020_regressions_report(): .with_dockerfile(dockerfile_path("base")) .with_repo_bundle() .with_script( - "python3 etc/layout-2020-regressions/gen.py %s %s" + "python3 tests/wpt/reftests-report/gen.py %s %s" % (CONFIG.tree_hash(), CONFIG.git_sha) ) .with_index_and_artifacts_expire_in(log_artifacts_expire_in) - .with_artifacts("/repo/etc/layout-2020-regressions/regressions.html") - .find_or_create("layout-2020-regressions-report") + .with_artifacts("/repo/tests/wpt/reftests-report/report.html") + .with_index_at("layout-2020-regressions-report") + .create() ) def macos_unit(): diff --git a/etc/taskcluster/decisionlib.py b/etc/taskcluster/decisionlib.py index 6deca94bacf..8435cd59137 100644 --- a/etc/taskcluster/decisionlib.py +++ b/etc/taskcluster/decisionlib.py @@ -157,6 +157,10 @@ class Task: with_extra = chaining(update_attr, "extra") + def with_index_at(self, index_path): + self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) + return self + def with_treeherder_required(self): self.treeherder_required = True return self @@ -291,7 +295,7 @@ class Task: if e.status_code != 404: # pragma: no cover raise if not CONFIG.index_read_only: - self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path)) + self.with_index_at(index_path) task_id = self.create() SHARED.found_or_created_indexed_tasks[index_path] = task_id diff --git a/servo-tidy.toml b/servo-tidy.toml index 8c0db763176..73220f9aea2 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -84,6 +84,7 @@ files = [ # Python 3 syntax causes "E901 SyntaxError" when flake8 runs in Python 2 "./etc/taskcluster/decision_task.py", "./etc/taskcluster/decisionlib.py", + "./tests/wpt/reftests-report/gen.py", "./components/style/properties/build.py", ] # Directories that are ignored for the non-WPT tidy check. diff --git a/etc/layout-2020-regressions/gen.py b/tests/wpt/reftests-report/gen.py similarity index 68% rename from etc/layout-2020-regressions/gen.py rename to tests/wpt/reftests-report/gen.py index fd01c4e6d46..bfb7485c86d 100755 --- a/etc/layout-2020-regressions/gen.py +++ b/tests/wpt/reftests-report/gen.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -41,33 +41,49 @@ def failing_reftests(platform, key): for chunk in range(1, total_chunks + 1): with fetch(task(platform, chunk, key) + "/artifacts/public/test-wpt.log") as response: - for line in response: - message = json.loads(line) - if message.get("status") not in {None, "OK", "PASS"}: - screenshots = message.get("extra", {}).get("reftest_screenshots") - if screenshots: - yield message["test"], screenshots + yield from parse(response) -def main(index_key, commit_sha): - failures_2013 = {url for url, _ in failing_reftests("linux_x64", index_key)} - failures_2020 = Directory() - for url, screenshots in failing_reftests("linux_x64_2020", index_key): - if url not in failures_2013: - assert url.startswith("/") - failures_2020.add(url[1:], screenshots) +def parse(file_like): + seen = set() + for line in file_like: + message = json.loads(line) + status = message.get("status") + if status not in {None, "OK", "PASS"}: + screenshots = message.get("extra", {}).get("reftest_screenshots") + if screenshots: + url = message["test"] + assert url.startswith("/") + yield url[1:], message.get("expected") == "PASS", screenshots + + +def main(source, commit_sha=None): + failures = Directory() + + if commit_sha: + title = "

Layout 2020 regressions in commit %s

" % commit_sha + failures_2013 = {url for url, _ in failing_reftests("linux_x64", source)} + for url, _expected_pass, screenshots in failing_reftests("linux_x64_2020", source): + if url not in failures_2013: + failures.add(url, screenshots) + else: + title = "Unexpected failures" + with open(source, "rb") as file_obj: + for url, expected_pass, screenshots in parse(file_obj): + if expected_pass: + failures.add(url, screenshots) here = os.path.dirname(__file__) with open(os.path.join(here, "prism.js")) as f: prism_js = f.read() with open(os.path.join(here, "prism.css")) as f: prism_css = f.read() - with open(os.path.join(here, "regressions.html"), "w", encoding="utf-8") as html: - os.chdir(os.path.join(here, "../../tests/wpt")) + with open(os.path.join(here, "report.html"), "w", encoding="utf-8") as html: + os.chdir(os.path.join(here, "..")) html.write(""" - Layout 2020 regressions + WPT reftests failures report -

Layout 2020 regressions in tree %s

- """ % (prism_css, commit_sha)) - failures_2020.write(html) + %s + """ % (prism_css, title)) + failures.write(html) html.write("""