mirror of
https://github.com/servo/servo.git
synced 2025-08-06 22:15:33 +01:00
Auto merge of #26437 - servo:reftest-report, r=jdm
Generalize the 2020 regression report to show local unexpected failures Example usage: ``` ./mach test-wpt --release --layout-2020 --log-raw /tmp/servo.log ./tests/wpt/reftests-report/gen.py /tmp/servo.log firefox ./tests/wpt/reftests-report/report.html ``` Produces a report similar https://community-tc.services.mozilla.com/api/index/v1/task/project.servo.layout-2020-regressions-report/artifacts/public/regressions.html, but showing unexpected reftest failures. The CI-generated one shows Layout 2020 failures (expected or not) that succeed in Layout 2013.
This commit is contained in:
bors-servo
GitHub
commit
856f03ae75
7 changed files with 55 additions and 29 deletions
|
|
@ -1,142 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from html import escape as html_escape
|
||||
|
||||
|
||||
TASKCLUSTER_ROOT_URL = "https://community-tc.services.mozilla.com"
|
||||
|
||||
|
||||
def fetch(url):
|
||||
url = TASKCLUSTER_ROOT_URL + "/api/" + url
|
||||
print("Fetching " + url)
|
||||
response = urllib.request.urlopen(url)
|
||||
assert response.getcode() == 200
|
||||
return response
|
||||
|
||||
|
||||
def fetch_json(url):
|
||||
with fetch(url) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def task(platform, chunk, key):
|
||||
return "index/v1/task/project.servo.%s_wpt_%s.%s" % (platform, chunk, key)
|
||||
|
||||
|
||||
def failing_reftests(platform, key):
|
||||
chunk_1_task_id = fetch_json(task(platform, 1, key))["taskId"]
|
||||
name = fetch_json("queue/v1/task/" + chunk_1_task_id)["metadata"]["name"]
|
||||
match = re.search("WPT chunk (\d+) / (\d+)", name)
|
||||
assert match.group(1) == "1"
|
||||
total_chunks = int(match.group(2))
|
||||
|
||||
for chunk in range(1, total_chunks + 1):
|
||||
with fetch(task(platform, chunk, key) + "/artifacts/public/test-wpt.log") as response:
|
||||
for line in response:
|
||||
message = json.loads(line)
|
||||
if message.get("status") not in {None, "OK", "PASS"}:
|
||||
screenshots = message.get("extra", {}).get("reftest_screenshots")
|
||||
if screenshots:
|
||||
yield message["test"], screenshots
|
||||
|
||||
|
||||
def main(index_key, commit_sha):
|
||||
failures_2013 = {url for url, _ in failing_reftests("linux_x64", index_key)}
|
||||
failures_2020 = Directory()
|
||||
for url, screenshots in failing_reftests("linux_x64_2020", index_key):
|
||||
if url not in failures_2013:
|
||||
assert url.startswith("/")
|
||||
failures_2020.add(url[1:], screenshots)
|
||||
|
||||
here = os.path.dirname(__file__)
|
||||
with open(os.path.join(here, "prism.js")) as f:
|
||||
prism_js = f.read()
|
||||
with open(os.path.join(here, "prism.css")) as f:
|
||||
prism_css = f.read()
|
||||
with open(os.path.join(here, "regressions.html"), "w", encoding="utf-8") as html:
|
||||
os.chdir(os.path.join(here, "../../tests/wpt"))
|
||||
html.write("""
|
||||
<!doctype html>
|
||||
<meta charset=utf-8>
|
||||
<title>Layout 2020 regressions</title>
|
||||
<link rel=stylesheet href=prism.css>
|
||||
<style>
|
||||
ul { padding-left: 1em }
|
||||
li { list-style: "⯈ " }
|
||||
li.expanded { list-style: "⯆ " }
|
||||
li:not(.expanded) > ul, li:not(.expanded) > div { display: none }
|
||||
li > div { display: grid; grid-gap: 1em; grid-template-columns: 1fr 1fr }
|
||||
li > div > p { grid-column: span 2 }
|
||||
li > div > img { grid-row: 2; width: 300px; box-shadow: 0 0 10px }
|
||||
li > div > img:hover { transform: scale(3); transform-origin: 0 0 }
|
||||
li > div > pre { grid-row: 3; font-size: 12px !important }
|
||||
pre code { white-space: pre-wrap !important }
|
||||
%s
|
||||
</style>
|
||||
<h1>Layout 2020 regressions in tree <code>%s</code></h1>
|
||||
""" % (prism_css, commit_sha))
|
||||
failures_2020.write(html)
|
||||
html.write("""
|
||||
<script>
|
||||
for (let li of document.getElementsByTagName("li")) {
|
||||
li.addEventListener('click', event => {
|
||||
li.classList.toggle("expanded")
|
||||
event.stopPropagation()
|
||||
})
|
||||
}
|
||||
%s
|
||||
</script>
|
||||
""" % prism_js)
|
||||
|
||||
|
||||
class Directory:
|
||||
def __init__(self):
|
||||
self.count = 0
|
||||
self.contents = {}
|
||||
|
||||
def add(self, path, screenshots):
|
||||
self.count += 1
|
||||
first, _, rest = path.partition("/")
|
||||
if rest:
|
||||
self.contents.setdefault(first, Directory()).add(rest, screenshots)
|
||||
else:
|
||||
assert path not in self.contents
|
||||
self.contents[path] = screenshots
|
||||
|
||||
def write(self, html):
|
||||
html.write("<ul>\n")
|
||||
for k, v in self.contents.items():
|
||||
html.write("<li><code>%s</code>\n" % k)
|
||||
if isinstance(v, Directory):
|
||||
html.write("<strong>%s</strong>\n" % v.count)
|
||||
v.write(html)
|
||||
else:
|
||||
a, rel, b = v
|
||||
html.write("<div>\n<p><code>%s</code> %s <code>%s</code></p>\n"
|
||||
% (a["url"], rel, b["url"]))
|
||||
for side in [a, b]:
|
||||
html.write("<img src='data:image/png;base64,%s'>\n" % side["screenshot"])
|
||||
url = side["url"]
|
||||
prefix = "/_mozilla/"
|
||||
if url.startswith(prefix):
|
||||
filename = "mozilla/tests/" + url[len(prefix):]
|
||||
else:
|
||||
filename = "web-platform-tests" + url
|
||||
with open(filename, encoding="utf-8") as f:
|
||||
src = html_escape(f.read())
|
||||
html.write("<pre><code class=language-html>%s</code></pre>\n" % src)
|
||||
html.write("</li>\n")
|
||||
html.write("</ul>\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(*sys.argv[1:]))
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
/* PrismJS 1.19.0
|
||||
https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */
|
||||
/**
|
||||
* prism.js default theme for JavaScript, CSS and HTML
|
||||
* Based on dabblet (http://dabblet.com)
|
||||
* @author Lea Verou
|
||||
*/
|
||||
|
||||
code[class*="language-"],
|
||||
pre[class*="language-"] {
|
||||
color: black;
|
||||
background: none;
|
||||
text-shadow: 0 1px white;
|
||||
font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
|
||||
font-size: 1em;
|
||||
text-align: left;
|
||||
white-space: pre;
|
||||
word-spacing: normal;
|
||||
word-break: normal;
|
||||
word-wrap: normal;
|
||||
line-height: 1.5;
|
||||
|
||||
-moz-tab-size: 4;
|
||||
-o-tab-size: 4;
|
||||
tab-size: 4;
|
||||
|
||||
-webkit-hyphens: none;
|
||||
-moz-hyphens: none;
|
||||
-ms-hyphens: none;
|
||||
hyphens: none;
|
||||
}
|
||||
|
||||
pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection,
|
||||
code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection {
|
||||
text-shadow: none;
|
||||
background: #b3d4fc;
|
||||
}
|
||||
|
||||
pre[class*="language-"]::selection, pre[class*="language-"] ::selection,
|
||||
code[class*="language-"]::selection, code[class*="language-"] ::selection {
|
||||
text-shadow: none;
|
||||
background: #b3d4fc;
|
||||
}
|
||||
|
||||
@media print {
|
||||
code[class*="language-"],
|
||||
pre[class*="language-"] {
|
||||
text-shadow: none;
|
||||
}
|
||||
}
|
||||
|
||||
/* Code blocks */
|
||||
pre[class*="language-"] {
|
||||
padding: 1em;
|
||||
margin: .5em 0;
|
||||
overflow: auto;
|
||||
}
|
||||
|
||||
:not(pre) > code[class*="language-"],
|
||||
pre[class*="language-"] {
|
||||
background: #f5f2f0;
|
||||
}
|
||||
|
||||
/* Inline code */
|
||||
:not(pre) > code[class*="language-"] {
|
||||
padding: .1em;
|
||||
border-radius: .3em;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.token.comment,
|
||||
.token.prolog,
|
||||
.token.doctype,
|
||||
.token.cdata {
|
||||
color: slategray;
|
||||
}
|
||||
|
||||
.token.punctuation {
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.token.namespace {
|
||||
opacity: .7;
|
||||
}
|
||||
|
||||
.token.property,
|
||||
.token.tag,
|
||||
.token.boolean,
|
||||
.token.number,
|
||||
.token.constant,
|
||||
.token.symbol,
|
||||
.token.deleted {
|
||||
color: #905;
|
||||
}
|
||||
|
||||
.token.selector,
|
||||
.token.attr-name,
|
||||
.token.string,
|
||||
.token.char,
|
||||
.token.builtin,
|
||||
.token.inserted {
|
||||
color: #690;
|
||||
}
|
||||
|
||||
.token.operator,
|
||||
.token.entity,
|
||||
.token.url,
|
||||
.language-css .token.string,
|
||||
.style .token.string {
|
||||
color: #9a6e3a;
|
||||
background: hsla(0, 0%, 100%, .5);
|
||||
}
|
||||
|
||||
.token.atrule,
|
||||
.token.attr-value,
|
||||
.token.keyword {
|
||||
color: #07a;
|
||||
}
|
||||
|
||||
.token.function,
|
||||
.token.class-name {
|
||||
color: #DD4A68;
|
||||
}
|
||||
|
||||
.token.regex,
|
||||
.token.important,
|
||||
.token.variable {
|
||||
color: #e90;
|
||||
}
|
||||
|
||||
.token.important,
|
||||
.token.bold {
|
||||
font-weight: bold;
|
||||
}
|
||||
.token.italic {
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.token.entity {
|
||||
cursor: help;
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -260,12 +260,13 @@ def layout_2020_regressions_report():
|
|||
.with_dockerfile(dockerfile_path("base"))
|
||||
.with_repo_bundle()
|
||||
.with_script(
|
||||
"python3 etc/layout-2020-regressions/gen.py %s %s"
|
||||
"python3 tests/wpt/reftests-report/gen.py %s %s"
|
||||
% (CONFIG.tree_hash(), CONFIG.git_sha)
|
||||
)
|
||||
.with_index_and_artifacts_expire_in(log_artifacts_expire_in)
|
||||
.with_artifacts("/repo/etc/layout-2020-regressions/regressions.html")
|
||||
.find_or_create("layout-2020-regressions-report")
|
||||
.with_artifacts("/repo/tests/wpt/reftests-report/report.html")
|
||||
.with_index_at("layout-2020-regressions-report")
|
||||
.create()
|
||||
)
|
||||
|
||||
def macos_unit():
|
||||
|
|
|
|||
|
|
@ -157,6 +157,10 @@ class Task:
|
|||
|
||||
with_extra = chaining(update_attr, "extra")
|
||||
|
||||
def with_index_at(self, index_path):
|
||||
self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path))
|
||||
return self
|
||||
|
||||
def with_treeherder_required(self):
|
||||
self.treeherder_required = True
|
||||
return self
|
||||
|
|
@ -291,7 +295,7 @@ class Task:
|
|||
if e.status_code != 404: # pragma: no cover
|
||||
raise
|
||||
if not CONFIG.index_read_only:
|
||||
self.routes.append("index.%s.%s" % (CONFIG.index_prefix, index_path))
|
||||
self.with_index_at(index_path)
|
||||
task_id = self.create()
|
||||
|
||||
SHARED.found_or_created_indexed_tasks[index_path] = task_id
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue