diff --git a/.gitignore b/.gitignore index 3f306a88921..60d0f3dd203 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /.vs /android-toolchains /target +/etc/layout-2020-regressions/regressions.html /ports/android/bin /ports/android/libs /ports/android/local.properties diff --git a/etc/layout-2020-regressions/gen.py b/etc/layout-2020-regressions/gen.py new file mode 100755 index 00000000000..fd01c4e6d46 --- /dev/null +++ b/etc/layout-2020-regressions/gen.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import json +import os +import re +import sys +import urllib.request +from html import escape as html_escape + + +TASKCLUSTER_ROOT_URL = "https://community-tc.services.mozilla.com" + + +def fetch(url): + url = TASKCLUSTER_ROOT_URL + "/api/" + url + print("Fetching " + url) + response = urllib.request.urlopen(url) + assert response.getcode() == 200 + return response + + +def fetch_json(url): + with fetch(url) as response: + return json.load(response) + + +def task(platform, chunk, key): + return "index/v1/task/project.servo.%s_wpt_%s.%s" % (platform, chunk, key) + + +def failing_reftests(platform, key): + chunk_1_task_id = fetch_json(task(platform, 1, key))["taskId"] + name = fetch_json("queue/v1/task/" + chunk_1_task_id)["metadata"]["name"] + match = re.search("WPT chunk (\d+) / (\d+)", name) + assert match.group(1) == "1" + total_chunks = int(match.group(2)) + + for chunk in range(1, total_chunks + 1): + with fetch(task(platform, chunk, key) + "/artifacts/public/test-wpt.log") as response: + for line in response: + message = json.loads(line) + if message.get("status") not in {None, "OK", "PASS"}: + screenshots = message.get("extra", {}).get("reftest_screenshots") + if screenshots: + yield message["test"], screenshots + + +def main(index_key, commit_sha): + failures_2013 = {url for url, _ in failing_reftests("linux_x64", index_key)} + failures_2020 = Directory() + for url, screenshots in failing_reftests("linux_x64_2020", index_key): + if url not in failures_2013: + assert url.startswith("/") + failures_2020.add(url[1:], screenshots) + + here = os.path.dirname(__file__) + with open(os.path.join(here, "prism.js")) as f: + prism_js = f.read() + with open(os.path.join(here, "prism.css")) as f: + prism_css = f.read() + with open(os.path.join(here, "regressions.html"), "w", encoding="utf-8") as html: + os.chdir(os.path.join(here, "../../tests/wpt")) + html.write(""" + + + Layout 2020 regressions + + +

Layout 2020 regressions in tree %s

+ """ % (prism_css, commit_sha)) + failures_2020.write(html) + html.write(""" + + """ % prism_js) + + +class Directory: + def __init__(self): + self.count = 0 + self.contents = {} + + def add(self, path, screenshots): + self.count += 1 + first, _, rest = path.partition("/") + if rest: + self.contents.setdefault(first, Directory()).add(rest, screenshots) + else: + assert path not in self.contents + self.contents[path] = screenshots + + def write(self, html): + html.write("\n") + + +if __name__ == "__main__": + sys.exit(main(*sys.argv[1:])) diff --git a/etc/layout-2020-regressions/prism.css b/etc/layout-2020-regressions/prism.css new file mode 100644 index 00000000000..6fa6fcc5be6 --- /dev/null +++ b/etc/layout-2020-regressions/prism.css @@ -0,0 +1,141 @@ +/* PrismJS 1.19.0 +https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ +/** + * prism.js default theme for JavaScript, CSS and HTML + * Based on dabblet (http://dabblet.com) + * @author Lea Verou + */ + +code[class*="language-"], +pre[class*="language-"] { + color: black; + background: none; + text-shadow: 0 1px white; + font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace; + font-size: 1em; + text-align: left; + white-space: pre; + word-spacing: normal; + word-break: normal; + word-wrap: normal; + line-height: 1.5; + + -moz-tab-size: 4; + -o-tab-size: 4; + tab-size: 4; + + -webkit-hyphens: none; + -moz-hyphens: none; + -ms-hyphens: none; + hyphens: none; +} + +pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection, +code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection { + text-shadow: none; + background: #b3d4fc; +} + +pre[class*="language-"]::selection, pre[class*="language-"] ::selection, +code[class*="language-"]::selection, code[class*="language-"] ::selection { + text-shadow: none; + background: #b3d4fc; +} + +@media print { + code[class*="language-"], + pre[class*="language-"] { + text-shadow: none; + } +} + +/* Code blocks */ +pre[class*="language-"] { + padding: 1em; + margin: .5em 0; + overflow: auto; +} + +:not(pre) > code[class*="language-"], +pre[class*="language-"] { + background: #f5f2f0; +} + +/* Inline code */ +:not(pre) > code[class*="language-"] { + padding: .1em; + border-radius: .3em; + white-space: normal; +} + +.token.comment, +.token.prolog, +.token.doctype, +.token.cdata { + color: slategray; +} + +.token.punctuation { + color: #999; +} + +.token.namespace { + opacity: .7; +} + +.token.property, +.token.tag, +.token.boolean, +.token.number, +.token.constant, +.token.symbol, +.token.deleted { + color: #905; +} + +.token.selector, +.token.attr-name, +.token.string, +.token.char, +.token.builtin, +.token.inserted { + color: #690; +} + +.token.operator, +.token.entity, +.token.url, +.language-css .token.string, +.style .token.string { + color: #9a6e3a; + background: hsla(0, 0%, 100%, .5); +} + +.token.atrule, +.token.attr-value, +.token.keyword { + color: #07a; +} + +.token.function, +.token.class-name { + color: #DD4A68; +} + +.token.regex, +.token.important, +.token.variable { + color: #e90; +} + +.token.important, +.token.bold { + font-weight: bold; +} +.token.italic { + font-style: italic; +} + +.token.entity { + cursor: help; +} diff --git a/etc/layout-2020-regressions/prism.js b/etc/layout-2020-regressions/prism.js new file mode 100644 index 00000000000..a43970450d6 --- /dev/null +++ b/etc/layout-2020-regressions/prism.js @@ -0,0 +1,7 @@ +/* PrismJS 1.19.0 +https://prismjs.com/download.html#themes=prism&languages=markup+css+clike+javascript */ +var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(u){var c=/\blang(?:uage)?-([\w-]+)\b/i,r=0,C={manual:u.Prism&&u.Prism.manual,disableWorkerMessageHandler:u.Prism&&u.Prism.disableWorkerMessageHandler,util:{encode:function e(r){return r instanceof _?new _(r.type,e(r.content),r.alias):Array.isArray(r)?r.map(e):r.replace(/&/g,"&").replace(/e.length)return;if(!(k instanceof _)){if(h&&y!=r.length-1){if(c.lastIndex=v,!(S=c.exec(e)))break;for(var b=S.index+(f&&S[1]?S[1].length:0),w=S.index+S[0].length,A=y,P=v,x=r.length;A"+a.content+""},!u.document)return u.addEventListener&&(C.disableWorkerMessageHandler||u.addEventListener("message",function(e){var r=JSON.parse(e.data),n=r.language,t=r.code,a=r.immediateClose;u.postMessage(C.highlight(t,C.languages[n],n)),a&&u.close()},!1)),C;var e=C.util.currentScript();function n(){C.manual||C.highlightAll()}if(e&&(C.filename=e.src,e.hasAttribute("data-manual")&&(C.manual=!0)),!C.manual){var t=document.readyState;"loading"===t||"interactive"===t&&e&&e.defer?document.addEventListener("DOMContentLoaded",n):window.requestAnimationFrame?window.requestAnimationFrame(n):window.setTimeout(n,16)}return C}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism); +Prism.languages.markup={comment://,prolog:/<\?[\s\S]+?\?>/,doctype:{pattern:/"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:(?!)*\]\s*)?>/i,greedy:!0},cdata://i,tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/i,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/i,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/i,inside:{punctuation:[/^=/,{pattern:/^(\s*)["']|["']$/,lookbehind:!0}]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:/&#?[\da-z]{1,8};/i},Prism.languages.markup.tag.inside["attr-value"].inside.entity=Prism.languages.markup.entity,Prism.hooks.add("wrap",function(a){"entity"===a.type&&(a.attributes.title=a.content.replace(/&/,"&"))}),Object.defineProperty(Prism.languages.markup.tag,"addInlined",{value:function(a,e){var s={};s["language-"+e]={pattern:/(^$)/i,lookbehind:!0,inside:Prism.languages[e]},s.cdata=/^$/i;var n={"included-cdata":{pattern://i,inside:s}};n["language-"+e]={pattern:/[\s\S]+/,inside:Prism.languages[e]};var t={};t[a]={pattern:RegExp("(<__[\\s\\S]*?>)(?:\\s*|[\\s\\S])*?(?=<\\/__>)".replace(/__/g,a),"i"),lookbehind:!0,greedy:!0,inside:n},Prism.languages.insertBefore("markup","cdata",t)}}),Prism.languages.xml=Prism.languages.extend("markup",{}),Prism.languages.html=Prism.languages.markup,Prism.languages.mathml=Prism.languages.markup,Prism.languages.svg=Prism.languages.markup; +!function(s){var e=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;s.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-]+[\s\S]*?(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\((?!\s*\))\s*)(?:[^()]|\((?:[^()]|\([^()]*\))*\))+?(?=\s*\))/,lookbehind:!0,alias:"selector"}}},url:{pattern:RegExp("url\\((?:"+e.source+"|[^\n\r()]*)\\)","i"),inside:{function:/^url/i,punctuation:/^\(|\)$/}},selector:RegExp("[^{}\\s](?:[^{};\"']|"+e.source+")*?(?=\\s*\\{)"),string:{pattern:e,greedy:!0},property:/[-_a-z\xA0-\uFFFF][-\w\xA0-\uFFFF]*(?=\s*:)/i,important:/!important\b/i,function:/[-a-z0-9]+(?=\()/i,punctuation:/[(){};:,]/},s.languages.css.atrule.inside.rest=s.languages.css;var t=s.languages.markup;t&&(t.tag.addInlined("style","css"),s.languages.insertBefore("inside","attr-value",{"style-attr":{pattern:/\s*style=("|')(?:\\[\s\S]|(?!\1)[^\\])*\1/i,inside:{"attr-name":{pattern:/^\s*style/i,inside:t.tag.inside},punctuation:/^\s*=\s*['"]|['"]\s*$/,"attr-value":{pattern:/.+/i,inside:s.languages.css}},alias:"language-css"}},t.tag))}(Prism); +Prism.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|interface|extends|implements|trait|instanceof|new)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:if|else|while|do|for|return|in|instanceof|function|new|try|throw|catch|finally|null|break|continue)\b/,boolean:/\b(?:true|false)\b/,function:/\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+\.?\d*|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/}; +Prism.languages.javascript=Prism.languages.extend("clike",{"class-name":[Prism.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])[_$A-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\.(?:prototype|constructor))/,lookbehind:!0}],keyword:[{pattern:/((?:^|})\s*)(?:catch|finally)\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],number:/\b(?:(?:0[xX](?:[\dA-Fa-f](?:_[\dA-Fa-f])?)+|0[bB](?:[01](?:_[01])?)+|0[oO](?:[0-7](?:_[0-7])?)+)n?|(?:\d(?:_\d)?)+n|NaN|Infinity)\b|(?:\b(?:\d(?:_\d)?)+\.?(?:\d(?:_\d)?)*|\B\.(?:\d(?:_\d)?)+)(?:[Ee][+-]?(?:\d(?:_\d)?)+)?/,function:/#?[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,operator:/--|\+\+|\*\*=?|=>|&&|\|\||[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?[.?]?|[~:]/}),Prism.languages.javascript["class-name"][0].pattern=/(\b(?:class|interface|extends|implements|instanceof|new)\s+)[\w.\\]+/,Prism.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s])\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*]|\\.|[^/\\\[\r\n])+\/[gimyus]{0,6}(?=(?:\s|\/\*[\s\S]*?\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0},"function-variable":{pattern:/#?[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|[_$a-zA-Z\xA0-\uFFFF][$\w\xA0-\uFFFF]*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+[_$A-Za-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*)?\s*\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\))/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/[_$a-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*(?=\s*=>)/i,inside:Prism.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\)\s*=>)/,lookbehind:!0,inside:Prism.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:[_$A-Za-z\xA0-\uFFFF][$\w\xA0-\uFFFF]*\s*)\(\s*)(?!\s)(?:[^()]|\([^()]*\))+?(?=\s*\)\s*\{)/,lookbehind:!0,inside:Prism.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),Prism.languages.insertBefore("javascript","string",{"template-string":{pattern:/`(?:\\[\s\S]|\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}|(?!\${)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\${(?:[^{}]|{(?:[^{}]|{[^}]*})*})+}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\${|}$/,alias:"punctuation"},rest:Prism.languages.javascript}},string:/[\s\S]+/}}}),Prism.languages.markup&&Prism.languages.markup.tag.addInlined("script","javascript"),Prism.languages.js=Prism.languages.javascript; diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 951cebe7afa..29d69034e00 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -48,6 +48,7 @@ def tasks(task_for): ], "master": [ upload_docs, + layout_2020_regressions_report, ], # The "try-*" keys match those in `servo_try_choosers` in Homu’s config: @@ -247,6 +248,21 @@ def upload_docs(): ) +def layout_2020_regressions_report(): + return ( + linux_task("Layout 2020 regressions report") + .with_treeherder("Linux x64", "RegressionsReport") + .with_dockerfile(dockerfile_path("base")) + .with_repo_bundle() + .with_script( + "python3 etc/layout-2020-regressions/gen.py %s %s" + % (CONFIG.tree_hash(), CONFIG.git_sha) + ) + .with_index_and_artifacts_expire_in(log_artifacts_expire_in) + .with_artifacts("/repo/etc/layout-2020-regressions/regressions.html") + .find_or_create("layout-2020-regressions-report") + ) + def macos_unit(): return ( macos_build_task("Dev build + unit tests")