diff --git a/tests/wpt/metadata-layout-2020/css/CSS2/floats/hit-test-floats-002.html.ini b/tests/wpt/metadata-layout-2020/css/CSS2/floats/hit-test-floats-002.html.ini deleted file mode 100644 index f64b45fea6b..00000000000 --- a/tests/wpt/metadata-layout-2020/css/CSS2/floats/hit-test-floats-002.html.ini +++ /dev/null @@ -1,4 +0,0 @@ -[hit-test-floats-002.html] - [Hit test float] - expected: FAIL - diff --git a/tests/wpt/metadata-layout-2020/css/CSS2/floats/hit-test-floats-004.html.ini b/tests/wpt/metadata-layout-2020/css/CSS2/floats/hit-test-floats-004.html.ini deleted file mode 100644 index 4bfb0c2053a..00000000000 --- a/tests/wpt/metadata-layout-2020/css/CSS2/floats/hit-test-floats-004.html.ini +++ /dev/null @@ -1,4 +0,0 @@ -[hit-test-floats-004.html] - [Miss float below something else] - expected: FAIL - diff --git a/tests/wpt/metadata-layout-2020/fetch/content-type/response.window.js.ini b/tests/wpt/metadata-layout-2020/fetch/content-type/response.window.js.ini index 56752f6df92..7679f3378c0 100644 --- a/tests/wpt/metadata-layout-2020/fetch/content-type/response.window.js.ini +++ b/tests/wpt/metadata-layout-2020/fetch/content-type/response.window.js.ini @@ -324,15 +324,9 @@ [ + + + diff --git a/tests/wpt/web-platform-tests/content-security-policy/reporting/support/not-embeddable-frame.html b/tests/wpt/web-platform-tests/content-security-policy/reporting/support/not-embeddable-frame.html new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/wpt/web-platform-tests/content-security-policy/reporting/support/not-embeddable-frame.html.sub.headers b/tests/wpt/web-platform-tests/content-security-policy/reporting/support/not-embeddable-frame.html.sub.headers new file mode 100644 index 00000000000..beecdb765cd --- /dev/null +++ b/tests/wpt/web-platform-tests/content-security-policy/reporting/support/not-embeddable-frame.html.sub.headers @@ -0,0 +1,5 @@ +Expires: Mon, 26 Jul 1997 05:00:00 GMT +Cache-Control: no-store, no-cache, must-revalidate +Cache-Control: post-check=0, pre-check=0, false +Pragma: no-cache +Content-Security-Policy: frame-ancestors 'none'; report-uri ../../support/report.py?op=put&reportID={{GET[reportID]}} \ No newline at end of file diff --git a/tests/wpt/web-platform-tests/content-security-policy/sandbox/meta-element.sub.html b/tests/wpt/web-platform-tests/content-security-policy/sandbox/meta-element.sub.html new file mode 100644 index 00000000000..cd8da8f14c4 --- /dev/null +++ b/tests/wpt/web-platform-tests/content-security-policy/sandbox/meta-element.sub.html @@ -0,0 +1,46 @@ + + + + + + + + + diff --git a/tests/wpt/web-platform-tests/content-security-policy/sandbox/support/post-origin-on-load-worker.js b/tests/wpt/web-platform-tests/content-security-policy/sandbox/support/post-origin-on-load-worker.js new file mode 100644 index 00000000000..21ce5748ab8 --- /dev/null +++ b/tests/wpt/web-platform-tests/content-security-policy/sandbox/support/post-origin-on-load-worker.js @@ -0,0 +1 @@ +postMessage(self.origin); diff --git a/tests/wpt/web-platform-tests/docs/running-tests/from-local-system.md b/tests/wpt/web-platform-tests/docs/running-tests/from-local-system.md index 8c71e535baa..91823444d5d 100644 --- a/tests/wpt/web-platform-tests/docs/running-tests/from-local-system.md +++ b/tests/wpt/web-platform-tests/docs/running-tests/from-local-system.md @@ -39,9 +39,6 @@ wherever you currently set your PATH. See also [additional setup required to run Safari](safari.md). ### Windows Setup -**Note:** In general, Windows Subsystem for Linux will provide the smoothest -user experience for running web-platform-tests on Windows, where installation -and usage are similar to Linux. Download and install [Python 2.7](https://www.python.org/downloads). The installer includes `pip` by default. @@ -63,6 +60,15 @@ started using: python wpt serve ``` +#### Windows Subsystem for Linux + +Optionally on Windows you can use the [Windows Subsystem for +Linux](https://docs.microsoft.com/en-us/windows/wsl/about) (WSL). If doing so, +installation and usage are similar to the Linux instructions. Be aware that WSL +may attempt to override `/etc/hosts` each time it is launched, which would then +require you to re-run [`hosts` File Setup](#hosts-file-setup). This behavior +[can be configured](https://docs.microsoft.com/en-us/windows/wsl/wsl-config#network). + ### `hosts` File Setup To get the tests running, you need to set up the test domains in your diff --git a/tests/wpt/web-platform-tests/html/cross-origin-opener-policy/reporting/navigation-reporting/reporting-redirect-with-same-origin-allow-popups.https.html b/tests/wpt/web-platform-tests/html/cross-origin-opener-policy/reporting/navigation-reporting/reporting-redirect-with-same-origin-allow-popups.https.html index 7dba76c4ef5..bb76df811da 100644 --- a/tests/wpt/web-platform-tests/html/cross-origin-opener-policy/reporting/navigation-reporting/reporting-redirect-with-same-origin-allow-popups.https.html +++ b/tests/wpt/web-platform-tests/html/cross-origin-opener-policy/reporting/navigation-reporting/reporting-redirect-with-same-origin-allow-popups.https.html @@ -8,7 +8,6 @@ - + + diff --git a/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/base-url-worker.sub.html b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/base-url-worker.sub.html new file mode 100644 index 00000000000..a12204281cc --- /dev/null +++ b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/base-url-worker.sub.html @@ -0,0 +1,8 @@ + +Base URLs used in resolving specifiers in dynamic imports from workers + + + diff --git a/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/base-url.sub.html b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/base-url.sub.html new file mode 100644 index 00000000000..f7d4927a104 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/base-url.sub.html @@ -0,0 +1,28 @@ + +Base URLs used in resolving specifiers in dynamic imports + + + + + + + + + + + + + + diff --git a/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/import.js b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/import.js new file mode 100644 index 00000000000..b2ac52df2a1 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/import.js @@ -0,0 +1 @@ +export const A = { "from": "alpha/import.js" }; diff --git a/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/worker-importScripts.sub.js b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/worker-importScripts.sub.js new file mode 100644 index 00000000000..904d32f9bfd --- /dev/null +++ b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/alpha/worker-importScripts.sub.js @@ -0,0 +1,15 @@ +"use strict"; + +importScripts("/resources/testharness.js"); + +// CORS-same-origin +self.testName = "same-origin importScripts()"; +self.baseUrlSanitized = false; +importScripts("../beta/redirect.py?location=http://{{host}}:{{ports[http][0]}}/html/semantics/scripting-1/the-script-element/module/dynamic-import/gamma/base-url.sub.js"); + +// CORS-cross-origin +self.testName = "cross-origin importScripts()"; +self.baseUrlSanitized = true; +importScripts("../beta/redirect.py?location=http://{{hosts[alt][]}}:{{ports[http][0]}}/html/semantics/scripting-1/the-script-element/module/dynamic-import/gamma/base-url.sub.js"); + +done(); diff --git a/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/beta/import.js b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/beta/import.js new file mode 100644 index 00000000000..7de1c681825 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/beta/import.js @@ -0,0 +1 @@ +export const A = { "from": "beta/import.js" }; diff --git a/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/beta/redirect.py b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/beta/redirect.py new file mode 100644 index 00000000000..f2fd1ebd51d --- /dev/null +++ b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/beta/redirect.py @@ -0,0 +1,19 @@ +def main(request, response): + """Simple handler that causes redirection. + + The request should typically have two query parameters: + status - The status to use for the redirection. Defaults to 302. + location - The resource to redirect to. + """ + status = 302 + if b"status" in request.GET: + try: + status = int(request.GET.first(b"status")) + except ValueError: + pass + + response.status = status + + location = request.GET.first(b"location") + + response.headers.set(b"Location", location) diff --git a/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/gamma/base-url.sub.js b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/gamma/base-url.sub.js new file mode 100644 index 00000000000..ec7784983d1 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/semantics/scripting-1/the-script-element/module/dynamic-import/gamma/base-url.sub.js @@ -0,0 +1,58 @@ +"use strict"; + +// This script triggers import(), and thus the base URL of this script +// (either loaded by ` + + + + +
    +

  1. First, register the handler: . +

  2. Then, run the test: . +

  3. Or, run the test with U+0000 NULL: . +

+
diff --git a/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-fragment-nosw-manual.https.html b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-fragment-nosw-manual.https.html new file mode 100644 index 00000000000..be3a6be6665 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-fragment-nosw-manual.https.html @@ -0,0 +1,20 @@ + + + + +registerProtocolHandler() and a handler with %s in the fragment (does not use a service worker) + + + + + +
    +

  1. First, register the handler: . +

  2. Then, run the test: . +

  3. Or, run the test with U+0000 NULL: . +

+
diff --git a/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-path-manual.https.html b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-path-manual.https.html new file mode 100644 index 00000000000..085c5723ec4 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-path-manual.https.html @@ -0,0 +1,20 @@ + + + + +registerProtocolHandler() and a handler with %s in the path + + + + + +
    +

  1. First, register the handler: . +

  2. Then, run the test: . +

  3. Or, run the test with U+0000 NULL: . +

+
diff --git a/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-query-manual.https.html b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-query-manual.https.html new file mode 100644 index 00000000000..8ce65a5bad8 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-query-manual.https.html @@ -0,0 +1,20 @@ + + + + +registerProtocolHandler() and a handler with %s in the query + + + + + +
    +

  1. First, register the handler: . +

  2. Then, run the test: . +

  3. Or, run the test with U+0000 NULL: . +

+
diff --git a/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-query-nosw-manual.https.html b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-query-nosw-manual.https.html new file mode 100644 index 00000000000..9b4473fb895 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/protocol-handler-query-nosw-manual.https.html @@ -0,0 +1,20 @@ + + + + +registerProtocolHandler() and a handler with %s in the query (does not use a service worker) + + + + + +
    +

  1. First, register the handler: . +

  2. Then, run the test: . +

  3. Or, run the test with U+0000 NULL: . +

+
diff --git a/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler-sw.js b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler-sw.js new file mode 100644 index 00000000000..5fd915d17f9 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler-sw.js @@ -0,0 +1,3 @@ +onfetch = e => { + e.respondWith(fetch("handler.html")); +} diff --git a/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler-tools.js b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler-tools.js new file mode 100644 index 00000000000..073287265cf --- /dev/null +++ b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler-tools.js @@ -0,0 +1,53 @@ +// These can be used in an environment that has these global variables defined: +// * type (one of "path", "query", or "fragment") +// * noSW (a boolean) + +if (type === "path" && noSW) { + throw new Error("There is no support for a path handler without a service worker."); +} + +const swString = noSW ? "" : "sw"; +const handler = { + "path": "PSS%sPSE/?QES\u2020QEE#FES\u2020FEE", + "query": "?QES\u2020QEEPSS%sPSE#FES\u2020FEE", + "fragment": "?QES\u2020QEE#FES\u2020FEEPSS%sPSE" +}[type]; +const scheme = `web+wpt${type}${swString}`; + +function register() { + const handlerURL = noSW ? `resources/handler.html${handler}${type}` : `resources/handler/${type}/${handler}`; + navigator.registerProtocolHandler(scheme, handlerURL, `WPT ${type} handler${noSW ? ", without service worker" : ""}`); +} + +function runTest({ includeNull = false } = {}) { + promise_test(async t => { + const bc = new BroadcastChannel(`protocol-handler-${type}${swString}`); + if (!noSW) { + const reg = await service_worker_unregister_and_register(t, "resources/handler-sw.js", "resources/handler/"); + t.add_cleanup(async () => await reg.unregister()); + await wait_for_state(t, reg.installing, 'activated'); + } + const a = document.body.appendChild(document.createElement("a")); + const codePoints = []; + let i = includeNull ? 0 : 1; + for (; i < 0x82; i++) { + codePoints.push(String.fromCharCode(i)); + } + a.href = `${scheme}:${codePoints.join("")}`; + a.target = "_blank"; + a.click(); + await new Promise(resolve => { + bc.onmessage = t.step_func(e => { + resultingURL = e.data; + assert_equals(stringBetweenMarkers(resultingURL, "QES", "QEE"), "%86", "query baseline"); + assert_equals(stringBetweenMarkers(resultingURL, "FES", "FEE"), "%E2%80%A0", "fragment baseline"); + assert_equals(stringBetweenMarkers(resultingURL, "PSS", "PSE"), `${encodeURIComponent(scheme)}%3A${includeNull ? "%2500" : ""}%2501%2502%2503%2504%2505%2506%2507%2508%250B%250C%250E%250F%2510%2511%2512%2513%2514%2515%2516%2517%2518%2519%251A%251B%251C%251D%251E%251F%20!%22%23%24%25%26${type === "query" ? "%27" : "'"}()*%2B%2C-.%2F0123456789%3A%3B%3C%3D%3E%3F%40ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%257F%25C2%2580%25C2%2581`, "actual test"); + resolve(); + }); + }); + }); +} + +function stringBetweenMarkers(string, start, end) { + return string.substring(string.indexOf(start) + start.length, string.indexOf(end)); +} diff --git a/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler.html b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler.html new file mode 100644 index 00000000000..552e5417842 --- /dev/null +++ b/tests/wpt/web-platform-tests/html/webappapis/system-state-and-capabilities/the-navigator-object/resources/handler.html @@ -0,0 +1,23 @@ + +

This popup can be closed if it does not close itself. +

+ diff --git a/tests/wpt/web-platform-tests/input-events/input-events-get-target-ranges-backspace.tentative.html b/tests/wpt/web-platform-tests/input-events/input-events-get-target-ranges-backspace.tentative.html new file mode 100644 index 00000000000..cf512269b07 --- /dev/null +++ b/tests/wpt/web-platform-tests/input-events/input-events-get-target-ranges-backspace.tentative.html @@ -0,0 +1,811 @@ + + +InputEvent.getTargetRanges() at Backspace + + + + + +

+ diff --git a/tests/wpt/web-platform-tests/input-events/input-events-get-target-ranges-forwarddelete.tentative.html b/tests/wpt/web-platform-tests/input-events/input-events-get-target-ranges-forwarddelete.tentative.html new file mode 100644 index 00000000000..3780324cf92 --- /dev/null +++ b/tests/wpt/web-platform-tests/input-events/input-events-get-target-ranges-forwarddelete.tentative.html @@ -0,0 +1,808 @@ + + +InputEvent.getTargetRanges() at Delete (forward delete) + + + + + +
+ diff --git a/tests/wpt/web-platform-tests/interfaces/web-share.idl b/tests/wpt/web-platform-tests/interfaces/web-share.idl index c29a29d0b4e..b66bcd81fe9 100644 --- a/tests/wpt/web-platform-tests/interfaces/web-share.idl +++ b/tests/wpt/web-platform-tests/interfaces/web-share.idl @@ -8,7 +8,7 @@ partial interface Navigator { }; dictionary ShareData { - FrozenArray files; + sequence files; USVString title; USVString text; USVString url; diff --git a/tests/wpt/web-platform-tests/largest-contentful-paint/invisible-images-composited-1.html b/tests/wpt/web-platform-tests/largest-contentful-paint/invisible-images-composited-1.html index 495645ab41e..7723d2f2bea 100644 --- a/tests/wpt/web-platform-tests/largest-contentful-paint/invisible-images-composited-1.html +++ b/tests/wpt/web-platform-tests/largest-contentful-paint/invisible-images-composited-1.html @@ -13,13 +13,17 @@ .displayNone { display: none; } - .composited { + .willChangeTransform { will-change: transform; } + .willChangeOpacity { + will-change: opacity; + } - - - + + + +
diff --git a/tests/wpt/web-platform-tests/layout-instability/inline-flow-shift-vertical-rl.html b/tests/wpt/web-platform-tests/layout-instability/inline-flow-shift-vertical-rl.html deleted file mode 100644 index 06bc34c37dd..00000000000 --- a/tests/wpt/web-platform-tests/layout-instability/inline-flow-shift-vertical-rl.html +++ /dev/null @@ -1,44 +0,0 @@ - -Layout Instability: simple block movement is detected - - - - - -
- 1AAAAAAA
- 2AAAAAAA
- 3AAAAAAA
-
4AAAAAAA

- 5AAAAAAA
- 6AAAAAAA
- 7AAAAAAA
-
- - diff --git a/tests/wpt/web-platform-tests/layout-instability/inline-flow-shift.html b/tests/wpt/web-platform-tests/layout-instability/inline-flow-shift.html deleted file mode 100644 index 39550da6588..00000000000 --- a/tests/wpt/web-platform-tests/layout-instability/inline-flow-shift.html +++ /dev/null @@ -1,42 +0,0 @@ - -Layout Instability: simple block movement is detected - - - - -
- 1AAAAAAA
- 2AAAAAAA
- 3AAAAAAA
-
4AAAAAAA

- 5AAAAAAA
- 6AAAAAAA
- 7AAAAAAA
-
- diff --git a/tests/wpt/web-platform-tests/layout-instability/outline.html b/tests/wpt/web-platform-tests/layout-instability/outline.html deleted file mode 100644 index 1fed8e92f5e..00000000000 --- a/tests/wpt/web-platform-tests/layout-instability/outline.html +++ /dev/null @@ -1,21 +0,0 @@ - -Layout Instability: outline doesn't contribute to layout shift - - - - -
- diff --git a/tests/wpt/web-platform-tests/tools/ci/taskcluster-run.py b/tests/wpt/web-platform-tests/tools/ci/taskcluster-run.py index 61d05689287..d228e21990a 100755 --- a/tests/wpt/web-platform-tests/tools/ci/taskcluster-run.py +++ b/tests/wpt/web-platform-tests/tools/ci/taskcluster-run.py @@ -84,6 +84,9 @@ def main(product, commit_range, wpt_args): command = ["python", "./wpt", "run"] + wpt_args + [product] logger.info("Executing command: %s" % " ".join(command)) + with open("/home/test/artifacts/checkrun.md", "a") as f: + f.write("\n**WPT Command:** `%s`\n\n" % " ".join(command)) + retcode = subprocess.call(command, env=dict(os.environ, TERM="dumb")) if retcode != 0: sys.exit(retcode) diff --git a/tests/wpt/web-platform-tests/tools/ci/tc/decision.py b/tests/wpt/web-platform-tests/tools/ci/tc/decision.py index 0820a6798b5..f52f5b093f0 100644 --- a/tests/wpt/web-platform-tests/tools/ci/tc/decision.py +++ b/tests/wpt/web-platform-tests/tools/ci/tc/decision.py @@ -241,6 +241,8 @@ def create_tc_task(event, task, taskgroup_id, depends_on_ids, env_extra=None): }, "routes": ["checks"] } + if "extra" in task: + task_data["extra"].update(task["extra"]) if env_extra: task_data["payload"]["env"].update(env_extra) if depends_on_ids: diff --git a/tests/wpt/web-platform-tests/tools/ci/tc/github_checks_output.py b/tests/wpt/web-platform-tests/tools/ci/tc/github_checks_output.py new file mode 100644 index 00000000000..d799be91166 --- /dev/null +++ b/tests/wpt/web-platform-tests/tools/ci/tc/github_checks_output.py @@ -0,0 +1,29 @@ +class GitHubChecksOutputter(object): + """Provides a method to output data to be shown in the GitHub Checks UI. + + This can be useful to provide a summary of a given check (e.g. the lint) + to enable developers to quickly understand what has gone wrong. The output + supports markdown format. + + See https://docs.taskcluster.net/docs/reference/integrations/github/checks#custom-text-output-in-checks + """ + def __init__(self, path): + self.path = path + + def output(self, line): + with open(self.path, 'a') as f: + f.write(line) + f.write('\n') + + +__outputter = None +def get_gh_checks_outputter(kwargs): + """Return the outputter for GitHub Checks output, if enabled. + + :param kwargs: The arguments passed to the program (to look for the + --github_checks_text_file flag) + """ + global __outputter + if kwargs['github_checks_text_file'] and __outputter is None: + __outputter = GitHubChecksOutputter(kwargs['github_checks_text_file']) + return __outputter diff --git a/tests/wpt/web-platform-tests/tools/ci/tc/tasks/test.yml b/tests/wpt/web-platform-tests/tools/ci/tc/tasks/test.yml index dd14fc71b91..2fc11e0b27b 100644 --- a/tests/wpt/web-platform-tests/tools/ci/tc/tasks/test.yml +++ b/tests/wpt/web-platform-tests/tools/ci/tc/tasks/test.yml @@ -10,6 +10,13 @@ components: public/results: path: /home/test/artifacts type: directory + extra: + github: + customCheckRun: + # We set both textArtifactName and annotationsArtifactName due + # to https://github.com/taskcluster/taskcluster/issues/3191 + textArtifactName: public/results/checkrun.md + annotationsArtifactName: public/results/checkrun.md wpt-testharness: chunks: 16 @@ -296,6 +303,7 @@ tasks: --channel=${vars.channel} --verify --verify-no-chaos-mode + --github-checks-text-file="/home/test/artifacts/checkrun.md" - wpt-${vars.browser}-${vars.channel}-results: use: diff --git a/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/stability.py b/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/stability.py index 3f8989729fd..605b40281a1 100644 --- a/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/stability.py +++ b/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/stability.py @@ -13,6 +13,7 @@ from mozlog.handlers import BaseHandler, StreamHandler, LogLevelFilter here = os.path.dirname(__file__) localpaths = imp.load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, os.pardir, "localpaths.py"))) +from ci.tc.github_checks_output import get_gh_checks_outputter from wpt.markdown import markdown_adjust, table @@ -178,8 +179,32 @@ def err_string(results_dict, iterations): return rv +def write_github_checks_summary_inconsistent(log, inconsistent, iterations): + """Outputs a summary of inconsistent tests for GitHub Checks.""" + log("Some affected tests had inconsistent (flaky) results:\n") + write_inconsistent(log, inconsistent, iterations) + log("\n") + log("These may be pre-existing or new flakes. Please try to reproduce (see " + "the above WPT command, though some flags may not be needed when " + "running locally) and determine if your change introduced the flake. " + "If you are unable to reproduce the problem, please tag " + "`@web-platform-tests/wpt-core-team` in a comment for help.\n") + + +def write_github_checks_summary_slow_tests(log, slow): + """Outputs a summary of slow tests for GitHub Checks.""" + log("Some affected tests had slow results:\n") + write_slow_tests(log, slow) + log("\n") + log("These may be pre-existing or newly slow tests. Slow tests indicate " + "that a test ran very close to the test timeout limit and so may " + "become TIMEOUT-flaky in the future. Consider speeding up the test or " + "breaking it into multiple tests. For help, please tag " + "`@web-platform-tests/wpt-core-team` in a comment.\n") + + def write_inconsistent(log, inconsistent, iterations): - """Output inconsistent tests to logger.error.""" + """Output inconsistent tests to the passed in logging function.""" log("## Unstable results ##\n") strings = [( "`%s`" % markdown_adjust(test), @@ -191,6 +216,7 @@ def write_inconsistent(log, inconsistent, iterations): def write_slow_tests(log, slow): + """Output slow tests to the passed in logging function.""" log("## Slow tests ##\n") strings = [( "`%s`" % markdown_adjust(test), @@ -321,6 +347,8 @@ def check_stability(logger, repeat_loop=10, repeat_restart=5, chaos_mode=True, m start_time = datetime.now() step_results = [] + github_checks_outputter = get_gh_checks_outputter(kwargs) + for desc, step_func in steps: if max_time and datetime.now() - start_time > max_time: logger.info("::: Test verification is taking too long: Giving up!") @@ -337,12 +365,16 @@ def check_stability(logger, repeat_loop=10, repeat_restart=5, chaos_mode=True, m if inconsistent: step_results.append((desc, "FAIL")) + if github_checks_outputter: + write_github_checks_summary_inconsistent(github_checks_outputter.output, inconsistent, iterations) write_inconsistent(logger.info, inconsistent, iterations) write_summary(logger, step_results, "FAIL") return 1 if slow: step_results.append((desc, "FAIL")) + if github_checks_outputter: + write_github_checks_summary_slow_tests(github_checks_outputter.output, slow) write_slow_tests(logger.info, slow) write_summary(logger, step_results, "FAIL") return 1 diff --git a/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/wptcommandline.py b/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/wptcommandline.py index f322d77fbdd..ea586825616 100644 --- a/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/wptcommandline.py +++ b/tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/wptcommandline.py @@ -348,6 +348,11 @@ scheme host and port.""") help="Command-line argument to forward to the " "Sauce Connect binary (repeatable)") + taskcluster_group = parser.add_argument_group("Taskcluster-specific") + taskcluster_group.add_argument("--github-checks-text-file", + dest="github_checks_text_file", + help="Path to GitHub checks output file") + webkit_group = parser.add_argument_group("WebKit-specific") webkit_group.add_argument("--webkit-port", dest="webkit_port", help="WebKit port") diff --git a/tests/wpt/web-platform-tests/user-timing/case-sensitivity.any.js b/tests/wpt/web-platform-tests/user-timing/case-sensitivity.any.js new file mode 100644 index 00000000000..1c0b0dcac36 --- /dev/null +++ b/tests/wpt/web-platform-tests/user-timing/case-sensitivity.any.js @@ -0,0 +1,25 @@ + test(function () { + assert_equals(typeof self.performance, "object"); + assert_equals(typeof self.performance.getEntriesByType, "function"); + + self.performance.mark("mark1"); + self.performance.measure("measure1"); + + const type = [ + 'mark', + 'measure', + ]; + type.forEach(function(entryType) { + if (PerformanceObserver.supportedEntryTypes.includes(entryType)) { + const entryTypeUpperCased = entryType.toUpperCase(); + const entryTypeCapitalized = entryType[0].toUpperCase() + entryType.substring(1); + const lowerList = self.performance.getEntriesByType(entryType); + const upperList = self.performance.getEntriesByType(entryTypeUpperCased); + const mixedList = self.performance.getEntriesByType(entryTypeCapitalized); + + assert_greater_than(lowerList.length, 0, "Entries exist"); + assert_equals(upperList.length, 0, "getEntriesByType('" + entryTypeCapitalized + "').length"); + assert_equals(mixedList.length, 0, "getEntriesByType('" + entryTypeCapitalized + "').length"); + } + }); + }, "getEntriesByType values are case sensitive"); diff --git a/tests/wpt/web-platform-tests/webdriver/tests/new_session/response.py b/tests/wpt/web-platform-tests/webdriver/tests/new_session/response.py index dc132899881..cbe9e7d7fb1 100644 --- a/tests/wpt/web-platform-tests/webdriver/tests/new_session/response.py +++ b/tests/wpt/web-platform-tests/webdriver/tests/new_session/response.py @@ -1,7 +1,7 @@ import uuid import pytest -from six import text_type +from six import string_types from tests.support.asserts import assert_success @@ -9,21 +9,21 @@ from tests.support.asserts import assert_success def test_sessionid(new_session, add_browser_capabilities): response, _ = new_session({"capabilities": {"alwaysMatch": add_browser_capabilities({})}}) value = assert_success(response) - assert isinstance(value["sessionId"], text_type) + assert isinstance(value["sessionId"], string_types) uuid.UUID(hex=value["sessionId"]) @pytest.mark.parametrize("capability, type", [ - ("browserName", text_type), - ("browserVersion", text_type), - ("platformName", text_type), + ("browserName", string_types), + ("browserVersion", string_types), + ("platformName", string_types), ("acceptInsecureCerts", bool), - ("pageLoadStrategy", text_type), + ("pageLoadStrategy", string_types), ("proxy", dict), ("setWindowRect", bool), ("timeouts", dict), ("strictFileInteractability", bool), - ("unhandledPromptBehavior", text_type), + ("unhandledPromptBehavior", string_types), ]) def test_capability_type(session, capability, type): assert isinstance(session.capabilities, dict) diff --git a/tests/wpt/webgl/meta/conformance2/rendering/draw-buffers.html.ini b/tests/wpt/webgl/meta/conformance2/rendering/draw-buffers.html.ini index abd1336b54a..fd7fbe111ff 100644 --- a/tests/wpt/webgl/meta/conformance2/rendering/draw-buffers.html.ini +++ b/tests/wpt/webgl/meta/conformance2/rendering/draw-buffers.html.ini @@ -209,3 +209,12 @@ [WebGL test #44: attachment 7 should be 0,255,0,255\nat (0, 0) expected: 0,255,0,255 was 0,0,255,0] expected: FAIL + [WebGL test #43: attachment 6 should be 0,255,0,255\nat (0, 0) expected: 0,255,0,255 was 0,0,0,255] + expected: FAIL + + [WebGL test #44: attachment 7 should be 0,255,0,255\nat (0, 0) expected: 0,255,0,255 was 255,0,0,255] + expected: FAIL + + [WebGL test #52: attachment 7 should be 0,255,0,255\nat (0, 0) expected: 0,255,0,255 was 255,0,0,255] + expected: FAIL +