Merge pull request #2942 from Ms2ger/run-wpt-2

Run web-platform-tests on Travis; r=jgraham+larsberg

mk/check.mk

@@ -58,7 +58,7 @@ check-test:
 
 ifeq ($(CFG_OSTYPE),apple-darwin)
 .PHONY: check
-check: $(DEPS_CHECK_TARGETS_FAST) check-servo check-content check-ref tidy
+check: $(DEPS_CHECK_TARGETS_FAST) check-servo check-wpt check-content check-ref tidy
 	@$(call E, check: all)
 
 .PHONY: check-all
@@ -99,6 +99,7 @@ check-content: contenttest
 
 .PHONY: check-wpt
 check-wpt:
+	@$(call E, check: web-platform-tests)
 	bash $(S)src/test/wpt/run.sh $(S) $(B) $(WPTARGS)
 
 .PHONY: tidy

src/test/wpt/config.ini

@@ -0,0 +1,11 @@
+[products]
+servo =
+
+[web-platform-tests]
+remote_url = https://github.com/w3c/web-platform-tests.git
+branch = master
+sync_path = web-platform-tests
+
+[paths]
+tests = web-platform-tests
+metadata = metadata

src/test/wpt/metadata/XMLHttpRequest/data-uri-basic.htm.ini

@@ -1,20 +1,30 @@
 [data-uri-basic.htm]
   type: testharness
+  expected: TIMEOUT
   [GET responseText]
-    expected: FAIL
+    expected: TIMEOUT
 
   [GET Content-Type]
-    expected: FAIL
+    expected: TIMEOUT
 
   [GET Content-Type with param]
-    expected: FAIL
+    expected: TIMEOUT
 
   [GET getAllResponseHeaders]
-    expected: FAIL
+    expected: TIMEOUT
 
   [POST responseText]
-    expected: FAIL
+    expected: TIMEOUT
 
   [POST status]
-    expected: FAIL
+    expected: TIMEOUT
+
+  [GET responseText base64]
+    expected: TIMEOUT
+
+  [GET status]
+    expected: TIMEOUT
+
+  [GET statusText]
+    expected: TIMEOUT
 

src/test/wpt/metadata/XMLHttpRequest/data-uri.htm.ini

@@ -1,29 +1,30 @@
 [data-uri.htm]
   type: testharness
+  expected: TIMEOUT
   [XHR method GET with charset text/plain]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method GET with charset text/plain (base64)]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method GET with charset text/html]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method GET with charset image/png]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method POST with charset text/plain]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method PUT with charset text/plain]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method DELETE with charset text/plain]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method HEAD with charset text/plain]
-    expected: FAIL
+    expected: TIMEOUT
 
   [XHR method UNICORN with charset text/plain]
-    expected: FAIL
+    expected: TIMEOUT
 

src/test/wpt/metadata/XMLHttpRequest/open-referer.htm.ini

@@ -0,0 +1,5 @@
+[open-referer.htm]
+  type: testharness
+  [XMLHttpRequest: open() - value of Referer header]
+    expected: FAIL
+

src/test/wpt/metadata/XMLHttpRequest/open-url-about-blank-window.htm.ini

@@ -1,5 +1,3 @@
 [open-url-about-blank-window.htm]
   type: testharness
-  [XMLHttpRequest: open() resolving URLs (about:blank iframe)]
+  expected: TIMEOUT
-    expected: FAIL
-

src/test/wpt/metadata/XMLHttpRequest/open-url-fragment.htm.ini

@@ -1,5 +0,0 @@
-[open-url-fragment.htm]
-  type: testharness
-  [make sure escaped # is not removed]
-    expected: FAIL
-

src/test/wpt/metadata/XMLHttpRequest/responsetext-decoding.htm.ini

@@ -1,11 +1,6 @@
 [responsetext-decoding.htm]
   type: testharness
-  [XMLHttpRequest: responseText decoding (text/plain;charset=windows-1252 %FF)]
+  expected: TIMEOUT
-    expected: FAIL
-
-  [XMLHttpRequest: responseText decoding (text/plain %FF)]
-    expected: FAIL
-
   [XMLHttpRequest: responseText decoding (text/plain %FE%FF)]
     expected: FAIL
 
@@ -18,9 +13,6 @@
   [XMLHttpRequest: responseText decoding (text/plain %EF%BB%BF%EF%BB%BF)]
     expected: FAIL
 
-  [XMLHttpRequest: responseText decoding (text/plain %C2)]
-    expected: FAIL
-
   [XMLHttpRequest: responseText decoding (text/xml %FE%FF)]
     expected: FAIL
 
@@ -33,9 +25,9 @@
   [XMLHttpRequest: responseText decoding (text/xml %EF%BB%BF%EF%BB%BF)]
     expected: FAIL
 
-  [XMLHttpRequest: responseText decoding (text/plain %E3%81%B2)]
+  [XMLHttpRequest: responseText decoding (application/xml %3C%3Fxml%20version%3D\'1.0\'%20encoding%3D\'windows-1252\'%3F%3E%3Cx%3E%FF%3C%2Fx%3E)]
     expected: FAIL
 
-  [XMLHttpRequest: responseText decoding (text/xml %3C%3Fxml%20version%3D\'1.0\'%20encoding%3D\'windows-1252\'%3F%3E%3Cx%3E%E3%81%B2%3C%2Fx%3E)]
+  [XMLHttpRequest: responseText decoding (text/html %3C!doctype%20html%3E%3Cmeta%20charset%3Dwindows-1252%3E%FF)]
     expected: FAIL
 

src/test/wpt/metadata/XMLHttpRequest/send-network-error-async-events.sub.htm.ini

@@ -1,5 +1,6 @@
 [send-network-error-async-events.sub.htm]
   type: testharness
+  expected: TIMEOUT
   [XmlHttpRequest: The send() method: Fire a progress event named error when Network error happens (synchronous flag is unset)]
-    expected: FAIL
+    expected: TIMEOUT
 

src/test/wpt/metadata/XMLHttpRequest/send-redirect-infinite-sync.htm.ini

@@ -0,0 +1,5 @@
+[send-redirect-infinite-sync.htm]
+  type: testharness
+  [XMLHttpRequest: send() - Redirects (infinite loop; sync) (301)]
+    expected: FAIL
+

src/test/wpt/metadata/XMLHttpRequest/send-redirect-infinite.htm.ini

@@ -0,0 +1,5 @@
+[send-redirect-infinite.htm]
+  type: testharness
+  [XMLHttpRequest: send() - Redirects (infinite loop)]
+    expected: FAIL
+

src/test/wpt/metadata/dom/errors/exceptions.html.ini

@@ -1,9 +1,3 @@
 [exceptions.html]
   type: testharness
   expected: TIMEOUT
-  [exception.hasOwnProperty("message")]
-    expected: FAIL
-
-  [Object.getOwnPropertyDescriptor(exception, "message")]
-    expected: FAIL
-

src/test/wpt/metadata/dom/events/Event-constants.html.ini

@@ -1,3 +0,0 @@
-[Event-constants.html]
-  type: testharness
-  expected: TIMEOUT

src/test/wpt/metadata/dom/events/EventTarget-dispatchEvent.html.ini

@@ -1,3 +0,0 @@
-[EventTarget-dispatchEvent.html]
-  type: testharness
-  expected: TIMEOUT

src/test/wpt/metadata/dom/nodes/Node-appendChild.html.ini

@@ -1,6 +1,3 @@
 [Node-appendChild.html]
   type: testharness
   expected: TIMEOUT
-  [WebIDL tests]
-    expected: TIMEOUT
-

src/test/wpt/metadata/dom/nodes/Node-compareDocumentPosition.html.ini

@@ -1,3 +1,3 @@
 [Node-compareDocumentPosition.html]
   type: testharness
-  expected: TIMEOUT
+  expected: ERROR

src/test/wpt/metadata/dom/nodes/Node-constants.html.ini

@@ -1,3 +0,0 @@
-[Node-constants.html]
-  type: testharness
-  expected: TIMEOUT

src/test/wpt/metadata/dom/nodes/Node-contains.html.ini

@@ -1,3 +1,3 @@
 [Node-contains.html]
   type: testharness
-  expected: TIMEOUT
+  expected: ERROR

src/test/wpt/metadata/dom/nodes/Node-properties.html.ini

@@ -1,3 +1,3 @@
 [Node-properties.html]
   type: testharness
-  expected: TIMEOUT
+  expected: ERROR

src/test/wpt/metadata/dom/ranges/Range-attributes.html.ini

@@ -1,5 +1,3 @@
 [Range-attributes.html]
   type: testharness
-  [Range attributes]
+  disabled: Range support
-    expected: FAIL
-

src/test/wpt/metadata/dom/ranges/Range-cloneContents.html.ini

@@ -1,3 +1,3 @@
 [Range-cloneContents.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-cloneRange.html.ini

@@ -1,3 +1,3 @@
 [Range-cloneRange.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-collapse.html.ini

@@ -1,3 +1,3 @@
 [Range-collapse.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-commonAncestorContainer-2.html.ini

@@ -1,17 +1,3 @@
 [Range-commonAncestorContainer-2.html]
   type: testharness
-  [Detached Range]
+  disabled: Range support
-    expected: FAIL
-
-  [Range.commonAncestorContainer]
-    expected: FAIL
-
-  [Range.commonAncestorContainer 1]
-    expected: FAIL
-
-  [Range.commonAncestorContainer 2]
-    expected: FAIL
-
-  [Range.commonAncestorContainer 3]
-    expected: FAIL
-

src/test/wpt/metadata/dom/ranges/Range-commonAncestorContainer.html.ini

@@ -1,3 +1,3 @@
 [Range-commonAncestorContainer.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-compareBoundaryPoints.html.ini

@@ -1,3 +1,3 @@
 [Range-compareBoundaryPoints.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-comparePoint-2.html.ini

@@ -1,11 +1,3 @@
 [Range-comparePoint-2.html]
   type: testharness
-  [Range.comparePoint]
+  disabled: Range support
-    expected: FAIL
-
-  [Range.comparePoint 1]
-    expected: FAIL
-
-  [Range.comparePoint 2]
-    expected: FAIL
-

src/test/wpt/metadata/dom/ranges/Range-comparePoint.html.ini

@@ -1,3 +1,3 @@
 [Range-comparePoint.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-deleteContents.html.ini

@@ -1,3 +1,3 @@
 [Range-deleteContents.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-detach.html.ini

@@ -1,5 +1,3 @@
 [Range-detach.html]
   type: testharness
-  [Range.detach]
+  disabled: Range support
-    expected: FAIL
-

src/test/wpt/metadata/dom/ranges/Range-extractContents.html.ini

@@ -1,3 +1,3 @@
 [Range-extractContents.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-insertNode.html.ini

@@ -1,3 +1,3 @@
 [Range-insertNode.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-intersectsNode-binding.html.ini

@@ -1,5 +1,3 @@
 [Range-intersectsNode-binding.html]
   type: testharness
-  [Calling intersectsNode without an argument or with an invalid argument should throw a TypeError.]
+  disabled: Range support
-    expected: FAIL
-

src/test/wpt/metadata/dom/ranges/Range-intersectsNode.html.ini

@@ -1,3 +1,3 @@
 [Range-intersectsNode.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-isPointInRange.html.ini

@@ -1,5 +1,3 @@
 [Range-isPointInRange.html]
   type: testharness
-  [Setup]
+  disabled: Range support
-    expected: FAIL
-

src/test/wpt/metadata/dom/ranges/Range-selectNode.html.ini

@@ -1,3 +1,3 @@
 [Range-selectNode.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-set.html.ini

@@ -1,3 +1,3 @@
 [Range-set.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/metadata/dom/ranges/Range-surroundContents.html.ini

@@ -1,3 +1,3 @@
 [Range-surroundContents.html]
   type: testharness
-  expected: TIMEOUT
+  disabled: Range support

src/test/wpt/run.py

@@ -29,17 +29,14 @@ def run_tests(**kwargs):
     return wptrunner.run_tests(**kwargs)
 
 def set_defaults(args):
-    args.metadata_root = args.metadata_root if args.metadata_root else wptsubdir("metadata")
-    args.tests_root = args.tests_root if args.tests_root else wptsubdir("web-platform-tests")
     args.include_manifest = args.include_manifest if args.include_manifest else wptsubdir("include.ini")
-    args.binary = args.binary if args.binary else os.path.join(servo_root, "build", "servo")
     args.product = "servo"
-    args.debug_args = None
+    rv = vars(args)
-    args.interactive = False
+    wptcommandline.check_args(rv)
-    return vars(args)
+    return rv
 
 def main():
-    parser = wptcommandline.create_parser(False)
+    parser = wptcommandline.create_parser()
     parser.add_argument('--update-manifest', dest='update_manifest', action='store_true')
     args = parser.parse_args()
     if args.update_manifest:

src/test/wpt/run.sh

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+set -e
+
 servo_root="$1"
 objdir="$2"
 shift 2
@@ -14,6 +16,10 @@ source _virtualenv/bin/activate
 if [[ $* == *--update-manifest* ]]; then
     (python -c "import html5lib" &>/dev/null) || pip install html5lib
 fi
-(python -c "import wptrunner"  &>/dev/null) || pip install wptrunner
+(python -c "import wptrunner"  &>/dev/null) || pip install 'wptrunner==1.0'
 
-python $servo_root/src/test/wpt/run.py --binary $objdir/../servo "$@"
+python $servo_root/src/test/wpt/run.py \
+  --config $servo_root/src/test/wpt/config.ini \
+  --binary $objdir/../servo \
+  --log-mach - \
+  "$@"

travis.osx.install.deps.sh

@@ -1,2 +1,4 @@
 brew install https://raw.github.com/Homebrew/homebrew-versions/master/autoconf213.rb
 brew install automake libtool pkg-config
+brew install python
+pip install virtualenv

travis.osx.script.sh

@@ -1,3 +1,4 @@
 cd build
 ../configure
 make tidy && make -j2 && make check-servo && make check-content && make check-ref-cpu
+WPTARGS="--processes=4" make check-wpt