Auto merge of #10631 - servo:wptrunner-20160415, r=KiChjang

Update wptrunner.

Fixes #10540.
Fixes #10392.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/10631)
<!-- Reviewable:end -->

python/servo/devenv_commands.py

@@ -152,11 +152,17 @@ class MachCommands(CommandBase):
             code = call(["git", "init"], env=self.build_env())
             if code:
                 return code
-            call(
+            code = call(
                 ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env())
+            if code:
+                return code
             code = call(["git", "fetch", "upstream"], env=self.build_env())
             if code:
                 return code
-            code = call(["git", "reset", '--', "hard", "remotes/upstream/master"], env=self.build_env())
+            code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=self.build_env())
             if code:
                 return code
+            code = call(["rm", "-rf", ".git"], env=self.build_env())
+            if code:
+                return code
+            return 0

tests/wpt/harness/README.rst

@@ -231,7 +231,7 @@ The web-platform-test harness knows about several keys:
   Any value indicates that the test is disabled.
 
 `type`
-  The test type e.g. `testharness` or `reftest`.
+  The test type e.g. `testharness`, `reftest`, or `wdspec`.
 
 `reftype`
   The type of comparison for reftests; either `==` or `!=`.

tests/wpt/harness/docs/expectation.rst

@@ -203,6 +203,10 @@ When used for expectation data, manifests have the following format:
    the (sub)test is disabled and should either not be run (for tests)
    or that its results should be ignored (subtests).
 
+ * A key ``restart-after`` which can be set to any value to indicate that
+   the runner should restart the browser after running this test (e.g. to
+   clear out unwanted state).
+
  * Variables ``debug``, ``os``, ``version``, ``processor`` and
    ``bits`` that describe the configuration of the browser under
    test. ``debug`` is a boolean indicating whether a build is a debug

tests/wpt/harness/wptrunner/browsers/chrome.py

@@ -3,7 +3,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from .base import Browser, ExecutorBrowser, require_arg
-from .webdriver import ChromedriverLocalServer
+from ..webdriver_server import ChromeDriverServer
 from ..executors import executor_kwargs as base_executor_kwargs
 from ..executors.executorselenium import (SeleniumTestharnessExecutor,
                                           SeleniumRefTestExecutor)
@@ -49,32 +49,33 @@ def env_options():
 
 class ChromeBrowser(Browser):
     """Chrome is backed by chromedriver, which is supplied through
-    ``browsers.webdriver.ChromedriverLocalServer``."""
+    ``wptrunner.webdriver.ChromeDriverServer``.
+    """
 
     def __init__(self, logger, binary, webdriver_binary="chromedriver"):
         """Creates a new representation of Chrome.  The `binary` argument gives
         the browser binary to use for testing."""
         Browser.__init__(self, logger)
         self.binary = binary
-        self.driver = ChromedriverLocalServer(self.logger, binary=webdriver_binary)
+        self.server = ChromeDriverServer(self.logger, binary=webdriver_binary)
 
     def start(self):
-        self.driver.start()
+        self.server.start(block=False)
 
     def stop(self):
-        self.driver.stop()
+        self.server.stop()
 
     def pid(self):
-        return self.driver.pid
+        return self.server.pid
 
     def is_alive(self):
         # TODO(ato): This only indicates the driver is alive,
         # and doesn't say anything about whether a browser session
         # is active.
-        return self.driver.is_alive()
+        return self.server.is_alive()
 
     def cleanup(self):
         self.stop()
 
     def executor_browser(self):
-        return ExecutorBrowser, {"webdriver_url": self.driver.url}
+        return ExecutorBrowser, {"webdriver_url": self.server.url}

tests/wpt/harness/wptrunner/browsers/firefox.py

@@ -13,18 +13,27 @@ from mozprofile.permissions import ServerLocations
 from mozrunner import FirefoxRunner
 from mozcrash import mozcrash
 
-from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg, browser_command
+from .base import (get_free_port,
+                   Browser,
+                   ExecutorBrowser,
+                   require_arg,
+                   cmd_arg,
+                   browser_command)
 from ..executors import executor_kwargs as base_executor_kwargs
-from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor
+from ..executors.executormarionette import (MarionetteTestharnessExecutor,
+                                            MarionetteRefTestExecutor,
+                                            MarionetteWdspecExecutor)
 from ..environment import hostnames
 
+
 here = os.path.join(os.path.split(__file__)[0])
 
 __wptrunner__ = {"product": "firefox",
                  "check_args": "check_args",
                  "browser": "FirefoxBrowser",
                  "executor": {"testharness": "MarionetteTestharnessExecutor",
-                              "reftest": "MarionetteRefTestExecutor"},
+                              "reftest": "MarionetteRefTestExecutor",
+                              "wdspec": "MarionetteWdspecExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options",
@@ -62,6 +71,8 @@ def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
                 executor_kwargs["timeout_multiplier"] = 2
         elif run_info_data["debug"] or run_info_data.get("asan"):
             executor_kwargs["timeout_multiplier"] = 3
+    if test_type == "wdspec":
+        executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
     return executor_kwargs
 
 
@@ -80,6 +91,7 @@ def run_info_extras(**kwargs):
 def update_properties():
     return ["debug", "e10s", "os", "version", "processor", "bits"], {"debug", "e10s"}
 
+
 class FirefoxBrowser(Browser):
     used_ports = set()
     init_timeout = 60

tests/wpt/harness/wptrunner/environment.py

@@ -32,7 +32,6 @@ def do_delayed_imports(logger, test_paths):
     global serve, sslutils
 
     serve_root = serve_path(test_paths)
-
     sys.path.insert(0, serve_root)
 
     failed = []
@@ -45,7 +44,6 @@ def do_delayed_imports(logger, test_paths):
     try:
         import sslutils
     except ImportError:
-        raise
         failed.append("sslutils")
 
     if failed:

tests/wpt/harness/wptrunner/executors/base.py

@@ -63,6 +63,7 @@ class TestharnessResultConverter(object):
                 [test.subtest_result_cls(name, self.test_codes[status], message, stack)
                  for name, status, message, stack in subtest_results])
 
+
 testharness_result_converter = TestharnessResultConverter()
 
 
@@ -71,11 +72,24 @@ def reftest_result_converter(self, test, result):
                             extra=result.get("extra")), [])
 
 
+def pytest_result_converter(self, test, data):
+    harness_data, subtest_data = data
+
+    if subtest_data is None:
+        subtest_data = []
+
+    harness_result = test.result_cls(*harness_data)
+    subtest_results = [test.subtest_result_cls(*item) for item in subtest_data]
+
+    return (harness_result, subtest_results)
+
+
 class ExecutorException(Exception):
     def __init__(self, status, message):
         self.status = status
         self.message = message
 
+
 class TestExecutor(object):
     __metaclass__ = ABCMeta
 
@@ -116,11 +130,13 @@ class TestExecutor(object):
 
         :param runner: TestRunner instance that is going to run the tests"""
         self.runner = runner
-        self.protocol.setup(runner)
+        if self.protocol is not None:
+            self.protocol.setup(runner)
 
     def teardown(self):
         """Run cleanup steps after tests have finished"""
-        self.protocol.teardown()
+        if self.protocol is not None:
+            self.protocol.teardown()
 
     def run_test(self, test):
         """Run a particular test.
@@ -137,6 +153,7 @@ class TestExecutor(object):
         if result is Stop:
             return result
 
+        # log result of parent test
         if result[0].status == "ERROR":
             self.logger.debug(result[0].message)
 
@@ -144,7 +161,6 @@ class TestExecutor(object):
 
         self.runner.send_message("test_ended", test, result)
 
-
     def server_url(self, protocol):
         return "%s://%s:%s" % (protocol,
                                self.server_config["host"],
@@ -191,6 +207,7 @@ class RefTestExecutor(TestExecutor):
 
         self.screenshot_cache = screenshot_cache
 
+
 class RefTestImplementation(object):
     def __init__(self, executor):
         self.timeout_multiplier = executor.timeout_multiplier
@@ -288,6 +305,11 @@ class RefTestImplementation(object):
         self.screenshot_cache[key] = hash_val, data
         return True, data
 
+
+class WdspecExecutor(TestExecutor):
+    convert_result = pytest_result_converter
+
+
 class Protocol(object):
     def __init__(self, executor, browser):
         self.executor = executor

tests/wpt/harness/wptrunner/executors/executormarionette.py

@@ -3,9 +3,9 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import hashlib
+import httplib
 import os
 import socket
-import sys
 import threading
 import time
 import traceback
@@ -13,10 +13,15 @@ import urlparse
 import uuid
 from collections import defaultdict
 
+from ..wpttest import WdspecResult, WdspecSubtestResult
+
+errors = None
 marionette = None
+webdriver = None
 
 here = os.path.join(os.path.split(__file__)[0])
 
+from . import pytestrunner
 from .base import (ExecutorException,
                    Protocol,
                    RefTestExecutor,
@@ -25,22 +30,29 @@ from .base import (ExecutorException,
                    TestharnessExecutor,
                    testharness_result_converter,
                    reftest_result_converter,
-                   strip_server)
+                   strip_server,
+                   WdspecExecutor)
 from ..testrunner import Stop
+from ..webdriver_server import GeckoDriverServer
 
 # Extra timeout to use after internal test timeout at which the harness
 # should force a timeout
 extra_timeout = 5 # seconds
 
+
 def do_delayed_imports():
-    global marionette
+    global errors, marionette, webdriver
-    global errors
+
+    # Marionette client used to be called marionette, recently it changed
+    # to marionette_driver for unfathomable reasons
     try:
         import marionette
         from marionette import errors
     except ImportError:
         from marionette_driver import marionette, errors
 
+    import webdriver
+
 
 class MarionetteProtocol(Protocol):
     def __init__(self, executor, browser):
@@ -54,8 +66,10 @@ class MarionetteProtocol(Protocol):
         """Connect to browser via Marionette."""
         Protocol.setup(self, runner)
 
-        self.logger.debug("Connecting to marionette on port %i" % self.marionette_port)
+        self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
-        self.marionette = marionette.Marionette(host='localhost', port=self.marionette_port)
+        self.marionette = marionette.Marionette(host='localhost',
+                                                port=self.marionette_port,
+                                                socket_timeout=None)
 
         # XXX Move this timeout somewhere
         self.logger.debug("Waiting for Marionette connection")
@@ -97,10 +111,10 @@ class MarionetteProtocol(Protocol):
             pass
         del self.marionette
 
+    @property
     def is_alive(self):
-        """Check if the marionette connection is still active"""
+        """Check if the Marionette connection is still active."""
         try:
-            # Get a simple property over the connection
             self.marionette.current_window_handle
         except Exception:
             return False
@@ -126,12 +140,18 @@ class MarionetteProtocol(Protocol):
             "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
 
     def wait(self):
+        socket_timeout = self.marionette.client.sock.gettimeout()
+        if socket_timeout:
+            self.marionette.set_script_timeout((socket_timeout / 2) * 1000)
+
         while True:
             try:
-                self.marionette.execute_async_script("");
+                self.marionette.execute_async_script("")
             except errors.ScriptTimeoutException:
+                self.logger.debug("Script timed out")
                 pass
             except (socket.timeout, IOError):
+                self.logger.debug("Socket closed")
                 break
             except Exception as e:
                 self.logger.error(traceback.format_exc(e))
@@ -213,7 +233,63 @@ class MarionetteProtocol(Protocol):
         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
             self.marionette.execute_script(script)
 
-class MarionetteRun(object):
+
+class RemoteMarionetteProtocol(Protocol):
+    def __init__(self, executor, browser):
+        do_delayed_imports()
+        Protocol.__init__(self, executor, browser)
+        self.session = None
+        self.webdriver_binary = executor.webdriver_binary
+        self.marionette_port = browser.marionette_port
+        self.server = None
+
+    def setup(self, runner):
+        """Connect to browser via the Marionette HTTP server."""
+        try:
+            self.server = GeckoDriverServer(
+                self.logger, self.marionette_port, binary=self.webdriver_binary)
+            self.server.start(block=False)
+            self.logger.info(
+                "WebDriver HTTP server listening at %s" % self.server.url)
+
+            self.logger.info(
+                "Establishing new WebDriver session with %s" % self.server.url)
+            self.session = webdriver.Session(
+                self.server.host, self.server.port, self.server.base_path)
+        except Exception:
+            self.logger.error(traceback.format_exc())
+            self.executor.runner.send_message("init_failed")
+        else:
+            self.executor.runner.send_message("init_succeeded")
+
+    def teardown(self):
+        try:
+            if self.session.session_id is not None:
+                self.session.end()
+        except Exception:
+            pass
+        if self.server is not None and self.server.is_alive:
+            self.server.stop()
+
+    @property
+    def is_alive(self):
+        """Test that the Marionette connection is still alive.
+
+        Because the remote communication happens over HTTP we need to
+        make an explicit request to the remote.  It is allowed for
+        WebDriver spec tests to not have a WebDriver session, since this
+        may be what is tested.
+
+        An HTTP request to an invalid path that results in a 404 is
+        proof enough to us that the server is alive and kicking.
+        """
+        conn = httplib.HTTPConnection(self.server.host, self.server.port)
+        conn.request("HEAD", self.server.base_path + "invalid")
+        res = conn.getresponse()
+        return res.status == 404
+
+
+class ExecuteAsyncScriptRun(object):
     def __init__(self, logger, func, marionette, url, timeout):
         self.logger = logger
         self.result = None
@@ -277,8 +353,8 @@ class MarionetteRun(object):
 
 
 class MarionetteTestharnessExecutor(TestharnessExecutor):
-    def __init__(self, browser, server_config, timeout_multiplier=1, close_after_done=True,
+    def __init__(self, browser, server_config, timeout_multiplier=1,
-                 debug_info=None):
+                 close_after_done=True, debug_info=None, **kwargs):
         """Marionette-based executor for testharness.js tests"""
         TestharnessExecutor.__init__(self, browser, server_config,
                                      timeout_multiplier=timeout_multiplier,
@@ -295,7 +371,7 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
             do_delayed_imports()
 
     def is_alive(self):
-        return self.protocol.is_alive()
+        return self.protocol.is_alive
 
     def on_environment_change(self, new_environment):
         self.protocol.on_environment_change(self.last_environment, new_environment)
@@ -307,11 +383,11 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
         timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
                    else None)
 
-        success, data = MarionetteRun(self.logger,
+        success, data = ExecuteAsyncScriptRun(self.logger,
-                                      self.do_testharness,
+                                              self.do_testharness,
-                                      self.protocol.marionette,
+                                              self.protocol.marionette,
-                                      self.test_url(test),
+                                              self.test_url(test),
-                                      timeout).run()
+                                              timeout).run()
         if success:
             return self.convert_result(test, data)
 
@@ -338,7 +414,9 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
 
 class MarionetteRefTestExecutor(RefTestExecutor):
     def __init__(self, browser, server_config, timeout_multiplier=1,
-                 screenshot_cache=None, close_after_done=True, debug_info=None):
+                 screenshot_cache=None, close_after_done=True,
+                 debug_info=None, **kwargs):
+
         """Marionette-based executor for reftests"""
         RefTestExecutor.__init__(self,
                                  browser,
@@ -358,7 +436,7 @@ class MarionetteRefTestExecutor(RefTestExecutor):
             self.wait_script = f.read()
 
     def is_alive(self):
-        return self.protocol.is_alive()
+        return self.protocol.is_alive
 
     def on_environment_change(self, new_environment):
         self.protocol.on_environment_change(self.last_environment, new_environment)
@@ -376,7 +454,6 @@ class MarionetteRefTestExecutor(RefTestExecutor):
             self.has_window = True
 
         result = self.implementation.run_test(test)
-
         return self.convert_result(test, result)
 
     def screenshot(self, test, viewport_size, dpi):
@@ -388,7 +465,7 @@ class MarionetteRefTestExecutor(RefTestExecutor):
 
         test_url = self.test_url(test)
 
-        return MarionetteRun(self.logger,
+        return ExecuteAsyncScriptRun(self.logger,
                              self._screenshot,
                              self.protocol.marionette,
                              test_url,
@@ -405,3 +482,78 @@ class MarionetteRefTestExecutor(RefTestExecutor):
             screenshot = screenshot.split(",", 1)[1]
 
         return screenshot
+
+
+class WdspecRun(object):
+    def __init__(self, func, session, path, timeout):
+        self.func = func
+        self.result = None
+        self.session = session
+        self.path = path
+        self.timeout = timeout
+        self.result_flag = threading.Event()
+
+    def run(self):
+        """Runs function in a thread and interrupts it if it exceeds the
+        given timeout.  Returns (True, (Result, [SubtestResult ...])) in
+        case of success, or (False, (status, extra information)) in the
+        event of failure.
+        """
+
+        executor = threading.Thread(target=self._run)
+        executor.start()
+
+        flag = self.result_flag.wait(self.timeout)
+        if self.result is None:
+            assert not flag
+            self.result = False, ("EXTERNAL-TIMEOUT", None)
+
+        return self.result
+
+    def _run(self):
+        try:
+            self.result = True, self.func(self.session, self.path, self.timeout)
+        except (socket.timeout, IOError):
+            self.result = False, ("CRASH", None)
+        except Exception as e:
+            message = getattr(e, "message")
+            if message:
+                message += "\n"
+            message += traceback.format_exc(e)
+            self.result = False, ("ERROR", message)
+        finally:
+            self.result_flag.set()
+
+
+class MarionetteWdspecExecutor(WdspecExecutor):
+    def __init__(self, browser, server_config, webdriver_binary,
+                 timeout_multiplier=1, close_after_done=True, debug_info=None):
+        WdspecExecutor.__init__(self, browser, server_config,
+                                timeout_multiplier=timeout_multiplier,
+                                debug_info=debug_info)
+        self.webdriver_binary = webdriver_binary
+        self.protocol = RemoteMarionetteProtocol(self, browser)
+
+    def is_alive(self):
+        return self.protocol.is_alive
+
+    def on_environment_change(self, new_environment):
+        pass
+
+    def do_test(self, test):
+        timeout = test.timeout * self.timeout_multiplier + extra_timeout
+
+        success, data = WdspecRun(self.do_wdspec,
+                                  self.protocol.session,
+                                  test.path,
+                                  timeout).run()
+
+        if success:
+            return self.convert_result(test, data)
+
+        return (test.result_cls(*data), [])
+
+    def do_wdspec(self, session, path, timeout):
+        harness_result = ("OK", None)
+        subtest_results = pytestrunner.run(path, session, timeout=timeout)
+        return (harness_result, subtest_results)

tests/wpt/harness/wptrunner/executors/executorservo.py

@@ -71,7 +71,7 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
         self.result_flag = threading.Event()
 
         args = [render_arg(self.browser.render_backend), "--hard-fail", "-u", "Servo/wptrunner",
-                "-z", self.test_url(test)]
+                "-Z", "replace-surrogates", "-z", self.test_url(test)]
         for stylesheet in self.browser.user_stylesheets:
             args += ["--user-stylesheet", stylesheet]
         for pref, value in test.environment.get('prefs', {}).iteritems():
@@ -204,7 +204,7 @@ class ServoRefTestExecutor(ProcessTestExecutor):
             debug_args, command = browser_command(
                 self.binary,
                 [render_arg(self.browser.render_backend), "--hard-fail", "--exit",
-                 "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously",
+                 "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
                  "--output=%s" % output_path, full_url],
                 self.debug_info)
 

tests/wpt/harness/wptrunner/executors/executorservodriver.py

@@ -14,16 +14,24 @@ from .base import (Protocol,
                    RefTestImplementation,
                    TestharnessExecutor,
                    strip_server)
-import webdriver
+from .. import webdriver
 from ..testrunner import Stop
 
+webdriver = None
+
 here = os.path.join(os.path.split(__file__)[0])
 
 extra_timeout = 5
 
 
+def do_delayed_imports():
+    global webdriver
+    import webdriver
+
+
 class ServoWebDriverProtocol(Protocol):
     def __init__(self, executor, browser, capabilities, **kwargs):
+        do_delayed_imports()
         Protocol.__init__(self, executor, browser)
         self.capabilities = capabilities
         self.host = browser.webdriver_host
@@ -34,10 +42,11 @@ class ServoWebDriverProtocol(Protocol):
         """Connect to browser via WebDriver."""
         self.runner = runner
 
+        url = "http://%s:%d" % (self.host, self.port)
         session_started = False
         try:
             self.session = webdriver.Session(self.host, self.port,
-                                             extension=webdriver.ServoExtensions)
+                extension=webdriver.servo.ServoCommandExtensions)
             self.session.start()
         except:
             self.logger.warning(
@@ -62,7 +71,7 @@ class ServoWebDriverProtocol(Protocol):
     def is_alive(self):
         try:
             # Get a simple property over the connection
-            self.session.handle
+            self.session.window_handle
         # TODO what exception?
         except Exception:
             return False

tests/wpt/harness/wptrunner/executors/pytestrunner/__init__.py

@@ -0,0 +1,6 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from . import fixtures
+from .runner import run

tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py

@@ -0,0 +1,58 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pytest
+
+
+"""pytest fixtures for use in Python-based WPT tests.
+
+The purpose of test fixtures is to provide a fixed baseline upon which
+tests can reliably and repeatedly execute.
+"""
+
+
+class Session(object):
+    """Fixture to allow access to wptrunner's existing WebDriver session
+    in tests.
+
+    The session is not created by default to enable testing of session
+    creation.  However, a module-scoped session will be implicitly created
+    at the first call to a WebDriver command.  This means methods such as
+    `session.send_command` and `session.session_id` are possible to use
+    without having a session.
+
+    To illustrate implicit session creation::
+
+        def test_session_scope(session):
+            # at this point there is no session
+            assert session.session_id is None
+
+            # window_id is a WebDriver command,
+            # and implicitly creates the session for us
+            assert session.window_id is not None
+
+            # we now have a session
+            assert session.session_id is not None
+
+    You can also access the session in custom fixtures defined in the
+    tests, such as a setup function::
+
+        @pytest.fixture(scope="function")
+        def setup(request, session):
+            session.url = "https://example.org"
+
+        def test_something(setup, session):
+            assert session.url == "https://example.org"
+
+    The session is closed when the test module goes out of scope by an
+    implicit call to `session.end`.
+    """
+
+    def __init__(self, client):
+        self.client = client
+
+    @pytest.fixture(scope="module")
+    def session(self, request):
+        request.addfinalizer(self.client.end)
+        return self.client

tests/wpt/harness/wptrunner/executors/pytestrunner/runner.py

@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Provides interface to deal with pytest.
+
+Usage::
+
+    session = webdriver.client.Session("127.0.0.1", "4444", "/")
+    harness_result = ("OK", None)
+    subtest_results = pytestrunner.run("/path/to/test", session.url)
+    return (harness_result, subtest_results)
+"""
+
+import errno
+import shutil
+import tempfile
+
+from . import fixtures
+
+
+pytest = None
+
+
+def do_delayed_imports():
+    global pytest
+    import pytest
+
+
+def run(path, session, timeout=0):
+    """Run Python test at ``path`` in pytest.  The provided ``session``
+    is exposed as a fixture available in the scope of the test functions.
+
+    :param path: Path to the test file.
+    :param session: WebDriver session to expose.
+    :param timeout: Duration before interrupting potentially hanging
+        tests.  If 0, there is no timeout.
+
+    :returns: List of subtest results, which are tuples of (test id,
+        status, message, stacktrace).
+    """
+
+    if pytest is None:
+        do_delayed_imports()
+
+    recorder = SubtestResultRecorder()
+    plugins = [recorder,
+               fixtures.Session(session)]
+
+    # TODO(ato): Deal with timeouts
+
+    with TemporaryDirectory() as cache:
+        pytest.main(["--strict",  # turn warnings into errors
+                     "--verbose",  # show each individual subtest
+                     "--capture", "no",  # enable stdout/stderr from tests
+                     "--basetemp", cache,  # temporary directory
+                     path],
+                    plugins=plugins)
+
+    return recorder.results
+
+
+class SubtestResultRecorder(object):
+    def __init__(self):
+        self.results = []
+
+    def pytest_runtest_logreport(self, report):
+        if report.passed and report.when == "call":
+            self.record_pass(report)
+        elif report.failed:
+            if report.when != "call":
+                self.record_error(report)
+            else:
+                self.record_fail(report)
+        elif report.skipped:
+            self.record_skip(report)
+
+    def record_pass(self, report):
+        self.record(report.nodeid, "PASS")
+
+    def record_fail(self, report):
+        self.record(report.nodeid, "FAIL", stack=report.longrepr)
+
+    def record_error(self, report):
+        # error in setup/teardown
+        if report.when != "call":
+            message = "%s error" % report.when
+        self.record(report.nodeid, "ERROR", message, report.longrepr)
+
+    def record_skip(self, report):
+        self.record(report.nodeid, "ERROR",
+                    "In-test skip decorators are disallowed, "
+                    "please use WPT metadata to ignore tests.")
+
+    def record(self, test, status, message=None, stack=None):
+        if stack is not None:
+            stack = str(stack)
+        new_result = (test, status, message, stack)
+        self.results.append(new_result)
+
+
+class TemporaryDirectory(object):
+    def __enter__(self):
+        self.path = tempfile.mkdtemp(prefix="pytest-")
+        return self.path
+
+    def __exit__(self, *args):
+        try:
+            shutil.rmtree(self.path)
+        except OSError as e:
+            # no such file or directory
+            if e.errno != errno.ENOENT:
+                raise

tests/wpt/harness/wptrunner/manifestexpected.py

@@ -29,10 +29,10 @@ def data_cls_getter(output_node, visited_node):
     raise ValueError
 
 
-def disabled(node):
+def bool_prop(name, node):
-    """Boolean indicating whether the test is disabled"""
+    """Boolean property"""
     try:
-        return node.get("disabled")
+        return node.get(name)
     except KeyError:
         return None
 
@@ -109,7 +109,11 @@ class ExpectedManifest(ManifestItem):
 
     @property
     def disabled(self):
-        return disabled(self)
+        return bool_prop("disabled", self)
+
+    @property
+    def restart_after(self):
+        return bool_prop("restart-after", self)
 
     @property
     def tags(self):
@@ -123,7 +127,11 @@ class ExpectedManifest(ManifestItem):
 class DirectoryManifest(ManifestItem):
     @property
     def disabled(self):
-        return disabled(self)
+        return bool_prop("disabled", self)
+
+    @property
+    def restart_after(self):
+        return bool_prop("restart-after", self)
 
     @property
     def tags(self):
@@ -164,7 +172,11 @@ class TestNode(ManifestItem):
 
     @property
     def disabled(self):
-        return disabled(self)
+        return bool_prop("disabled", self)
+
+    @property
+    def restart_after(self):
+        return bool_prop("restart-after", self)
 
     @property
     def tags(self):

tests/wpt/harness/wptrunner/testrunner.py

@@ -524,7 +524,8 @@ class TestRunnerManager(threading.Thread):
 
         self.test = None
 
-        restart_before_next = (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
+        restart_before_next = (test.restart_after or
+                               file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
                                subtest_unexpected or is_unexpected)
 
         if (self.pause_after_test or

tests/wpt/harness/wptrunner/webdriver_server.py

@@ -0,0 +1,206 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import abc
+import errno
+import os
+import platform
+import socket
+import threading
+import time
+import traceback
+import urlparse
+
+import mozprocess
+
+
+__all__ = ["SeleniumServer", "ChromeDriverServer",
+           "GeckoDriverServer", "WebDriverServer"]
+
+
+class WebDriverServer(object):
+    __metaclass__ = abc.ABCMeta
+
+    default_base_path = "/"
+    _used_ports = set()
+
+    def __init__(self, logger, binary, host="127.0.0.1", port=None,
+                 base_path="", env=None):
+        self.logger = logger
+        self.binary = binary
+        self.host = host
+        if base_path == "":
+            self.base_path = self.default_base_path
+        else:
+            self.base_path = base_path
+        self.env = os.environ.copy() if env is None else env
+
+        self._port = port
+        self._cmd = None
+        self._proc = None
+
+    @abc.abstractmethod
+    def make_command(self):
+        """Returns the full command for starting the server process as a list."""
+
+    def start(self, block=True):
+        try:
+            self._run(block)
+        except KeyboardInterrupt:
+            self.stop()
+
+    def _run(self, block):
+        self._cmd = self.make_command()
+        self._proc = mozprocess.ProcessHandler(
+            self._cmd,
+            processOutputLine=self.on_output,
+            env=self.env,
+            storeOutput=False)
+
+        try:
+            self._proc.run()
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                raise IOError(
+                    "WebDriver HTTP server executable not found: %s" % self.binary)
+            raise
+
+        self.logger.debug(
+            "Waiting for server to become accessible: %s" % self.url)
+        try:
+            wait_for_service((self.host, self.port))
+        except:
+            self.logger.error(
+                "WebDriver HTTP server was not accessible "
+                "within the timeout:\n%s" % traceback.format_exc())
+            raise
+
+        if block:
+            self._proc.wait()
+
+    def stop(self):
+        if self.is_alive:
+            return self._proc.kill()
+        return not self.is_alive
+
+    @property
+    def is_alive(self):
+        return (self._proc is not None and
+                self._proc.proc is not None and
+                self._proc.poll() is None)
+
+    def on_output(self, line):
+        self.logger.process_output(self.pid,
+                                   line.decode("utf8", "replace"),
+                                   command=" ".join(self._cmd))
+
+    @property
+    def pid(self):
+        if self._proc is not None:
+            return self._proc.pid
+
+    @property
+    def url(self):
+        return "http://%s:%i%s" % (self.host, self.port, self.base_path)
+
+    @property
+    def port(self):
+        if self._port is None:
+            self._port = self._find_next_free_port()
+        return self._port
+
+    @staticmethod
+    def _find_next_free_port():
+        port = get_free_port(4444, exclude=WebDriverServer._used_ports)
+        WebDriverServer._used_ports.add(port)
+        return port
+
+
+class SeleniumServer(WebDriverServer):
+    default_base_path = "/wd/hub"
+
+    def make_command(self):
+        return ["java", "-jar", self.binary, "-port", str(self.port)]
+
+
+class ChromeDriverServer(WebDriverServer):
+    default_base_path = "/wd/hub"
+
+    def __init__(self, logger, binary="chromedriver", port=None,
+                 base_path=""):
+        WebDriverServer.__init__(
+            self, logger, binary, port=port, base_path=base_path)
+
+    def make_command(self):
+        return [self.binary,
+                cmd_arg("port", str(self.port)),
+                cmd_arg("url-base", self.base_path) if self.base_path else ""]
+
+
+class GeckoDriverServer(WebDriverServer):
+    def __init__(self, logger, marionette_port=2828, binary="wires",
+                 host="127.0.0.1", port=None):
+        env = os.environ.copy()
+        env["RUST_BACKTRACE"] = "1"
+        WebDriverServer.__init__(self, logger, binary, host=host, port=port, env=env)
+        self.marionette_port = marionette_port
+
+    def make_command(self):
+        return [self.binary,
+                "--connect-existing",
+                "--marionette-port", str(self.marionette_port),
+                "--webdriver-host", self.host,
+                "--webdriver-port", str(self.port)]
+
+
+def cmd_arg(name, value=None):
+    prefix = "-" if platform.system() == "Windows" else "--"
+    rv = prefix + name
+    if value is not None:
+        rv += "=" + value
+    return rv
+
+
+def get_free_port(start_port, exclude=None):
+    """Get the first port number after start_port (inclusive) that is
+    not currently bound.
+
+    :param start_port: Integer port number at which to start testing.
+    :param exclude: Set of port numbers to skip"""
+    port = start_port
+    while True:
+        if exclude and port in exclude:
+            port += 1
+            continue
+        s = socket.socket()
+        try:
+            s.bind(("127.0.0.1", port))
+        except socket.error:
+            port += 1
+        else:
+            return port
+        finally:
+            s.close()
+
+
+def wait_for_service(addr, timeout=15):
+    """Waits until network service given as a tuple of (host, port) becomes
+    available or the `timeout` duration is reached, at which point
+    ``socket.error`` is raised."""
+    end = time.time() + timeout
+    while end > time.time():
+        so = socket.socket()
+        try:
+            so.connect(addr)
+        except socket.timeout:
+            pass
+        except socket.error as e:
+            if e[0] != errno.ECONNREFUSED:
+                raise
+        else:
+            return True
+        finally:
+            so.close()
+        time.sleep(0.5)
+    raise socket.error("Service is unavailable: %s:%i" % addr)

tests/wpt/harness/wptrunner/wptcommandline.py

@@ -10,6 +10,7 @@ from collections import OrderedDict
 from distutils.spawn import find_executable
 
 import config
+import wpttest
 
 
 def abs_path(path):
@@ -25,6 +26,7 @@ def url_or_path(path):
     else:
         return abs_path(path)
 
+
 def require_arg(kwargs, name, value_func=None):
     if value_func is None:
         value_func = lambda x: x is not None
@@ -101,8 +103,8 @@ def create_parser(product_choices=None):
 
     test_selection_group = parser.add_argument_group("Test Selection")
     test_selection_group.add_argument("--test-types", action="store",
-                                      nargs="*", default=["testharness", "reftest"],
+                                      nargs="*", default=wpttest.enabled_tests,
-                                      choices=["testharness", "reftest"],
+                                      choices=wpttest.enabled_tests,
                                       help="Test types to run")
     test_selection_group.add_argument("--include", action="append",
                                       help="URL prefix to include")
@@ -159,8 +161,8 @@ def create_parser(product_choices=None):
     gecko_group = parser.add_argument_group("Gecko-specific")
     gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
                              help="Path to the folder containing browser prefs")
-    gecko_group.add_argument("--e10s", dest="gecko_e10s", action="store_true",
+    gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
-                             help="Run tests with electrolysis preferences")
+                             help="Run tests without electrolysis preferences")
 
     b2g_group = parser.add_argument_group("B2G-specific")
     b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False,
@@ -343,12 +345,14 @@ def check_args(kwargs):
 
     return kwargs
 
+
 def check_args_update(kwargs):
     set_from_config(kwargs)
 
     if kwargs["product"] is None:
         kwargs["product"] = "firefox"
 
+
 def create_parser_update(product_choices=None):
     from mozlog.structured import commandline
 

tests/wpt/harness/wptrunner/wptlogging.py

@@ -74,6 +74,7 @@ class LoggingWrapper(StringIO):
     instead"""
 
     def __init__(self, queue, prefix=None):
+        StringIO.__init__(self)
         self.queue = queue
         self.prefix = prefix
 
@@ -94,6 +95,7 @@ class LoggingWrapper(StringIO):
     def flush(self):
         pass
 
+
 class CaptureIO(object):
     def __init__(self, logger, do_capture):
         self.logger = logger

tests/wpt/harness/wptrunner/wpttest.py

@@ -12,6 +12,8 @@ import mozinfo
 from wptmanifest.parser import atoms
 
 atom_reset = atoms["Reset"]
+enabled_tests = set(["testharness", "reftest", "wdspec"])
+
 
 class Result(object):
     def __init__(self, status, message, expected=None, extra=None):
@@ -22,6 +24,9 @@ class Result(object):
         self.expected = expected
         self.extra = extra
 
+    def __repr__(self):
+        return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status)
+
 
 class SubtestResult(object):
     def __init__(self, name, status, message, stack=None, expected=None):
@@ -33,20 +38,33 @@ class SubtestResult(object):
         self.stack = stack
         self.expected = expected
 
+    def __repr__(self):
+        return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status)
+
 
 class TestharnessResult(Result):
     default_expected = "OK"
     statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
 
 
+class TestharnessSubtestResult(SubtestResult):
+    default_expected = "PASS"
+    statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"])
+
+
 class ReftestResult(Result):
     default_expected = "PASS"
     statuses = set(["PASS", "FAIL", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
 
 
-class TestharnessSubtestResult(SubtestResult):
+class WdspecResult(Result):
+    default_expected = "OK"
+    statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
+
+
+class WdspecSubtestResult(SubtestResult):
     default_expected = "PASS"
-    statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"])
+    statuses = set(["PASS", "FAIL", "ERROR"])
 
 
 def get_run_info(metadata_root, product, **kwargs):
@@ -82,6 +100,7 @@ class RunInfo(dict):
 
         mozinfo.find_and_update_from_json(*dirs)
 
+
 class B2GRunInfo(RunInfo):
     def __init__(self, *args, **kwargs):
         RunInfo.__init__(self, *args, **kwargs)
@@ -115,7 +134,6 @@ class Test(object):
                    path=manifest_item.path,
                    protocol="https" if hasattr(manifest_item, "https") and manifest_item.https else "http")
 
-
     @property
     def id(self):
         return self.url
@@ -141,7 +159,6 @@ class Test(object):
                 if subtest_meta is not None:
                     yield subtest_meta
 
-
     def disabled(self, subtest=None):
         for meta in self.itermeta(subtest):
             disabled = meta.disabled
@@ -149,6 +166,14 @@ class Test(object):
                 return disabled
         return None
 
+    @property
+    def restart_after(self):
+        for meta in self.itermeta(None):
+            restart_after = meta.restart_after
+            if restart_after is not None:
+                return True
+        return False
+
     @property
     def tags(self):
         tags = set()
@@ -191,6 +216,9 @@ class Test(object):
         except KeyError:
             return default
 
+    def __repr__(self):
+        return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id)
+
 
 class TestharnessTest(Test):
     result_cls = TestharnessResult
@@ -293,12 +321,18 @@ class ReftestTest(Test):
         return ("reftype", "refurl")
 
 
+class WdspecTest(Test):
+    result_cls = WdspecResult
+    subtest_result_cls = WdspecSubtestResult
+    test_type = "wdspec"
+
+
 manifest_test_cls = {"reftest": ReftestTest,
                      "testharness": TestharnessTest,
-                     "manual": ManualTest}
+                     "manual": ManualTest,
+                     "wdspec": WdspecTest}
 
 
 def from_manifest(manifest_test, inherit_metadata, test_metadata):
     test_cls = manifest_test_cls[manifest_test.item_type]
-
     return test_cls.from_manifest(manifest_test, inherit_metadata, test_metadata)

tests/wpt/metadata-css/cssom-1_dev/html/escape.htm.ini

@@ -1,3 +1,14 @@
 [escape.htm]
   type: testharness
-  expected: CRASH
+  [Null bytes]
+    expected: FAIL
+    bug: https://github.com/servo/servo/issues/10685
+
+  [Various tests]
+    expected: FAIL
+    bug: https://github.com/servo/servo/issues/10685
+
+  [Surrogates]
+    expected: FAIL
+    bug: https://github.com/servo/servo/issues/6564
+

tests/wpt/metadata/webgl/conformance-1.0.3/conformance/programs/invalid-UTF-16.html.ini

@@ -1,3 +0,0 @@
-[invalid-UTF-16.html]
-  type: testharness
-  expected: CRASH

tests/wpt/metadata/webstorage/storage_setitem.html.ini

@@ -1,3 +1,19 @@
 [storage_setitem.html]
   type: testharness
-  expected: CRASH
+  expected: TIMEOUT
+  [localStorage[\] = "<22>"]
+    expected: FAIL
+    bug: https://github.com/servo/servo/issues/6564
+
+  [localStorage[\] = "<22>a"]
+    expected: FAIL
+    bug: https://github.com/servo/servo/issues/6564
+
+  [localStorage[\] = "a<>"]
+    expected: FAIL
+    bug: https://github.com/servo/servo/issues/6564
+
+  [localStorage["0"\]]
+    expected: TIMEOUT
+    bug: https://github.com/servo/servo/issues/10686
+