diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py
index 8163340c501..8618041b756 100644
--- a/python/servo/devenv_commands.py
+++ b/python/servo/devenv_commands.py
@@ -170,25 +170,28 @@ class MachCommands(CommandBase):
print(cargo_path)
call(["cargo", "fetch"], env=self.build_env())
- @Command('wpt-upgrade',
+ @Command('wptrunner-upgrade',
description='upgrade wptrunner.',
category='devenv')
def upgrade_wpt_runner(self):
+ env = self.build_env()
with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')):
- code = call(["git", "init"], env=self.build_env())
+ code = call(["git", "init"], env=env)
if code:
return code
+ # No need to report an error if this fails, as it will for the first use
+ call(["git", "remote", "rm", "upstream"], env=env)
code = call(
- ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env())
+ ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=env)
if code:
return code
- code = call(["git", "fetch", "upstream"], env=self.build_env())
+ code = call(["git", "fetch", "upstream"], env=env)
if code:
return code
- code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=self.build_env())
+ code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=env)
if code:
return code
- code = call(["rm", "-rf", ".git"], env=self.build_env())
+ code = call(["rm", "-rf", ".git"], env=env)
if code:
return code
return 0
diff --git a/tests/wpt/harness/.travis.yml b/tests/wpt/harness/.travis.yml
new file mode 100644
index 00000000000..add6efd12ca
--- /dev/null
+++ b/tests/wpt/harness/.travis.yml
@@ -0,0 +1,20 @@
+language: python
+python: 2.7
+
+sudo: false
+
+cache:
+ directories:
+ - $HOME/.cache/pip
+
+env:
+ - TOXENV="{py27,pypy}-base"
+ - TOXENV="{py27,pypy}-chrome"
+ - TOXENV="{py27,pypy}-firefox"
+ - TOXENV="{py27,pypy}-servo"
+
+install:
+ - pip install -U tox
+
+script:
+ - tox
diff --git a/tests/wpt/harness/tox.ini b/tests/wpt/harness/tox.ini
new file mode 100644
index 00000000000..844a8d05b20
--- /dev/null
+++ b/tests/wpt/harness/tox.ini
@@ -0,0 +1,15 @@
+[pytest]
+xfail_strict=true
+
+[tox]
+envlist = {py27,pypy}-{base,b2g,chrome,firefox,servo}
+
+[testenv]
+deps =
+ pytest>=2.9
+ -r{toxinidir}/requirements.txt
+ chrome: -r{toxinidir}/requirements_chrome.txt
+ firefox: -r{toxinidir}/requirements_firefox.txt
+ servo: -r{toxinidir}/requirements_servo.txt
+
+commands = py.test []
diff --git a/tests/wpt/harness/wptrunner/browsers/__init__.py b/tests/wpt/harness/wptrunner/browsers/__init__.py
index ffc5aedc830..8b34cc3963f 100644
--- a/tests/wpt/harness/wptrunner/browsers/__init__.py
+++ b/tests/wpt/harness/wptrunner/browsers/__init__.py
@@ -26,8 +26,8 @@ All classes and functions named in the above dict must be imported into the
module global scope.
"""
-product_list = ["b2g",
- "chrome",
+product_list = ["chrome",
+ "edge",
"firefox",
"servo",
"servodriver"]
diff --git a/tests/wpt/harness/wptrunner/browsers/edge.py b/tests/wpt/harness/wptrunner/browsers/edge.py
new file mode 100644
index 00000000000..7c993fce1ac
--- /dev/null
+++ b/tests/wpt/harness/wptrunner/browsers/edge.py
@@ -0,0 +1,71 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from .base import Browser, ExecutorBrowser, require_arg
+from ..webdriver_server import EdgeDriverServer
+from ..executors import executor_kwargs as base_executor_kwargs
+from ..executors.executorselenium import (SeleniumTestharnessExecutor,
+ SeleniumRefTestExecutor)
+
+__wptrunner__ = {"product": "edge",
+ "check_args": "check_args",
+ "browser": "EdgeBrowser",
+ "executor": {"testharness": "SeleniumTestharnessExecutor",
+ "reftest": "SeleniumRefTestExecutor"},
+ "browser_kwargs": "browser_kwargs",
+ "executor_kwargs": "executor_kwargs",
+ "env_options": "env_options"}
+
+
+def check_args(**kwargs):
+ require_arg(kwargs, "webdriver_binary")
+
+def browser_kwargs(**kwargs):
+ return {"webdriver_binary": kwargs["webdriver_binary"]}
+
+def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
+ **kwargs):
+ from selenium.webdriver import DesiredCapabilities
+
+ executor_kwargs = base_executor_kwargs(test_type, server_config,
+ cache_manager, **kwargs)
+ executor_kwargs["close_after_done"] = True
+ executor_kwargs["capabilities"] = dict(DesiredCapabilities.EDGE.items())
+ return executor_kwargs
+
+def env_options():
+ return {"host": "web-platform.test",
+ "bind_hostname": "true",
+ "supports_debugger": False}
+
+class EdgeBrowser(Browser):
+ used_ports = set()
+
+ def __init__(self, logger, webdriver_binary):
+ Browser.__init__(self, logger)
+ self.server = EdgeDriverServer(self.logger, binary=webdriver_binary)
+ self.webdriver_host = "localhost"
+ self.webdriver_port = self.server.port
+
+ def start(self):
+ print self.server.url
+ self.server.start()
+
+ def stop(self):
+ self.server.stop()
+
+ def pid(self):
+ return self.server.pid
+
+ def is_alive(self):
+ # TODO(ato): This only indicates the server is alive,
+ # and doesn't say anything about whether a browser session
+ # is active.
+ return self.server.is_alive()
+
+ def cleanup(self):
+ self.stop()
+
+ def executor_browser(self):
+ return ExecutorBrowser, {"webdriver_url": self.server.url}
diff --git a/tests/wpt/harness/wptrunner/browsers/firefox.py b/tests/wpt/harness/wptrunner/browsers/firefox.py
index dcd7a4f1d17..a3c6f3f5753 100644
--- a/tests/wpt/harness/wptrunner/browsers/firefox.py
+++ b/tests/wpt/harness/wptrunner/browsers/firefox.py
@@ -3,6 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
+import platform
import subprocess
import sys
@@ -128,10 +129,16 @@ class FirefoxBrowser(Browser):
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False,
- "network.dns.localDomains": ",".join(hostnames)})
+ "network.dns.localDomains": ",".join(hostnames),
+ "places.history.enabled": False})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
+ # Bug 1262954: winxp + e10s, disable hwaccel
+ if (self.e10s and platform.system() in ("Windows", "Microsoft") and
+ '5.1' in platform.version()):
+ self.profile.set_preferences({"layers.acceleration.disabled": True})
+
if self.ca_certificate_path is not None:
self.setup_ssl()
diff --git a/tests/wpt/harness/wptrunner/browsers/servo.py b/tests/wpt/harness/wptrunner/browsers/servo.py
index 1ea017377fe..2eeb5aaa158 100644
--- a/tests/wpt/harness/wptrunner/browsers/servo.py
+++ b/tests/wpt/harness/wptrunner/browsers/servo.py
@@ -6,7 +6,7 @@ import os
from .base import NullBrowser, ExecutorBrowser, require_arg
from ..executors import executor_kwargs as base_executor_kwargs
-from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor
+from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor, ServoWdspecExecutor
here = os.path.join(os.path.split(__file__)[0])
@@ -14,7 +14,8 @@ __wptrunner__ = {"product": "servo",
"check_args": "check_args",
"browser": "ServoBrowser",
"executor": {"testharness": "ServoTestharnessExecutor",
- "reftest": "ServoRefTestExecutor"},
+ "reftest": "ServoRefTestExecutor",
+ "wdspec": "ServoWdspecExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options",
diff --git a/tests/wpt/harness/wptrunner/executors/executormarionette.py b/tests/wpt/harness/wptrunner/executors/executormarionette.py
index c4b1cba689a..8495d3b45ee 100644
--- a/tests/wpt/harness/wptrunner/executors/executormarionette.py
+++ b/tests/wpt/harness/wptrunner/executors/executormarionette.py
@@ -17,11 +17,11 @@ from ..wpttest import WdspecResult, WdspecSubtestResult
errors = None
marionette = None
+pytestrunner = None
webdriver = None
here = os.path.join(os.path.split(__file__)[0])
-from . import pytestrunner
from .base import (ExecutorException,
Protocol,
RefTestExecutor,
@@ -41,7 +41,7 @@ extra_timeout = 5 # seconds
def do_delayed_imports():
- global errors, marionette, webdriver
+ global errors, marionette
# Marionette client used to be called marionette, recently it changed
# to marionette_driver for unfathomable reasons
@@ -51,8 +51,6 @@ def do_delayed_imports():
except ImportError:
from marionette_driver import marionette, errors
- import webdriver
-
class MarionetteProtocol(Protocol):
def __init__(self, executor, browser):
@@ -292,7 +290,7 @@ class RemoteMarionetteProtocol(Protocol):
class ExecuteAsyncScriptRun(object):
def __init__(self, logger, func, marionette, url, timeout):
self.logger = logger
- self.result = None
+ self.result = (None, None)
self.marionette = marionette
self.func = func
self.url = url
@@ -323,11 +321,9 @@ class ExecuteAsyncScriptRun(object):
wait_timeout = None
flag = self.result_flag.wait(wait_timeout)
- if self.result is None:
+ if self.result[1] is None:
self.logger.debug("Timed out waiting for a result")
- assert not flag
self.result = False, ("EXTERNAL-TIMEOUT", None)
-
return self.result
def _run(self):
@@ -409,7 +405,8 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
"timeout": timeout_ms,
"explicit_timeout": timeout is None}
- return marionette.execute_async_script(script, new_sandbox=False)
+ rv = marionette.execute_async_script(script, new_sandbox=False)
+ return rv
class MarionetteRefTestExecutor(RefTestExecutor):
@@ -487,7 +484,7 @@ class MarionetteRefTestExecutor(RefTestExecutor):
class WdspecRun(object):
def __init__(self, func, session, path, timeout):
self.func = func
- self.result = None
+ self.result = (None, None)
self.session = session
self.path = path
self.timeout = timeout
@@ -504,8 +501,7 @@ class WdspecRun(object):
executor.start()
flag = self.result_flag.wait(self.timeout)
- if self.result is None:
- assert not flag
+ if self.result[1] is None:
self.result = False, ("EXTERNAL-TIMEOUT", None)
return self.result
@@ -528,6 +524,7 @@ class WdspecRun(object):
class MarionetteWdspecExecutor(WdspecExecutor):
def __init__(self, browser, server_config, webdriver_binary,
timeout_multiplier=1, close_after_done=True, debug_info=None):
+ self.do_delayed_imports()
WdspecExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
@@ -557,3 +554,8 @@ class MarionetteWdspecExecutor(WdspecExecutor):
harness_result = ("OK", None)
subtest_results = pytestrunner.run(path, session, timeout=timeout)
return (harness_result, subtest_results)
+
+ def do_delayed_imports(self):
+ global pytestrunner, webdriver
+ from . import pytestrunner
+ import webdriver
diff --git a/tests/wpt/harness/wptrunner/executors/executorselenium.py b/tests/wpt/harness/wptrunner/executors/executorselenium.py
index 587c49c7c92..f5d65f499b0 100644
--- a/tests/wpt/harness/wptrunner/executors/executorselenium.py
+++ b/tests/wpt/harness/wptrunner/executors/executorselenium.py
@@ -22,20 +22,21 @@ from .base import (ExecutorException,
strip_server)
from ..testrunner import Stop
-
here = os.path.join(os.path.split(__file__)[0])
webdriver = None
exceptions = None
+RemoteConnection = None
extra_timeout = 5
def do_delayed_imports():
global webdriver
global exceptions
+ global RemoteConnection
from selenium import webdriver
from selenium.common import exceptions
-
+ from selenium.webdriver.remote.remote_connection import RemoteConnection
class SeleniumProtocol(Protocol):
def __init__(self, executor, browser, capabilities, **kwargs):
@@ -53,8 +54,9 @@ class SeleniumProtocol(Protocol):
session_started = False
try:
- self.webdriver = webdriver.Remote(
- self.url, desired_capabilities=self.capabilities)
+ self.webdriver = webdriver.Remote(command_executor=RemoteConnection(self.url.strip("/"),
+ resolve_ip=False),
+ desired_capabilities=self.capabilities)
except:
self.logger.warning(
"Connecting to Selenium failed:\n%s" % traceback.format_exc())
@@ -231,17 +233,7 @@ class SeleniumRefTestExecutor(RefTestExecutor):
def do_test(self, test):
self.logger.info("Test requires OS-level window focus")
- if self.close_after_done and self.has_window:
- self.protocol.webdriver.close()
- self.protocol.webdriver.switch_to_window(
- self.protocol.webdriver.window_handles[-1])
- self.has_window = False
-
- if not self.has_window:
- self.protocol.webdriver.execute_script(self.script)
- self.protocol.webdriver.switch_to_window(
- self.protocol.webdriver.window_handles[-1])
- self.has_window = True
+ self.protocol.webdriver.set_window_size(600, 600)
result = self.implementation.run_test(test)
diff --git a/tests/wpt/harness/wptrunner/executors/executorservo.py b/tests/wpt/harness/wptrunner/executors/executorservo.py
index 15092056a93..b627223a7df 100644
--- a/tests/wpt/harness/wptrunner/executors/executorservo.py
+++ b/tests/wpt/harness/wptrunner/executors/executorservo.py
@@ -4,11 +4,13 @@
import base64
import hashlib
+import httplib
import json
import os
import subprocess
import tempfile
import threading
+import traceback
import urlparse
import uuid
from collections import defaultdict
@@ -19,11 +21,19 @@ from .base import (ExecutorException,
Protocol,
RefTestImplementation,
testharness_result_converter,
- reftest_result_converter)
+ reftest_result_converter,
+ WdspecExecutor)
from .process import ProcessTestExecutor
from ..browsers.base import browser_command
-render_arg = None
+from ..wpttest import WdspecResult, WdspecSubtestResult
+from ..webdriver_server import ServoDriverServer
+from .executormarionette import WdspecRun
+pytestrunner = None
+render_arg = None
+webdriver = None
+
+extra_timeout = 5 # seconds
def do_delayed_imports():
global render_arg
@@ -205,7 +215,7 @@ class ServoRefTestExecutor(ProcessTestExecutor):
self.binary,
[render_arg(self.browser.render_backend), "--hard-fail", "--exit",
"-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
- "--output=%s" % output_path, full_url],
+ "--output=%s" % output_path, full_url] + self.browser.binary_args,
self.debug_info)
for stylesheet in self.browser.user_stylesheets:
@@ -214,10 +224,7 @@ class ServoRefTestExecutor(ProcessTestExecutor):
for pref, value in test.environment.get('prefs', {}).iteritems():
command += ["--pref", "%s=%s" % (pref, value)]
- if viewport_size:
- command += ["--resolution", viewport_size]
- else:
- command += ["--resolution", "800x600"]
+ command += ["--resolution", viewport_size or "800x600"]
if dpi:
command += ["--device-pixel-ratio", dpi]
@@ -278,3 +285,83 @@ class ServoRefTestExecutor(ProcessTestExecutor):
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))
+
+class ServoWdspecProtocol(Protocol):
+ def __init__(self, executor, browser):
+ self.do_delayed_imports()
+ Protocol.__init__(self, executor, browser)
+ self.session = None
+ self.server = None
+
+ def setup(self, runner):
+ try:
+ self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args, render_backend=self.browser.render_backend)
+ self.server.start(block=False)
+ self.logger.info(
+ "WebDriver HTTP server listening at %s" % self.server.url)
+
+ self.logger.info(
+ "Establishing new WebDriver session with %s" % self.server.url)
+ self.session = webdriver.Session(
+ self.server.host, self.server.port, self.server.base_path)
+ except Exception:
+ self.logger.error(traceback.format_exc())
+ self.executor.runner.send_message("init_failed")
+ else:
+ self.executor.runner.send_message("init_succeeded")
+
+ def teardown(self):
+ if self.server is not None:
+ try:
+ if self.session.session_id is not None:
+ self.session.end()
+ except Exception:
+ pass
+ if self.server.is_alive:
+ self.server.stop()
+
+ @property
+ def is_alive(self):
+ conn = httplib.HTTPConnection(self.server.host, self.server.port)
+ conn.request("HEAD", self.server.base_path + "invalid")
+ res = conn.getresponse()
+ return res.status == 404
+
+ def do_delayed_imports(self):
+ global pytestrunner, webdriver
+ from . import pytestrunner
+ import webdriver
+
+
+class ServoWdspecExecutor(WdspecExecutor):
+ def __init__(self, browser, server_config,
+ timeout_multiplier=1, close_after_done=True, debug_info=None,
+ **kwargs):
+ WdspecExecutor.__init__(self, browser, server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.protocol = ServoWdspecProtocol(self, browser)
+
+ def is_alive(self):
+ return self.protocol.is_alive
+
+ def on_environment_change(self, new_environment):
+ pass
+
+ def do_test(self, test):
+ timeout = test.timeout * self.timeout_multiplier + extra_timeout
+
+ success, data = WdspecRun(self.do_wdspec,
+ self.protocol.session,
+ test.path,
+ timeout).run()
+
+ if success:
+ return self.convert_result(test, data)
+
+ return (test.result_cls(*data), [])
+
+ def do_wdspec(self, session, path, timeout):
+ harness_result = ("OK", None)
+ subtest_results = pytestrunner.run(path, session, timeout=timeout)
+ return (harness_result, subtest_results)
diff --git a/tests/wpt/harness/wptrunner/executors/executorservodriver.py b/tests/wpt/harness/wptrunner/executors/executorservodriver.py
index fceeb58fad2..279658d0cd1 100644
--- a/tests/wpt/harness/wptrunner/executors/executorservodriver.py
+++ b/tests/wpt/harness/wptrunner/executors/executorservodriver.py
@@ -14,7 +14,6 @@ from .base import (Protocol,
RefTestImplementation,
TestharnessExecutor,
strip_server)
-from .. import webdriver
from ..testrunner import Stop
webdriver = None
@@ -26,7 +25,7 @@ extra_timeout = 5
def do_delayed_imports():
global webdriver
- import webdriver
+ from tools import webdriver
class ServoWebDriverProtocol(Protocol):
diff --git a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py
index 77afb4a3684..81796d69883 100644
--- a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py
+++ b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py
@@ -3,6 +3,10 @@
# You can obtain one at http://mozilla.org/MPL/2.0/.
import pytest
+import webdriver
+
+import contextlib
+import httplib
"""pytest fixtures for use in Python-based WPT tests.
@@ -17,7 +21,7 @@ class Session(object):
in tests.
The session is not created by default to enable testing of session
- creation. However, a module-scoped session will be implicitly created
+ creation. However, a function-scoped session will be implicitly created
at the first call to a WebDriver command. This means methods such as
`session.send_command` and `session.session_id` are possible to use
without having a session.
@@ -45,14 +49,88 @@ class Session(object):
def test_something(setup, session):
assert session.url == "https://example.org"
- The session is closed when the test module goes out of scope by an
- implicit call to `session.end`.
+ When the test function goes out of scope, any remaining user prompts
+ and opened windows are closed, and the current browsing context is
+ switched back to the top-level browsing context.
"""
def __init__(self, client):
self.client = client
- @pytest.fixture(scope="module")
+ @pytest.fixture(scope="function")
def session(self, request):
- request.addfinalizer(self.client.end)
+ # finalisers are popped off a stack,
+ # making their ordering reverse
+ request.addfinalizer(self.switch_to_top_level_browsing_context)
+ request.addfinalizer(self.restore_windows)
+ request.addfinalizer(self.dismiss_user_prompts)
+
return self.client
+
+ def dismiss_user_prompts(self):
+ """Dismisses any open user prompts in windows."""
+ current_window = self.client.window_handle
+
+ for window in self.windows():
+ self.client.window_handle = window
+ try:
+ self.client.alert.dismiss()
+ except webdriver.NoSuchAlertException:
+ pass
+
+ self.client.window_handle = current_window
+
+ def restore_windows(self):
+ """Closes superfluous windows opened by the test without ending
+ the session implicitly by closing the last window.
+ """
+ current_window = self.client.window_handle
+
+ for window in self.windows(exclude=[current_window]):
+ self.client.window_handle = window
+ if len(self.client.window_handles) > 1:
+ self.client.close()
+
+ self.client.window_handle = current_window
+
+ def switch_to_top_level_browsing_context(self):
+ """If the current browsing context selected by WebDriver is a
+ `` or an `