Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-20 07:08:59 +01:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #10631 - servo:wptrunner-20160415, r=KiChjang

Browse source 
Update wptrunner.

Fixes #10540.
Fixes #10392.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/10631)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2016-04-21 13:54:19 +05:30
commit 9c172f49d0
22 changed files with 732 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

10
python/servo/devenv_commands.py
View file

@ -152,11 +152,17 @@ class MachCommands(CommandBase):
code = call(["git", "init"], env=self.build_env())
if code:
return code
call(
code = call(
["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env())
if code:
return code
code = call(["git", "fetch", "upstream"], env=self.build_env())
if code:
return code
code = call(["git", "reset", '--', "hard", "remotes/upstream/master"], env=self.build_env())
code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=self.build_env())
if code:
return code
code = call(["rm", "-rf", ".git"], env=self.build_env())
if code:
return code
return 0

2
tests/wpt/harness/README.rst
View file

@ -231,7 +231,7 @@ The web-platform-test harness knows about several keys:
Any value indicates that the test is disabled.
`type`
The test type e.g. `testharness` or `reftest`.
The test type e.g. `testharness`, `reftest`, or `wdspec`.
`reftype`
The type of comparison for reftests; either `==` or `!=`.

4
tests/wpt/harness/docs/expectation.rst
View file

@ -203,6 +203,10 @@ When used for expectation data, manifests have the following format:
the (sub)test is disabled and should either not be run (for tests)
or that its results should be ignored (subtests).
* A key ``restart-after`` which can be set to any value to indicate that
the runner should restart the browser after running this test (e.g. to
clear out unwanted state).
* Variables ``debug``, ``os``, ``version``, ``processor`` and
``bits`` that describe the configuration of the browser under
test. ``debug`` is a boolean indicating whether a build is a debug

17
tests/wpt/harness/wptrunner/browsers/chrome.py
View file

@ -3,7 +3,7 @@
# You can obtain one at http://mozilla.org/MPL/2.0/.
from .base import Browser, ExecutorBrowser, require_arg
from .webdriver import ChromedriverLocalServer
from ..webdriver_server import ChromeDriverServer
from ..executors import executor_kwargs as base_executor_kwargs
from ..executors.executorselenium import (SeleniumTestharnessExecutor,
SeleniumRefTestExecutor)
@ -49,32 +49,33 @@ def env_options():
class ChromeBrowser(Browser):
"""Chrome is backed by chromedriver, which is supplied through
``browsers.webdriver.ChromedriverLocalServer``."""
``wptrunner.webdriver.ChromeDriverServer``.
"""
def __init__(self, logger, binary, webdriver_binary="chromedriver"):
"""Creates a new representation of Chrome. The `binary` argument gives
the browser binary to use for testing."""
Browser.__init__(self, logger)
self.binary = binary
self.driver = ChromedriverLocalServer(self.logger, binary=webdriver_binary)
self.server = ChromeDriverServer(self.logger, binary=webdriver_binary)
def start(self):
self.driver.start()
self.server.start(block=False)
def stop(self):
self.driver.stop()
self.server.stop()
def pid(self):
return self.driver.pid
return self.server.pid
def is_alive(self):
# TODO(ato): This only indicates the driver is alive,
# and doesn't say anything about whether a browser session
# is active.
return self.driver.is_alive()
return self.server.is_alive()
def cleanup(self):
self.stop()
def executor_browser(self):
return ExecutorBrowser, {"webdriver_url": self.driver.url}
return ExecutorBrowser, {"webdriver_url": self.server.url}

18
tests/wpt/harness/wptrunner/browsers/firefox.py
View file

@ -13,18 +13,27 @@ from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozcrash import mozcrash
from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg, browser_command
from .base import (get_free_port,
Browser,
ExecutorBrowser,
require_arg,
cmd_arg,
browser_command)
from ..executors import executor_kwargs as base_executor_kwargs
from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor
from ..executors.executormarionette import (MarionetteTestharnessExecutor,
MarionetteRefTestExecutor,
MarionetteWdspecExecutor)
from ..environment import hostnames
here = os.path.join(os.path.split(__file__)[0])
__wptrunner__ = {"product": "firefox",
"check_args": "check_args",
"browser": "FirefoxBrowser",
"executor": {"testharness": "MarionetteTestharnessExecutor",
"reftest": "MarionetteRefTestExecutor"},
"reftest": "MarionetteRefTestExecutor",
"wdspec": "MarionetteWdspecExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options",
@ -62,6 +71,8 @@ def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
executor_kwargs["timeout_multiplier"] = 2
elif run_info_data["debug"] or run_info_data.get("asan"):
executor_kwargs["timeout_multiplier"] = 3
if test_type == "wdspec":
executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
return executor_kwargs
@ -80,6 +91,7 @@ def run_info_extras(**kwargs):
def update_properties():
return ["debug", "e10s", "os", "version", "processor", "bits"], {"debug", "e10s"}
class FirefoxBrowser(Browser):
used_ports = set()
init_timeout = 60

2
tests/wpt/harness/wptrunner/environment.py
View file

@ -32,7 +32,6 @@ def do_delayed_imports(logger, test_paths):
global serve, sslutils
serve_root = serve_path(test_paths)
sys.path.insert(0, serve_root)
failed = []
@ -45,7 +44,6 @@ def do_delayed_imports(logger, test_paths):
try:
import sslutils
except ImportError:
raise
failed.append("sslutils")
if failed:

24
tests/wpt/harness/wptrunner/executors/base.py
View file

@ -63,6 +63,7 @@ class TestharnessResultConverter(object):
[test.subtest_result_cls(name, self.test_codes[status], message, stack)
for name, status, message, stack in subtest_results])
testharness_result_converter = TestharnessResultConverter()
@ -71,11 +72,24 @@ def reftest_result_converter(self, test, result):
extra=result.get("extra")), [])
def pytest_result_converter(self, test, data):
harness_data, subtest_data = data
if subtest_data is None:
subtest_data = []
harness_result = test.result_cls(*harness_data)
subtest_results = [test.subtest_result_cls(*item) for item in subtest_data]
return (harness_result, subtest_results)
class ExecutorException(Exception):
def __init__(self, status, message):
self.status = status
self.message = message
class TestExecutor(object):
__metaclass__ = ABCMeta
@ -116,10 +130,12 @@ class TestExecutor(object):
:param runner: TestRunner instance that is going to run the tests"""
self.runner = runner
if self.protocol is not None:
self.protocol.setup(runner)
def teardown(self):
"""Run cleanup steps after tests have finished"""
if self.protocol is not None:
self.protocol.teardown()
def run_test(self, test):
@ -137,6 +153,7 @@ class TestExecutor(object):
if result is Stop:
return result
# log result of parent test
if result[0].status == "ERROR":
self.logger.debug(result[0].message)
@ -144,7 +161,6 @@ class TestExecutor(object):
self.runner.send_message("test_ended", test, result)
def server_url(self, protocol):
return "%s://%s:%s" % (protocol,
self.server_config["host"],
@ -191,6 +207,7 @@ class RefTestExecutor(TestExecutor):
self.screenshot_cache = screenshot_cache
class RefTestImplementation(object):
def __init__(self, executor):
self.timeout_multiplier = executor.timeout_multiplier
@ -288,6 +305,11 @@ class RefTestImplementation(object):
self.screenshot_cache[key] = hash_val, data
return True, data
class WdspecExecutor(TestExecutor):
convert_result = pytest_result_converter
class Protocol(object):
def __init__(self, executor, browser):
self.executor = executor

188
tests/wpt/harness/wptrunner/executors/executormarionette.py
View file

@ -3,9 +3,9 @@
# You can obtain one at http://mozilla.org/MPL/2.0/.
import hashlib
import httplib
import os
import socket
import sys
import threading
import time
import traceback
@ -13,10 +13,15 @@ import urlparse
import uuid
from collections import defaultdict
from ..wpttest import WdspecResult, WdspecSubtestResult
errors = None
marionette = None
webdriver = None
here = os.path.join(os.path.split(__file__)[0])
from . import pytestrunner
from .base import (ExecutorException,
Protocol,
RefTestExecutor,
@ -25,22 +30,29 @@ from .base import (ExecutorException,
TestharnessExecutor,
testharness_result_converter,
reftest_result_converter,
strip_server)
strip_server,
WdspecExecutor)
from ..testrunner import Stop
from ..webdriver_server import GeckoDriverServer
# Extra timeout to use after internal test timeout at which the harness
# should force a timeout
extra_timeout = 5 # seconds
def do_delayed_imports():
global marionette
global errors
global errors, marionette, webdriver
# Marionette client used to be called marionette, recently it changed
# to marionette_driver for unfathomable reasons
try:
import marionette
from marionette import errors
except ImportError:
from marionette_driver import marionette, errors
import webdriver
class MarionetteProtocol(Protocol):
def __init__(self, executor, browser):
@ -54,8 +66,10 @@ class MarionetteProtocol(Protocol):
"""Connect to browser via Marionette."""
Protocol.setup(self, runner)
self.logger.debug("Connecting to marionette on port %i" % self.marionette_port)
self.marionette = marionette.Marionette(host='localhost', port=self.marionette_port)
self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
self.marionette = marionette.Marionette(host='localhost',
port=self.marionette_port,
socket_timeout=None)
# XXX Move this timeout somewhere
self.logger.debug("Waiting for Marionette connection")
@ -97,10 +111,10 @@ class MarionetteProtocol(Protocol):
pass
del self.marionette
@property
def is_alive(self):
"""Check if the marionette connection is still active"""
"""Check if the Marionette connection is still active."""
try:
# Get a simple property over the connection
self.marionette.current_window_handle
except Exception:
return False
@ -126,12 +140,18 @@ class MarionetteProtocol(Protocol):
"document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
def wait(self):
socket_timeout = self.marionette.client.sock.gettimeout()
if socket_timeout:
self.marionette.set_script_timeout((socket_timeout / 2) * 1000)
while True:
try:
self.marionette.execute_async_script("");
self.marionette.execute_async_script("")
except errors.ScriptTimeoutException:
self.logger.debug("Script timed out")
pass
except (socket.timeout, IOError):
self.logger.debug("Socket closed")
break
except Exception as e:
self.logger.error(traceback.format_exc(e))
@ -213,7 +233,63 @@ class MarionetteProtocol(Protocol):
with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
self.marionette.execute_script(script)
class MarionetteRun(object):
class RemoteMarionetteProtocol(Protocol):
def __init__(self, executor, browser):
do_delayed_imports()
Protocol.__init__(self, executor, browser)
self.session = None
self.webdriver_binary = executor.webdriver_binary
self.marionette_port = browser.marionette_port
self.server = None
def setup(self, runner):
"""Connect to browser via the Marionette HTTP server."""
try:
self.server = GeckoDriverServer(
self.logger, self.marionette_port, binary=self.webdriver_binary)
self.server.start(block=False)
self.logger.info(
"WebDriver HTTP server listening at %s" % self.server.url)
self.logger.info(
"Establishing new WebDriver session with %s" % self.server.url)
self.session = webdriver.Session(
self.server.host, self.server.port, self.server.base_path)
except Exception:
self.logger.error(traceback.format_exc())
self.executor.runner.send_message("init_failed")
else:
self.executor.runner.send_message("init_succeeded")
def teardown(self):
try:
if self.session.session_id is not None:
self.session.end()
except Exception:
pass
if self.server is not None and self.server.is_alive:
self.server.stop()
@property
def is_alive(self):
"""Test that the Marionette connection is still alive.
Because the remote communication happens over HTTP we need to
make an explicit request to the remote. It is allowed for
WebDriver spec tests to not have a WebDriver session, since this
may be what is tested.
An HTTP request to an invalid path that results in a 404 is
proof enough to us that the server is alive and kicking.
"""
conn = httplib.HTTPConnection(self.server.host, self.server.port)
conn.request("HEAD", self.server.base_path + "invalid")
res = conn.getresponse()
return res.status == 404
class ExecuteAsyncScriptRun(object):
def __init__(self, logger, func, marionette, url, timeout):
self.logger = logger
self.result = None
@ -277,8 +353,8 @@ class MarionetteRun(object):
class MarionetteTestharnessExecutor(TestharnessExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1, close_after_done=True,
debug_info=None):
def __init__(self, browser, server_config, timeout_multiplier=1,
close_after_done=True, debug_info=None, **kwargs):
"""Marionette-based executor for testharness.js tests"""
TestharnessExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
@ -295,7 +371,7 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
do_delayed_imports()
def is_alive(self):
return self.protocol.is_alive()
return self.protocol.is_alive
def on_environment_change(self, new_environment):
self.protocol.on_environment_change(self.last_environment, new_environment)
@ -307,7 +383,7 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
else None)
success, data = MarionetteRun(self.logger,
success, data = ExecuteAsyncScriptRun(self.logger,
self.do_testharness,
self.protocol.marionette,
self.test_url(test),
@ -338,7 +414,9 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
class MarionetteRefTestExecutor(RefTestExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
screenshot_cache=None, close_after_done=True, debug_info=None):
screenshot_cache=None, close_after_done=True,
debug_info=None, **kwargs):
"""Marionette-based executor for reftests"""
RefTestExecutor.__init__(self,
browser,
@ -358,7 +436,7 @@ class MarionetteRefTestExecutor(RefTestExecutor):
self.wait_script = f.read()
def is_alive(self):
return self.protocol.is_alive()
return self.protocol.is_alive
def on_environment_change(self, new_environment):
self.protocol.on_environment_change(self.last_environment, new_environment)
@ -376,7 +454,6 @@ class MarionetteRefTestExecutor(RefTestExecutor):
self.has_window = True
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def screenshot(self, test, viewport_size, dpi):
@ -388,7 +465,7 @@ class MarionetteRefTestExecutor(RefTestExecutor):
test_url = self.test_url(test)
return MarionetteRun(self.logger,
return ExecuteAsyncScriptRun(self.logger,
self._screenshot,
self.protocol.marionette,
test_url,
@ -405,3 +482,78 @@ class MarionetteRefTestExecutor(RefTestExecutor):
screenshot = screenshot.split(",", 1)[1]
return screenshot
class WdspecRun(object):
def __init__(self, func, session, path, timeout):
self.func = func
self.result = None
self.session = session
self.path = path
self.timeout = timeout
self.result_flag = threading.Event()
def run(self):
"""Runs function in a thread and interrupts it if it exceeds the
given timeout. Returns (True, (Result, [SubtestResult ...])) in
case of success, or (False, (status, extra information)) in the
event of failure.
"""
executor = threading.Thread(target=self._run)
executor.start()
flag = self.result_flag.wait(self.timeout)
if self.result is None:
assert not flag
self.result = False, ("EXTERNAL-TIMEOUT", None)
return self.result
def _run(self):
try:
self.result = True, self.func(self.session, self.path, self.timeout)
except (socket.timeout, IOError):
self.result = False, ("CRASH", None)
except Exception as e:
message = getattr(e, "message")
if message:
message += "\n"
message += traceback.format_exc(e)
self.result = False, ("ERROR", message)
finally:
self.result_flag.set()
class MarionetteWdspecExecutor(WdspecExecutor):
def __init__(self, browser, server_config, webdriver_binary,
timeout_multiplier=1, close_after_done=True, debug_info=None):
WdspecExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.webdriver_binary = webdriver_binary
self.protocol = RemoteMarionetteProtocol(self, browser)
def is_alive(self):
return self.protocol.is_alive
def on_environment_change(self, new_environment):
pass
def do_test(self, test):
timeout = test.timeout * self.timeout_multiplier + extra_timeout
success, data = WdspecRun(self.do_wdspec,
self.protocol.session,
test.path,
timeout).run()
if success:
return self.convert_result(test, data)
return (test.result_cls(*data), [])
def do_wdspec(self, session, path, timeout):
harness_result = ("OK", None)
subtest_results = pytestrunner.run(path, session, timeout=timeout)
return (harness_result, subtest_results)

4
tests/wpt/harness/wptrunner/executors/executorservo.py
View file

@ -71,7 +71,7 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
self.result_flag = threading.Event()
args = [render_arg(self.browser.render_backend), "--hard-fail", "-u", "Servo/wptrunner",
"-z", self.test_url(test)]
"-Z", "replace-surrogates", "-z", self.test_url(test)]
for stylesheet in self.browser.user_stylesheets:
args += ["--user-stylesheet", stylesheet]
for pref, value in test.environment.get('prefs', {}).iteritems():
@ -204,7 +204,7 @@ class ServoRefTestExecutor(ProcessTestExecutor):
debug_args, command = browser_command(
self.binary,
[render_arg(self.browser.render_backend), "--hard-fail", "--exit",
"-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously",
"-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
"--output=%s" % output_path, full_url],
self.debug_info)

15
tests/wpt/harness/wptrunner/executors/executorservodriver.py
View file

@ -14,16 +14,24 @@ from .base import (Protocol,
RefTestImplementation,
TestharnessExecutor,
strip_server)
import webdriver
from .. import webdriver
from ..testrunner import Stop
webdriver = None
here = os.path.join(os.path.split(__file__)[0])
extra_timeout = 5
def do_delayed_imports():
global webdriver
import webdriver
class ServoWebDriverProtocol(Protocol):
def __init__(self, executor, browser, capabilities, **kwargs):
do_delayed_imports()
Protocol.__init__(self, executor, browser)
self.capabilities = capabilities
self.host = browser.webdriver_host
@ -34,10 +42,11 @@ class ServoWebDriverProtocol(Protocol):
"""Connect to browser via WebDriver."""
self.runner = runner
url = "http://%s:%d" % (self.host, self.port)
session_started = False
try:
self.session = webdriver.Session(self.host, self.port,
extension=webdriver.ServoExtensions)
extension=webdriver.servo.ServoCommandExtensions)
self.session.start()
except:
self.logger.warning(
@ -62,7 +71,7 @@ class ServoWebDriverProtocol(Protocol):
def is_alive(self):
try:
# Get a simple property over the connection
self.session.handle
self.session.window_handle
# TODO what exception?
except Exception:
return False

6
tests/wpt/harness/wptrunner/executors/pytestrunner/__init__.py Normal file
View file

@ -0,0 +1,6 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from . import fixtures
from .runner import run

58
tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py Normal file
View file

@ -0,0 +1,58 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import pytest
"""pytest fixtures for use in Python-based WPT tests.
The purpose of test fixtures is to provide a fixed baseline upon which
tests can reliably and repeatedly execute.
"""
class Session(object):
"""Fixture to allow access to wptrunner's existing WebDriver session
in tests.
The session is not created by default to enable testing of session
creation. However, a module-scoped session will be implicitly created
at the first call to a WebDriver command. This means methods such as
`session.send_command` and `session.session_id` are possible to use
without having a session.
To illustrate implicit session creation::
def test_session_scope(session):
# at this point there is no session
assert session.session_id is None
# window_id is a WebDriver command,
# and implicitly creates the session for us
assert session.window_id is not None
# we now have a session
assert session.session_id is not None
You can also access the session in custom fixtures defined in the
tests, such as a setup function::
@pytest.fixture(scope="function")
def setup(request, session):
session.url = "https://example.org"
def test_something(setup, session):
assert session.url == "https://example.org"
The session is closed when the test module goes out of scope by an
implicit call to `session.end`.
"""
def __init__(self, client):
self.client = client
@pytest.fixture(scope="module")
def session(self, request):
request.addfinalizer(self.client.end)
return self.client

113
tests/wpt/harness/wptrunner/executors/pytestrunner/runner.py Normal file
View file

@ -0,0 +1,113 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""Provides interface to deal with pytest.
Usage::
session = webdriver.client.Session("127.0.0.1", "4444", "/")
harness_result = ("OK", None)
subtest_results = pytestrunner.run("/path/to/test", session.url)
return (harness_result, subtest_results)
"""
import errno
import shutil
import tempfile
from . import fixtures
pytest = None
def do_delayed_imports():
global pytest
import pytest
def run(path, session, timeout=0):
"""Run Python test at ``path`` in pytest. The provided ``session``
is exposed as a fixture available in the scope of the test functions.
:param path: Path to the test file.
:param session: WebDriver session to expose.
:param timeout: Duration before interrupting potentially hanging
tests. If 0, there is no timeout.
:returns: List of subtest results, which are tuples of (test id,
status, message, stacktrace).
"""
if pytest is None:
do_delayed_imports()
recorder = SubtestResultRecorder()
plugins = [recorder,
fixtures.Session(session)]
# TODO(ato): Deal with timeouts
with TemporaryDirectory() as cache:
pytest.main(["--strict", # turn warnings into errors
"--verbose", # show each individual subtest
"--capture", "no", # enable stdout/stderr from tests
"--basetemp", cache, # temporary directory
path],
plugins=plugins)
return recorder.results
class SubtestResultRecorder(object):
def __init__(self):
self.results = []
def pytest_runtest_logreport(self, report):
if report.passed and report.when == "call":
self.record_pass(report)
elif report.failed:
if report.when != "call":
self.record_error(report)
else:
self.record_fail(report)
elif report.skipped:
self.record_skip(report)
def record_pass(self, report):
self.record(report.nodeid, "PASS")
def record_fail(self, report):
self.record(report.nodeid, "FAIL", stack=report.longrepr)
def record_error(self, report):
# error in setup/teardown
if report.when != "call":
message = "%s error" % report.when
self.record(report.nodeid, "ERROR", message, report.longrepr)
def record_skip(self, report):
self.record(report.nodeid, "ERROR",
"In-test skip decorators are disallowed, "
"please use WPT metadata to ignore tests.")
def record(self, test, status, message=None, stack=None):
if stack is not None:
stack = str(stack)
new_result = (test, status, message, stack)
self.results.append(new_result)
class TemporaryDirectory(object):
def __enter__(self):
self.path = tempfile.mkdtemp(prefix="pytest-")
return self.path
def __exit__(self, *args):
try:
shutil.rmtree(self.path)
except OSError as e:
# no such file or directory
if e.errno != errno.ENOENT:
raise

24
tests/wpt/harness/wptrunner/manifestexpected.py
View file

@ -29,10 +29,10 @@ def data_cls_getter(output_node, visited_node):
raise ValueError
def disabled(node):
"""Boolean indicating whether the test is disabled"""
def bool_prop(name, node):
"""Boolean property"""
try:
return node.get("disabled")
return node.get(name)
except KeyError:
return None
@ -109,7 +109,11 @@ class ExpectedManifest(ManifestItem):
@property
def disabled(self):
return disabled(self)
return bool_prop("disabled", self)
@property
def restart_after(self):
return bool_prop("restart-after", self)
@property
def tags(self):
@ -123,7 +127,11 @@ class ExpectedManifest(ManifestItem):
class DirectoryManifest(ManifestItem):
@property
def disabled(self):
return disabled(self)
return bool_prop("disabled", self)
@property
def restart_after(self):
return bool_prop("restart-after", self)
@property
def tags(self):
@ -164,7 +172,11 @@ class TestNode(ManifestItem):
@property
def disabled(self):
return disabled(self)
return bool_prop("disabled", self)
@property
def restart_after(self):
return bool_prop("restart-after", self)
@property
def tags(self):

3
tests/wpt/harness/wptrunner/testrunner.py
View file

@ -524,7 +524,8 @@ class TestRunnerManager(threading.Thread):
self.test = None
restart_before_next = (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
restart_before_next = (test.restart_after or
file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
subtest_unexpected or is_unexpected)
if (self.pause_after_test or

206
tests/wpt/harness/wptrunner/webdriver_server.py Normal file
View file

@ -0,0 +1,206 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import abc
import errno
import os
import platform
import socket
import threading
import time
import traceback
import urlparse
import mozprocess
__all__ = ["SeleniumServer", "ChromeDriverServer",
"GeckoDriverServer", "WebDriverServer"]
class WebDriverServer(object):
__metaclass__ = abc.ABCMeta
default_base_path = "/"
_used_ports = set()
def __init__(self, logger, binary, host="127.0.0.1", port=None,
base_path="", env=None):
self.logger = logger
self.binary = binary
self.host = host
if base_path == "":
self.base_path = self.default_base_path
else:
self.base_path = base_path
self.env = os.environ.copy() if env is None else env
self._port = port
self._cmd = None
self._proc = None
@abc.abstractmethod
def make_command(self):
"""Returns the full command for starting the server process as a list."""
def start(self, block=True):
try:
self._run(block)
except KeyboardInterrupt:
self.stop()
def _run(self, block):
self._cmd = self.make_command()
self._proc = mozprocess.ProcessHandler(
self._cmd,
processOutputLine=self.on_output,
env=self.env,
storeOutput=False)
try:
self._proc.run()
except OSError as e:
if e.errno == errno.ENOENT:
raise IOError(
"WebDriver HTTP server executable not found: %s" % self.binary)
raise
self.logger.debug(
"Waiting for server to become accessible: %s" % self.url)
try:
wait_for_service((self.host, self.port))
except:
self.logger.error(
"WebDriver HTTP server was not accessible "
"within the timeout:\n%s" % traceback.format_exc())
raise
if block:
self._proc.wait()
def stop(self):
if self.is_alive:
return self._proc.kill()
return not self.is_alive
@property
def is_alive(self):
return (self._proc is not None and
self._proc.proc is not None and
self._proc.poll() is None)
def on_output(self, line):
self.logger.process_output(self.pid,
line.decode("utf8", "replace"),
command=" ".join(self._cmd))
@property
def pid(self):
if self._proc is not None:
return self._proc.pid
@property
def url(self):
return "http://%s:%i%s" % (self.host, self.port, self.base_path)
@property
def port(self):
if self._port is None:
self._port = self._find_next_free_port()
return self._port
@staticmethod
def _find_next_free_port():
port = get_free_port(4444, exclude=WebDriverServer._used_ports)
WebDriverServer._used_ports.add(port)
return port
class SeleniumServer(WebDriverServer):
default_base_path = "/wd/hub"
def make_command(self):
return ["java", "-jar", self.binary, "-port", str(self.port)]
class ChromeDriverServer(WebDriverServer):
default_base_path = "/wd/hub"
def __init__(self, logger, binary="chromedriver", port=None,
base_path=""):
WebDriverServer.__init__(
self, logger, binary, port=port, base_path=base_path)
def make_command(self):
return [self.binary,
cmd_arg("port", str(self.port)),
cmd_arg("url-base", self.base_path) if self.base_path else ""]
class GeckoDriverServer(WebDriverServer):
def __init__(self, logger, marionette_port=2828, binary="wires",
host="127.0.0.1", port=None):
env = os.environ.copy()
env["RUST_BACKTRACE"] = "1"
WebDriverServer.__init__(self, logger, binary, host=host, port=port, env=env)
self.marionette_port = marionette_port
def make_command(self):
return [self.binary,
"--connect-existing",
"--marionette-port", str(self.marionette_port),
"--webdriver-host", self.host,
"--webdriver-port", str(self.port)]
def cmd_arg(name, value=None):
prefix = "-" if platform.system() == "Windows" else "--"
rv = prefix + name
if value is not None:
rv += "=" + value
return rv
def get_free_port(start_port, exclude=None):
"""Get the first port number after start_port (inclusive) that is
not currently bound.
:param start_port: Integer port number at which to start testing.
:param exclude: Set of port numbers to skip"""
port = start_port
while True:
if exclude and port in exclude:
port += 1
continue
s = socket.socket()
try:
s.bind(("127.0.0.1", port))
except socket.error:
port += 1
else:
return port
finally:
s.close()
def wait_for_service(addr, timeout=15):
"""Waits until network service given as a tuple of (host, port) becomes
available or the `timeout` duration is reached, at which point
``socket.error`` is raised."""
end = time.time() + timeout
while end > time.time():
so = socket.socket()
try:
so.connect(addr)
except socket.timeout:
pass
except socket.error as e:
if e[0] != errno.ECONNREFUSED:
raise
else:
return True
finally:
so.close()
time.sleep(0.5)
raise socket.error("Service is unavailable: %s:%i" % addr)

12
tests/wpt/harness/wptrunner/wptcommandline.py
View file

@ -10,6 +10,7 @@ from collections import OrderedDict
from distutils.spawn import find_executable
import config
import wpttest
def abs_path(path):
@ -25,6 +26,7 @@ def url_or_path(path):
else:
return abs_path(path)
def require_arg(kwargs, name, value_func=None):
if value_func is None:
value_func = lambda x: x is not None
@ -101,8 +103,8 @@ def create_parser(product_choices=None):
test_selection_group = parser.add_argument_group("Test Selection")
test_selection_group.add_argument("--test-types", action="store",
nargs="*", default=["testharness", "reftest"],
choices=["testharness", "reftest"],
nargs="*", default=wpttest.enabled_tests,
choices=wpttest.enabled_tests,
help="Test types to run")
test_selection_group.add_argument("--include", action="append",
help="URL prefix to include")
@ -159,8 +161,8 @@ def create_parser(product_choices=None):
gecko_group = parser.add_argument_group("Gecko-specific")
gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
help="Path to the folder containing browser prefs")
gecko_group.add_argument("--e10s", dest="gecko_e10s", action="store_true",
help="Run tests with electrolysis preferences")
gecko_group.add_argument("--disable-e10s", dest="gecko_e10s", action="store_false", default=True,
help="Run tests without electrolysis preferences")
b2g_group = parser.add_argument_group("B2G-specific")
b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False,
@ -343,12 +345,14 @@ def check_args(kwargs):
return kwargs
def check_args_update(kwargs):
set_from_config(kwargs)
if kwargs["product"] is None:
kwargs["product"] = "firefox"
def create_parser_update(product_choices=None):
from mozlog.structured import commandline

2
tests/wpt/harness/wptrunner/wptlogging.py
View file

@ -74,6 +74,7 @@ class LoggingWrapper(StringIO):
instead"""
def __init__(self, queue, prefix=None):
StringIO.__init__(self)
self.queue = queue
self.prefix = prefix
@ -94,6 +95,7 @@ class LoggingWrapper(StringIO):
def flush(self):
pass
class CaptureIO(object):
def __init__(self, logger, do_capture):
self.logger = logger

46
tests/wpt/harness/wptrunner/wpttest.py
View file

@ -12,6 +12,8 @@ import mozinfo
from wptmanifest.parser import atoms
atom_reset = atoms["Reset"]
enabled_tests = set(["testharness", "reftest", "wdspec"])
class Result(object):
def __init__(self, status, message, expected=None, extra=None):
@ -22,6 +24,9 @@ class Result(object):
self.expected = expected
self.extra = extra
def __repr__(self):
return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status)
class SubtestResult(object):
def __init__(self, name, status, message, stack=None, expected=None):
@ -33,20 +38,33 @@ class SubtestResult(object):
self.stack = stack
self.expected = expected
def __repr__(self):
return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status)
class TestharnessResult(Result):
default_expected = "OK"
statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
class TestharnessSubtestResult(SubtestResult):
default_expected = "PASS"
statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"])
class ReftestResult(Result):
default_expected = "PASS"
statuses = set(["PASS", "FAIL", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
class TestharnessSubtestResult(SubtestResult):
class WdspecResult(Result):
default_expected = "OK"
statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
class WdspecSubtestResult(SubtestResult):
default_expected = "PASS"
statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"])
statuses = set(["PASS", "FAIL", "ERROR"])
def get_run_info(metadata_root, product, **kwargs):
@ -82,6 +100,7 @@ class RunInfo(dict):
mozinfo.find_and_update_from_json(*dirs)
class B2GRunInfo(RunInfo):
def __init__(self, *args, **kwargs):
RunInfo.__init__(self, *args, **kwargs)
@ -115,7 +134,6 @@ class Test(object):
path=manifest_item.path,
protocol="https" if hasattr(manifest_item, "https") and manifest_item.https else "http")
@property
def id(self):
return self.url
@ -141,7 +159,6 @@ class Test(object):
if subtest_meta is not None:
yield subtest_meta
def disabled(self, subtest=None):
for meta in self.itermeta(subtest):
disabled = meta.disabled
@ -149,6 +166,14 @@ class Test(object):
return disabled
return None
@property
def restart_after(self):
for meta in self.itermeta(None):
restart_after = meta.restart_after
if restart_after is not None:
return True
return False
@property
def tags(self):
tags = set()
@ -191,6 +216,9 @@ class Test(object):
except KeyError:
return default
def __repr__(self):
return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id)
class TestharnessTest(Test):
result_cls = TestharnessResult
@ -293,12 +321,18 @@ class ReftestTest(Test):
return ("reftype", "refurl")
class WdspecTest(Test):
result_cls = WdspecResult
subtest_result_cls = WdspecSubtestResult
test_type = "wdspec"
manifest_test_cls = {"reftest": ReftestTest,
"testharness": TestharnessTest,
"manual": ManualTest}
"manual": ManualTest,
"wdspec": WdspecTest}
def from_manifest(manifest_test, inherit_metadata, test_metadata):
test_cls = manifest_test_cls[manifest_test.item_type]
return test_cls.from_manifest(manifest_test, inherit_metadata, test_metadata)

13
tests/wpt/metadata-css/cssom-1_dev/html/escape.htm.ini
View file

@ -1,3 +1,14 @@
[escape.htm]
type: testharness
expected: CRASH
[Null bytes]
expected: FAIL
bug: https://github.com/servo/servo/issues/10685
[Various tests]
expected: FAIL
bug: https://github.com/servo/servo/issues/10685
[Surrogates]
expected: FAIL
bug: https://github.com/servo/servo/issues/6564

3
tests/wpt/metadata/webgl/conformance-1.0.3/conformance/programs/invalid-UTF-16.html.ini
View file

@ -1,3 +0,0 @@
[invalid-UTF-16.html]
type: testharness
expected: CRASH

18
tests/wpt/metadata/webstorage/storage_setitem.html.ini
View file

@ -1,3 +1,19 @@
[storage_setitem.html]
type: testharness
expected: CRASH
expected: TIMEOUT
[localStorage[\] = "<22>"]
expected: FAIL
bug: https://github.com/servo/servo/issues/6564
[localStorage[\] = "<22>a"]
expected: FAIL
bug: https://github.com/servo/servo/issues/6564
[localStorage[\] = "a<>"]
expected: FAIL
bug: https://github.com/servo/servo/issues/6564
[localStorage["0"\]]
expected: TIMEOUT
bug: https://github.com/servo/servo/issues/10686