From 906b7b33efc788d169a4d746b8a2a90c136d9b73 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Mon, 4 Jul 2016 10:29:30 -0400 Subject: [PATCH 1/4] Rename mach command to upgrade wptrunner and make it work more than once. --- python/servo/devenv_commands.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py index 5a41b8b1b37..c99012a9a41 100644 --- a/python/servo/devenv_commands.py +++ b/python/servo/devenv_commands.py @@ -170,7 +170,7 @@ class MachCommands(CommandBase): print(cargo_path) call(["cargo", "fetch"], env=self.build_env()) - @Command('wpt-upgrade', + @Command('wptrunner-upgrade', description='upgrade wptrunner.', category='devenv') def upgrade_wpt_runner(self): @@ -178,6 +178,8 @@ class MachCommands(CommandBase): code = call(["git", "init"], env=self.build_env()) if code: return code + # No need to report an error if this fails, as it will for the first use + call(["git", "remote", "rm", "upstream"], env=self.build_env()) code = call( ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env()) if code: From a2bc9d77758d8c851636e7721b788f22dc4df80d Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Thu, 7 Jul 2016 12:06:54 -0400 Subject: [PATCH 2/4] Avoid #12321 by creating environment before switching directories. --- python/servo/devenv_commands.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py index c99012a9a41..b7a40858165 100644 --- a/python/servo/devenv_commands.py +++ b/python/servo/devenv_commands.py @@ -174,23 +174,24 @@ class MachCommands(CommandBase): description='upgrade wptrunner.', category='devenv') def upgrade_wpt_runner(self): + env = self.build_env() with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')): - code = call(["git", "init"], env=self.build_env()) + code = call(["git", "init"], env=env) if code: return code # No need to report an error if this fails, as it will for the first use - call(["git", "remote", "rm", "upstream"], env=self.build_env()) + call(["git", "remote", "rm", "upstream"], env=env) code = call( - ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env()) + ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=env) if code: return code - code = call(["git", "fetch", "upstream"], env=self.build_env()) + code = call(["git", "fetch", "upstream"], env=env) if code: return code - code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=self.build_env()) + code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=env) if code: return code - code = call(["rm", "-rf", ".git"], env=self.build_env()) + code = call(["rm", "-rf", ".git"], env=env) if code: return code return 0 From d6318c8ea5dc3c8e249631973a444ba95310c676 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Mon, 3 Oct 2016 12:43:22 -0400 Subject: [PATCH 3/4] Upgrade wptrunner to 83fb6b727817a7dfeb24763d099aca95bc674b8e. --- tests/wpt/harness/.travis.yml | 20 ++++ tests/wpt/harness/tox.ini | 15 +++ .../harness/wptrunner/browsers/__init__.py | 4 +- tests/wpt/harness/wptrunner/browsers/edge.py | 71 ++++++++++++ .../wpt/harness/wptrunner/browsers/firefox.py | 9 +- tests/wpt/harness/wptrunner/browsers/servo.py | 5 +- .../wptrunner/executors/executormarionette.py | 26 +++-- .../wptrunner/executors/executorselenium.py | 22 ++-- .../wptrunner/executors/executorservo.py | 101 +++++++++++++++-- .../executors/executorservodriver.py | 3 +- .../executors/pytestrunner/fixtures.py | 88 ++++++++++++++- .../executors/pytestrunner/runner.py | 1 + .../executors/testharness_webdriver.js | 14 ++- tests/wpt/harness/wptrunner/testloader.py | 1 + tests/wpt/harness/wptrunner/testrunner.py | 9 +- .../harness/wptrunner/tests/test_chunker.py | 37 ++++--- .../wpt/harness/wptrunner/tests/test_hosts.py | 5 +- .../harness/wptrunner/tests/test_update.py | 10 ++ .../wpt/harness/wptrunner/webdriver_server.py | 46 ++++++-- tests/wpt/harness/wptrunner/wptcommandline.py | 104 +++++++++--------- .../harness/wptrunner/wptmanifest/parser.py | 9 +- .../wptmanifest/tests/test_serializer.py | 4 + tests/wpt/harness/wptrunner/wptrunner.py | 1 + tests/wpt/harness/wptrunner/wpttest.py | 13 +-- 24 files changed, 478 insertions(+), 140 deletions(-) create mode 100644 tests/wpt/harness/.travis.yml create mode 100644 tests/wpt/harness/tox.ini create mode 100644 tests/wpt/harness/wptrunner/browsers/edge.py diff --git a/tests/wpt/harness/.travis.yml b/tests/wpt/harness/.travis.yml new file mode 100644 index 00000000000..add6efd12ca --- /dev/null +++ b/tests/wpt/harness/.travis.yml @@ -0,0 +1,20 @@ +language: python +python: 2.7 + +sudo: false + +cache: + directories: + - $HOME/.cache/pip + +env: + - TOXENV="{py27,pypy}-base" + - TOXENV="{py27,pypy}-chrome" + - TOXENV="{py27,pypy}-firefox" + - TOXENV="{py27,pypy}-servo" + +install: + - pip install -U tox + +script: + - tox diff --git a/tests/wpt/harness/tox.ini b/tests/wpt/harness/tox.ini new file mode 100644 index 00000000000..844a8d05b20 --- /dev/null +++ b/tests/wpt/harness/tox.ini @@ -0,0 +1,15 @@ +[pytest] +xfail_strict=true + +[tox] +envlist = {py27,pypy}-{base,b2g,chrome,firefox,servo} + +[testenv] +deps = + pytest>=2.9 + -r{toxinidir}/requirements.txt + chrome: -r{toxinidir}/requirements_chrome.txt + firefox: -r{toxinidir}/requirements_firefox.txt + servo: -r{toxinidir}/requirements_servo.txt + +commands = py.test [] diff --git a/tests/wpt/harness/wptrunner/browsers/__init__.py b/tests/wpt/harness/wptrunner/browsers/__init__.py index ffc5aedc830..8b34cc3963f 100644 --- a/tests/wpt/harness/wptrunner/browsers/__init__.py +++ b/tests/wpt/harness/wptrunner/browsers/__init__.py @@ -26,8 +26,8 @@ All classes and functions named in the above dict must be imported into the module global scope. """ -product_list = ["b2g", - "chrome", +product_list = ["chrome", + "edge", "firefox", "servo", "servodriver"] diff --git a/tests/wpt/harness/wptrunner/browsers/edge.py b/tests/wpt/harness/wptrunner/browsers/edge.py new file mode 100644 index 00000000000..7c993fce1ac --- /dev/null +++ b/tests/wpt/harness/wptrunner/browsers/edge.py @@ -0,0 +1,71 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from .base import Browser, ExecutorBrowser, require_arg +from ..webdriver_server import EdgeDriverServer +from ..executors import executor_kwargs as base_executor_kwargs +from ..executors.executorselenium import (SeleniumTestharnessExecutor, + SeleniumRefTestExecutor) + +__wptrunner__ = {"product": "edge", + "check_args": "check_args", + "browser": "EdgeBrowser", + "executor": {"testharness": "SeleniumTestharnessExecutor", + "reftest": "SeleniumRefTestExecutor"}, + "browser_kwargs": "browser_kwargs", + "executor_kwargs": "executor_kwargs", + "env_options": "env_options"} + + +def check_args(**kwargs): + require_arg(kwargs, "webdriver_binary") + +def browser_kwargs(**kwargs): + return {"webdriver_binary": kwargs["webdriver_binary"]} + +def executor_kwargs(test_type, server_config, cache_manager, run_info_data, + **kwargs): + from selenium.webdriver import DesiredCapabilities + + executor_kwargs = base_executor_kwargs(test_type, server_config, + cache_manager, **kwargs) + executor_kwargs["close_after_done"] = True + executor_kwargs["capabilities"] = dict(DesiredCapabilities.EDGE.items()) + return executor_kwargs + +def env_options(): + return {"host": "web-platform.test", + "bind_hostname": "true", + "supports_debugger": False} + +class EdgeBrowser(Browser): + used_ports = set() + + def __init__(self, logger, webdriver_binary): + Browser.__init__(self, logger) + self.server = EdgeDriverServer(self.logger, binary=webdriver_binary) + self.webdriver_host = "localhost" + self.webdriver_port = self.server.port + + def start(self): + print self.server.url + self.server.start() + + def stop(self): + self.server.stop() + + def pid(self): + return self.server.pid + + def is_alive(self): + # TODO(ato): This only indicates the server is alive, + # and doesn't say anything about whether a browser session + # is active. + return self.server.is_alive() + + def cleanup(self): + self.stop() + + def executor_browser(self): + return ExecutorBrowser, {"webdriver_url": self.server.url} diff --git a/tests/wpt/harness/wptrunner/browsers/firefox.py b/tests/wpt/harness/wptrunner/browsers/firefox.py index dcd7a4f1d17..a3c6f3f5753 100644 --- a/tests/wpt/harness/wptrunner/browsers/firefox.py +++ b/tests/wpt/harness/wptrunner/browsers/firefox.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os +import platform import subprocess import sys @@ -128,10 +129,16 @@ class FirefoxBrowser(Browser): self.profile.set_preferences({"marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False, - "network.dns.localDomains": ",".join(hostnames)}) + "network.dns.localDomains": ",".join(hostnames), + "places.history.enabled": False}) if self.e10s: self.profile.set_preferences({"browser.tabs.remote.autostart": True}) + # Bug 1262954: winxp + e10s, disable hwaccel + if (self.e10s and platform.system() in ("Windows", "Microsoft") and + '5.1' in platform.version()): + self.profile.set_preferences({"layers.acceleration.disabled": True}) + if self.ca_certificate_path is not None: self.setup_ssl() diff --git a/tests/wpt/harness/wptrunner/browsers/servo.py b/tests/wpt/harness/wptrunner/browsers/servo.py index 1ea017377fe..2eeb5aaa158 100644 --- a/tests/wpt/harness/wptrunner/browsers/servo.py +++ b/tests/wpt/harness/wptrunner/browsers/servo.py @@ -6,7 +6,7 @@ import os from .base import NullBrowser, ExecutorBrowser, require_arg from ..executors import executor_kwargs as base_executor_kwargs -from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor +from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor, ServoWdspecExecutor here = os.path.join(os.path.split(__file__)[0]) @@ -14,7 +14,8 @@ __wptrunner__ = {"product": "servo", "check_args": "check_args", "browser": "ServoBrowser", "executor": {"testharness": "ServoTestharnessExecutor", - "reftest": "ServoRefTestExecutor"}, + "reftest": "ServoRefTestExecutor", + "wdspec": "ServoWdspecExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_options": "env_options", diff --git a/tests/wpt/harness/wptrunner/executors/executormarionette.py b/tests/wpt/harness/wptrunner/executors/executormarionette.py index c4b1cba689a..8495d3b45ee 100644 --- a/tests/wpt/harness/wptrunner/executors/executormarionette.py +++ b/tests/wpt/harness/wptrunner/executors/executormarionette.py @@ -17,11 +17,11 @@ from ..wpttest import WdspecResult, WdspecSubtestResult errors = None marionette = None +pytestrunner = None webdriver = None here = os.path.join(os.path.split(__file__)[0]) -from . import pytestrunner from .base import (ExecutorException, Protocol, RefTestExecutor, @@ -41,7 +41,7 @@ extra_timeout = 5 # seconds def do_delayed_imports(): - global errors, marionette, webdriver + global errors, marionette # Marionette client used to be called marionette, recently it changed # to marionette_driver for unfathomable reasons @@ -51,8 +51,6 @@ def do_delayed_imports(): except ImportError: from marionette_driver import marionette, errors - import webdriver - class MarionetteProtocol(Protocol): def __init__(self, executor, browser): @@ -292,7 +290,7 @@ class RemoteMarionetteProtocol(Protocol): class ExecuteAsyncScriptRun(object): def __init__(self, logger, func, marionette, url, timeout): self.logger = logger - self.result = None + self.result = (None, None) self.marionette = marionette self.func = func self.url = url @@ -323,11 +321,9 @@ class ExecuteAsyncScriptRun(object): wait_timeout = None flag = self.result_flag.wait(wait_timeout) - if self.result is None: + if self.result[1] is None: self.logger.debug("Timed out waiting for a result") - assert not flag self.result = False, ("EXTERNAL-TIMEOUT", None) - return self.result def _run(self): @@ -409,7 +405,8 @@ class MarionetteTestharnessExecutor(TestharnessExecutor): "timeout": timeout_ms, "explicit_timeout": timeout is None} - return marionette.execute_async_script(script, new_sandbox=False) + rv = marionette.execute_async_script(script, new_sandbox=False) + return rv class MarionetteRefTestExecutor(RefTestExecutor): @@ -487,7 +484,7 @@ class MarionetteRefTestExecutor(RefTestExecutor): class WdspecRun(object): def __init__(self, func, session, path, timeout): self.func = func - self.result = None + self.result = (None, None) self.session = session self.path = path self.timeout = timeout @@ -504,8 +501,7 @@ class WdspecRun(object): executor.start() flag = self.result_flag.wait(self.timeout) - if self.result is None: - assert not flag + if self.result[1] is None: self.result = False, ("EXTERNAL-TIMEOUT", None) return self.result @@ -528,6 +524,7 @@ class WdspecRun(object): class MarionetteWdspecExecutor(WdspecExecutor): def __init__(self, browser, server_config, webdriver_binary, timeout_multiplier=1, close_after_done=True, debug_info=None): + self.do_delayed_imports() WdspecExecutor.__init__(self, browser, server_config, timeout_multiplier=timeout_multiplier, debug_info=debug_info) @@ -557,3 +554,8 @@ class MarionetteWdspecExecutor(WdspecExecutor): harness_result = ("OK", None) subtest_results = pytestrunner.run(path, session, timeout=timeout) return (harness_result, subtest_results) + + def do_delayed_imports(self): + global pytestrunner, webdriver + from . import pytestrunner + import webdriver diff --git a/tests/wpt/harness/wptrunner/executors/executorselenium.py b/tests/wpt/harness/wptrunner/executors/executorselenium.py index 587c49c7c92..f5d65f499b0 100644 --- a/tests/wpt/harness/wptrunner/executors/executorselenium.py +++ b/tests/wpt/harness/wptrunner/executors/executorselenium.py @@ -22,20 +22,21 @@ from .base import (ExecutorException, strip_server) from ..testrunner import Stop - here = os.path.join(os.path.split(__file__)[0]) webdriver = None exceptions = None +RemoteConnection = None extra_timeout = 5 def do_delayed_imports(): global webdriver global exceptions + global RemoteConnection from selenium import webdriver from selenium.common import exceptions - + from selenium.webdriver.remote.remote_connection import RemoteConnection class SeleniumProtocol(Protocol): def __init__(self, executor, browser, capabilities, **kwargs): @@ -53,8 +54,9 @@ class SeleniumProtocol(Protocol): session_started = False try: - self.webdriver = webdriver.Remote( - self.url, desired_capabilities=self.capabilities) + self.webdriver = webdriver.Remote(command_executor=RemoteConnection(self.url.strip("/"), + resolve_ip=False), + desired_capabilities=self.capabilities) except: self.logger.warning( "Connecting to Selenium failed:\n%s" % traceback.format_exc()) @@ -231,17 +233,7 @@ class SeleniumRefTestExecutor(RefTestExecutor): def do_test(self, test): self.logger.info("Test requires OS-level window focus") - if self.close_after_done and self.has_window: - self.protocol.webdriver.close() - self.protocol.webdriver.switch_to_window( - self.protocol.webdriver.window_handles[-1]) - self.has_window = False - - if not self.has_window: - self.protocol.webdriver.execute_script(self.script) - self.protocol.webdriver.switch_to_window( - self.protocol.webdriver.window_handles[-1]) - self.has_window = True + self.protocol.webdriver.set_window_size(600, 600) result = self.implementation.run_test(test) diff --git a/tests/wpt/harness/wptrunner/executors/executorservo.py b/tests/wpt/harness/wptrunner/executors/executorservo.py index 15092056a93..b627223a7df 100644 --- a/tests/wpt/harness/wptrunner/executors/executorservo.py +++ b/tests/wpt/harness/wptrunner/executors/executorservo.py @@ -4,11 +4,13 @@ import base64 import hashlib +import httplib import json import os import subprocess import tempfile import threading +import traceback import urlparse import uuid from collections import defaultdict @@ -19,11 +21,19 @@ from .base import (ExecutorException, Protocol, RefTestImplementation, testharness_result_converter, - reftest_result_converter) + reftest_result_converter, + WdspecExecutor) from .process import ProcessTestExecutor from ..browsers.base import browser_command -render_arg = None +from ..wpttest import WdspecResult, WdspecSubtestResult +from ..webdriver_server import ServoDriverServer +from .executormarionette import WdspecRun +pytestrunner = None +render_arg = None +webdriver = None + +extra_timeout = 5 # seconds def do_delayed_imports(): global render_arg @@ -205,7 +215,7 @@ class ServoRefTestExecutor(ProcessTestExecutor): self.binary, [render_arg(self.browser.render_backend), "--hard-fail", "--exit", "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates", - "--output=%s" % output_path, full_url], + "--output=%s" % output_path, full_url] + self.browser.binary_args, self.debug_info) for stylesheet in self.browser.user_stylesheets: @@ -214,10 +224,7 @@ class ServoRefTestExecutor(ProcessTestExecutor): for pref, value in test.environment.get('prefs', {}).iteritems(): command += ["--pref", "%s=%s" % (pref, value)] - if viewport_size: - command += ["--resolution", viewport_size] - else: - command += ["--resolution", "800x600"] + command += ["--resolution", viewport_size or "800x600"] if dpi: command += ["--device-pixel-ratio", dpi] @@ -278,3 +285,83 @@ class ServoRefTestExecutor(ProcessTestExecutor): self.logger.process_output(self.proc.pid, line, " ".join(self.command)) + +class ServoWdspecProtocol(Protocol): + def __init__(self, executor, browser): + self.do_delayed_imports() + Protocol.__init__(self, executor, browser) + self.session = None + self.server = None + + def setup(self, runner): + try: + self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args, render_backend=self.browser.render_backend) + self.server.start(block=False) + self.logger.info( + "WebDriver HTTP server listening at %s" % self.server.url) + + self.logger.info( + "Establishing new WebDriver session with %s" % self.server.url) + self.session = webdriver.Session( + self.server.host, self.server.port, self.server.base_path) + except Exception: + self.logger.error(traceback.format_exc()) + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + if self.server is not None: + try: + if self.session.session_id is not None: + self.session.end() + except Exception: + pass + if self.server.is_alive: + self.server.stop() + + @property + def is_alive(self): + conn = httplib.HTTPConnection(self.server.host, self.server.port) + conn.request("HEAD", self.server.base_path + "invalid") + res = conn.getresponse() + return res.status == 404 + + def do_delayed_imports(self): + global pytestrunner, webdriver + from . import pytestrunner + import webdriver + + +class ServoWdspecExecutor(WdspecExecutor): + def __init__(self, browser, server_config, + timeout_multiplier=1, close_after_done=True, debug_info=None, + **kwargs): + WdspecExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = ServoWdspecProtocol(self, browser) + + def is_alive(self): + return self.protocol.is_alive + + def on_environment_change(self, new_environment): + pass + + def do_test(self, test): + timeout = test.timeout * self.timeout_multiplier + extra_timeout + + success, data = WdspecRun(self.do_wdspec, + self.protocol.session, + test.path, + timeout).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_wdspec(self, session, path, timeout): + harness_result = ("OK", None) + subtest_results = pytestrunner.run(path, session, timeout=timeout) + return (harness_result, subtest_results) diff --git a/tests/wpt/harness/wptrunner/executors/executorservodriver.py b/tests/wpt/harness/wptrunner/executors/executorservodriver.py index fceeb58fad2..279658d0cd1 100644 --- a/tests/wpt/harness/wptrunner/executors/executorservodriver.py +++ b/tests/wpt/harness/wptrunner/executors/executorservodriver.py @@ -14,7 +14,6 @@ from .base import (Protocol, RefTestImplementation, TestharnessExecutor, strip_server) -from .. import webdriver from ..testrunner import Stop webdriver = None @@ -26,7 +25,7 @@ extra_timeout = 5 def do_delayed_imports(): global webdriver - import webdriver + from tools import webdriver class ServoWebDriverProtocol(Protocol): diff --git a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py index 77afb4a3684..81796d69883 100644 --- a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py +++ b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py @@ -3,6 +3,10 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. import pytest +import webdriver + +import contextlib +import httplib """pytest fixtures for use in Python-based WPT tests. @@ -17,7 +21,7 @@ class Session(object): in tests. The session is not created by default to enable testing of session - creation. However, a module-scoped session will be implicitly created + creation. However, a function-scoped session will be implicitly created at the first call to a WebDriver command. This means methods such as `session.send_command` and `session.session_id` are possible to use without having a session. @@ -45,14 +49,88 @@ class Session(object): def test_something(setup, session): assert session.url == "https://example.org" - The session is closed when the test module goes out of scope by an - implicit call to `session.end`. + When the test function goes out of scope, any remaining user prompts + and opened windows are closed, and the current browsing context is + switched back to the top-level browsing context. """ def __init__(self, client): self.client = client - @pytest.fixture(scope="module") + @pytest.fixture(scope="function") def session(self, request): - request.addfinalizer(self.client.end) + # finalisers are popped off a stack, + # making their ordering reverse + request.addfinalizer(self.switch_to_top_level_browsing_context) + request.addfinalizer(self.restore_windows) + request.addfinalizer(self.dismiss_user_prompts) + return self.client + + def dismiss_user_prompts(self): + """Dismisses any open user prompts in windows.""" + current_window = self.client.window_handle + + for window in self.windows(): + self.client.window_handle = window + try: + self.client.alert.dismiss() + except webdriver.NoSuchAlertException: + pass + + self.client.window_handle = current_window + + def restore_windows(self): + """Closes superfluous windows opened by the test without ending + the session implicitly by closing the last window. + """ + current_window = self.client.window_handle + + for window in self.windows(exclude=[current_window]): + self.client.window_handle = window + if len(self.client.window_handles) > 1: + self.client.close() + + self.client.window_handle = current_window + + def switch_to_top_level_browsing_context(self): + """If the current browsing context selected by WebDriver is a + `` or an `