mirror of
https://github.com/servo/servo.git
synced 2025-08-10 16:05:43 +01:00
Allow running testharness/testdriver/reftests in servodriver (#34550)
* Make servodriver a thin wrapper over the base webdriver browser/executor classes. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Make ServoWebDriverRefTestExecutor a thin shell over the webdriver reftest executor. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Wait for the initial load to complete when opening a new tab via webdriver. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Remove assumption of a single tab from the webdriver server. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Serialize all keys of JS objects when converting to webdriver values. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Formatting. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Cleanup, docs, etc. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Use webview terminology more consistently. Signed-off-by: Josh Matthews <josh@joshmatthews.net> * Fix flake8 errors. Signed-off-by: Josh Matthews <josh@joshmatthews.net> --------- Signed-off-by: Josh Matthews <josh@joshmatthews.net>
This commit is contained in:
Josh Matthews
GitHub
parent
25f242b652
commit
7b160700d0
9 changed files with 336 additions and 407 deletions
8
tests/wpt/meta/MANIFEST.json
vendored
8
tests/wpt/meta/MANIFEST.json
vendored
|
|
@ -499057,7 +499057,7 @@
|
|||
[]
|
||||
],
|
||||
"servodriver.py": [
|
||||
"2cb638be1569c91a963bf2cb3824f2c07788f8cc",
|
||||
"1e9a2f3090ef1ff826dac4ca3a88a958242ffae6",
|
||||
[]
|
||||
],
|
||||
"webkit.py": [
|
||||
|
|
@ -499123,7 +499123,7 @@
|
|||
[]
|
||||
],
|
||||
"executorservodriver.py": [
|
||||
"5d7d55f30b551f59bc0b16aacc8641c0fc24e39c",
|
||||
"41b8ed9ac1891edb7ec332c61923314f8b1e5f19",
|
||||
[]
|
||||
],
|
||||
"executorwebdriver.py": [
|
||||
|
|
@ -499269,10 +499269,6 @@
|
|||
"d6616739e6ed63a79e8b2f5d8aee1d5b2ced7f49",
|
||||
[]
|
||||
],
|
||||
"testharnessreport-servodriver.js": [
|
||||
"7819538dbb8f4a807d5db2649c2540854996c865",
|
||||
[]
|
||||
],
|
||||
"testharnessreport-wktr.js": [
|
||||
"b7d350a4262cb6f0d38337b17311fea7bd73eb70",
|
||||
[]
|
||||
|
|
|
|||
|
|
@ -1,19 +1,13 @@
|
|||
# mypy: allow-untyped-defs
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from mozprocess import ProcessHandler
|
||||
|
||||
from tools.serve.serve import make_hosts_file
|
||||
|
||||
from .base import (Browser,
|
||||
ExecutorBrowser,
|
||||
OutputHandler,
|
||||
from .base import (WebDriverBrowser,
|
||||
require_arg,
|
||||
get_free_port,
|
||||
browser_command)
|
||||
get_free_port)
|
||||
from .base import get_timeout_multiplier # noqa: F401
|
||||
from ..executors import executor_kwargs as base_executor_kwargs
|
||||
from ..executors.executorservodriver import (ServoWebDriverTestharnessExecutor, # noqa: F401
|
||||
|
|
@ -64,8 +58,7 @@ def env_extras(**kwargs):
|
|||
|
||||
def env_options():
|
||||
return {"server_host": "127.0.0.1",
|
||||
"testharnessreport": "testharnessreport-servodriver.js",
|
||||
"supports_debugger": True}
|
||||
"supports_debugger": False}
|
||||
|
||||
|
||||
def update_properties():
|
||||
|
|
@ -79,107 +72,40 @@ def write_hosts_file(config):
|
|||
return hosts_path
|
||||
|
||||
|
||||
class ServoWebDriverBrowser(Browser):
|
||||
class ServoWebDriverBrowser(WebDriverBrowser):
|
||||
init_timeout = 300 # Large timeout for cases where we're booting an Android emulator
|
||||
|
||||
def __init__(self, logger, binary, debug_info=None, webdriver_host="127.0.0.1",
|
||||
server_config=None, binary_args=None,
|
||||
user_stylesheets=None, headless=None, **kwargs):
|
||||
Browser.__init__(self, logger)
|
||||
self.binary = binary
|
||||
self.binary_args = binary_args or []
|
||||
self.webdriver_host = webdriver_host
|
||||
self.webdriver_port = None
|
||||
self.proc = None
|
||||
self.debug_info = debug_info
|
||||
self.hosts_path = write_hosts_file(server_config)
|
||||
self.server_ports = server_config.ports if server_config else {}
|
||||
self.command = None
|
||||
self.user_stylesheets = user_stylesheets if user_stylesheets else []
|
||||
self.headless = headless if headless else False
|
||||
self.ca_certificate_path = server_config.ssl_config["ca_cert_path"]
|
||||
self.output_handler = None
|
||||
|
||||
def start(self, **kwargs):
|
||||
self.webdriver_port = get_free_port()
|
||||
|
||||
hosts_path = write_hosts_file(server_config)
|
||||
port = get_free_port()
|
||||
env = os.environ.copy()
|
||||
env["HOST_FILE"] = self.hosts_path
|
||||
env["HOST_FILE"] = hosts_path
|
||||
env["RUST_BACKTRACE"] = "1"
|
||||
env["EMULATOR_REVERSE_FORWARD_PORTS"] = ",".join(
|
||||
str(port)
|
||||
for _protocol, ports in self.server_ports.items()
|
||||
for port in ports
|
||||
if port
|
||||
)
|
||||
|
||||
debug_args, command = browser_command(
|
||||
self.binary,
|
||||
self.binary_args + [
|
||||
"--hard-fail",
|
||||
"--webdriver=%s" % self.webdriver_port,
|
||||
"about:blank",
|
||||
],
|
||||
self.debug_info
|
||||
)
|
||||
args = [
|
||||
"--hard-fail",
|
||||
"--webdriver=%s" % port,
|
||||
"about:blank",
|
||||
]
|
||||
|
||||
if self.headless:
|
||||
command += ["--headless"]
|
||||
ca_cert_path = server_config.ssl_config["ca_cert_path"]
|
||||
if ca_cert_path:
|
||||
args += ["--certificate-path", ca_cert_path]
|
||||
if binary_args:
|
||||
args += binary_args
|
||||
if user_stylesheets:
|
||||
for stylesheet in user_stylesheets:
|
||||
args += ["--user-stylesheet", stylesheet]
|
||||
if headless:
|
||||
args += ["--headless"]
|
||||
|
||||
if self.ca_certificate_path:
|
||||
command += ["--certificate-path", self.ca_certificate_path]
|
||||
|
||||
for stylesheet in self.user_stylesheets:
|
||||
command += ["--user-stylesheet", stylesheet]
|
||||
|
||||
self.command = command
|
||||
|
||||
self.command = debug_args + self.command
|
||||
|
||||
if not self.debug_info or not self.debug_info.interactive:
|
||||
self.output_handler = OutputHandler(self.logger, self.command)
|
||||
self.proc = ProcessHandler(self.command,
|
||||
processOutputLine=[self.on_output],
|
||||
env=env,
|
||||
storeOutput=False)
|
||||
self.proc.run()
|
||||
self.output_handler.after_process_start(self.proc.pid)
|
||||
self.output_handler.start()
|
||||
else:
|
||||
self.proc = subprocess.Popen(self.command, env=env)
|
||||
|
||||
self.logger.debug("Servo Started")
|
||||
|
||||
def stop(self, force=False):
|
||||
self.logger.debug("Stopping browser")
|
||||
if self.proc is not None:
|
||||
try:
|
||||
self.proc.kill()
|
||||
except OSError:
|
||||
# This can happen on Windows if the process is already dead
|
||||
pass
|
||||
if self.output_handler is not None:
|
||||
self.output_handler.after_process_stop()
|
||||
|
||||
@property
|
||||
def pid(self):
|
||||
if self.proc is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
return self.proc.pid
|
||||
except AttributeError:
|
||||
return None
|
||||
|
||||
def is_alive(self):
|
||||
return self.proc.poll() is None
|
||||
WebDriverBrowser.__init__(self, env=env, logger=logger, host=webdriver_host, port=port,
|
||||
supports_pac=False, webdriver_binary=binary, webdriver_args=args,
|
||||
binary=binary)
|
||||
self.hosts_path = hosts_path
|
||||
|
||||
def cleanup(self):
|
||||
self.stop()
|
||||
WebDriverBrowser.cleanup(self)
|
||||
os.remove(self.hosts_path)
|
||||
|
||||
def executor_browser(self):
|
||||
assert self.webdriver_port is not None
|
||||
return ExecutorBrowser, {"webdriver_host": self.webdriver_host,
|
||||
"webdriver_port": self.webdriver_port,
|
||||
"init_timeout": self.init_timeout}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,8 @@
|
|||
# mypy: allow-untyped-defs
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import traceback
|
||||
|
||||
from .base import (Protocol,
|
||||
RefTestExecutor,
|
||||
RefTestImplementation,
|
||||
TestharnessExecutor,
|
||||
TimedRunner,
|
||||
strip_server)
|
||||
from .protocol import BaseProtocolPart
|
||||
from ..environment import wait_for_service
|
||||
from .executorwebdriver import WebDriverProtocol, WebDriverTestharnessExecutor, WebDriverRefTestExecutor
|
||||
|
||||
webdriver = None
|
||||
ServoCommandExtensions = None
|
||||
|
|
@ -64,240 +54,57 @@ def parse_pref_value(value):
|
|||
return value
|
||||
|
||||
|
||||
class ServoBaseProtocolPart(BaseProtocolPart):
|
||||
def execute_script(self, script, asynchronous=False):
|
||||
pass
|
||||
|
||||
def set_timeout(self, timeout):
|
||||
pass
|
||||
|
||||
def wait(self):
|
||||
return False
|
||||
|
||||
def set_window(self, handle):
|
||||
pass
|
||||
|
||||
def window_handles(self):
|
||||
return []
|
||||
|
||||
def load(self, url):
|
||||
pass
|
||||
|
||||
|
||||
class ServoWebDriverProtocol(Protocol):
|
||||
implements = [ServoBaseProtocolPart]
|
||||
|
||||
class ServoWebDriverProtocol(WebDriverProtocol):
|
||||
def __init__(self, executor, browser, capabilities, **kwargs):
|
||||
do_delayed_imports()
|
||||
Protocol.__init__(self, executor, browser)
|
||||
self.capabilities = capabilities
|
||||
self.host = browser.webdriver_host
|
||||
self.port = browser.webdriver_port
|
||||
self.init_timeout = browser.init_timeout
|
||||
self.session = None
|
||||
WebDriverProtocol.__init__(self, executor, browser, capabilities, **kwargs)
|
||||
|
||||
def connect(self):
|
||||
"""Connect to browser via WebDriver."""
|
||||
wait_for_service(self.logger, self.host, self.port, timeout=self.init_timeout)
|
||||
"""Connect to browser via WebDriver and crete a WebDriver session."""
|
||||
self.logger.debug("Connecting to WebDriver on URL: %s" % self.url)
|
||||
|
||||
self.session = webdriver.Session(self.host, self.port, extension=ServoCommandExtensions)
|
||||
self.session.start()
|
||||
host, port = self.url.split(":")[1].strip("/"), self.url.split(':')[-1].strip("/")
|
||||
|
||||
def after_connect(self):
|
||||
pass
|
||||
|
||||
def teardown(self):
|
||||
self.logger.debug("Hanging up on WebDriver session")
|
||||
try:
|
||||
self.session.end()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def is_alive(self):
|
||||
try:
|
||||
# Get a simple property over the connection
|
||||
self.session.window_handle
|
||||
# TODO what exception?
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
def wait(self):
|
||||
while True:
|
||||
try:
|
||||
return self.session.execute_async_script("""let callback = arguments[arguments.length - 1];
|
||||
addEventListener("__test_restart", e => {e.preventDefault(); callback(true)})""")
|
||||
except webdriver.TimeoutException:
|
||||
pass
|
||||
except (socket.timeout, OSError):
|
||||
break
|
||||
except Exception:
|
||||
self.logger.error(traceback.format_exc())
|
||||
break
|
||||
return False
|
||||
capabilities = {"alwaysMatch": self.capabilities}
|
||||
self.webdriver = webdriver.Session(host, port,
|
||||
capabilities=capabilities,
|
||||
enable_bidi=self.enable_bidi,
|
||||
extension=ServoCommandExtensions)
|
||||
self.webdriver.start()
|
||||
|
||||
|
||||
class ServoWebDriverRun(TimedRunner):
|
||||
def set_timeout(self):
|
||||
pass
|
||||
|
||||
def run_func(self):
|
||||
try:
|
||||
self.result = True, self.func(self.protocol.session, self.url, self.timeout)
|
||||
except webdriver.TimeoutException:
|
||||
self.result = False, ("EXTERNAL-TIMEOUT", None)
|
||||
except (socket.timeout, OSError):
|
||||
self.result = False, ("CRASH", None)
|
||||
except Exception as e:
|
||||
message = getattr(e, "message", "")
|
||||
if message:
|
||||
message += "\n"
|
||||
message += traceback.format_exc()
|
||||
self.result = False, ("INTERNAL-ERROR", e)
|
||||
finally:
|
||||
self.result_flag.set()
|
||||
|
||||
|
||||
class ServoWebDriverTestharnessExecutor(TestharnessExecutor):
|
||||
class ServoWebDriverTestharnessExecutor(WebDriverTestharnessExecutor):
|
||||
supports_testdriver = True
|
||||
protocol_cls = ServoWebDriverProtocol
|
||||
|
||||
def __init__(self, logger, browser, server_config, timeout_multiplier=1,
|
||||
close_after_done=True, capabilities=None, debug_info=None,
|
||||
close_after_done=True, capabilities={}, debug_info=None,
|
||||
**kwargs):
|
||||
TestharnessExecutor.__init__(self, logger, browser, server_config, timeout_multiplier=1,
|
||||
debug_info=None)
|
||||
self.protocol = ServoWebDriverProtocol(self, browser, capabilities=capabilities)
|
||||
with open(os.path.join(here, "testharness_servodriver.js")) as f:
|
||||
self.script = f.read()
|
||||
self.timeout = None
|
||||
|
||||
def on_protocol_change(self, new_protocol):
|
||||
pass
|
||||
|
||||
def is_alive(self):
|
||||
return self.protocol.is_alive()
|
||||
|
||||
def do_test(self, test):
|
||||
url = self.test_url(test)
|
||||
|
||||
timeout = test.timeout * self.timeout_multiplier + self.extra_timeout
|
||||
|
||||
if timeout != self.timeout:
|
||||
try:
|
||||
self.protocol.session.timeouts.script = timeout
|
||||
self.timeout = timeout
|
||||
except OSError:
|
||||
msg = "Lost WebDriver connection"
|
||||
self.logger.error(msg)
|
||||
return ("INTERNAL-ERROR", msg)
|
||||
|
||||
success, data = ServoWebDriverRun(self.logger,
|
||||
self.do_testharness,
|
||||
self.protocol,
|
||||
url,
|
||||
timeout,
|
||||
self.extra_timeout).run()
|
||||
|
||||
if success:
|
||||
return self.convert_result(test, data)
|
||||
|
||||
return (test.make_result(*data), [])
|
||||
|
||||
def do_testharness(self, session, url, timeout):
|
||||
session.url = url
|
||||
result = json.loads(
|
||||
session.execute_async_script(
|
||||
self.script % {"abs_url": url,
|
||||
"url": strip_server(url),
|
||||
"timeout_multiplier": self.timeout_multiplier,
|
||||
"timeout": timeout * 1000}))
|
||||
# Prevent leaking every page in history until Servo develops a more sane
|
||||
# page cache
|
||||
session.back()
|
||||
return result
|
||||
WebDriverTestharnessExecutor.__init__(self, logger, browser, server_config,
|
||||
timeout_multiplier, capabilities=capabilities,
|
||||
debug_info=debug_info, close_after_done=close_after_done,
|
||||
cleanup_after_test=False)
|
||||
|
||||
def on_environment_change(self, new_environment):
|
||||
self.protocol.session.extension.change_prefs(
|
||||
self.protocol.webdriver.extension.change_prefs(
|
||||
self.last_environment.get("prefs", {}),
|
||||
new_environment.get("prefs", {})
|
||||
)
|
||||
|
||||
|
||||
class TimeoutError(Exception):
|
||||
pass
|
||||
class ServoWebDriverRefTestExecutor(WebDriverRefTestExecutor):
|
||||
protocol_cls = ServoWebDriverProtocol
|
||||
|
||||
|
||||
class ServoWebDriverRefTestExecutor(RefTestExecutor):
|
||||
def __init__(self, logger, browser, server_config, timeout_multiplier=1,
|
||||
screenshot_cache=None, capabilities=None, debug_info=None,
|
||||
screenshot_cache=None, capabilities={}, debug_info=None,
|
||||
**kwargs):
|
||||
"""Selenium WebDriver-based executor for reftests"""
|
||||
RefTestExecutor.__init__(self,
|
||||
logger,
|
||||
browser,
|
||||
server_config,
|
||||
screenshot_cache=screenshot_cache,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_info=debug_info)
|
||||
self.protocol = ServoWebDriverProtocol(self, browser,
|
||||
capabilities=capabilities)
|
||||
self.implementation = RefTestImplementation(self)
|
||||
self.timeout = None
|
||||
with open(os.path.join(here, "test-wait.js")) as f:
|
||||
self.wait_script = f.read() % {"classname": "reftest-wait"}
|
||||
|
||||
def reset(self):
|
||||
self.implementation.reset()
|
||||
|
||||
def is_alive(self):
|
||||
return self.protocol.is_alive()
|
||||
|
||||
def do_test(self, test):
|
||||
try:
|
||||
result = self.implementation.run_test(test)
|
||||
return self.convert_result(test, result)
|
||||
except OSError:
|
||||
return test.make_result("CRASH", None), []
|
||||
except TimeoutError:
|
||||
return test.make_result("TIMEOUT", None), []
|
||||
except Exception as e:
|
||||
message = getattr(e, "message", "")
|
||||
if message:
|
||||
message += "\n"
|
||||
message += traceback.format_exc()
|
||||
return test.make_result("INTERNAL-ERROR", message), []
|
||||
|
||||
def screenshot(self, test, viewport_size, dpi, page_ranges):
|
||||
# https://github.com/web-platform-tests/wpt/issues/7135
|
||||
assert viewport_size is None
|
||||
assert dpi is None
|
||||
|
||||
timeout = (test.timeout * self.timeout_multiplier + self.extra_timeout
|
||||
if self.debug_info is None else None)
|
||||
|
||||
if self.timeout != timeout:
|
||||
try:
|
||||
self.protocol.session.timeouts.script = timeout
|
||||
self.timeout = timeout
|
||||
except OSError:
|
||||
msg = "Lost webdriver connection"
|
||||
self.logger.error(msg)
|
||||
return ("INTERNAL-ERROR", msg)
|
||||
|
||||
return ServoWebDriverRun(self.logger,
|
||||
self._screenshot,
|
||||
self.protocol,
|
||||
self.test_url(test),
|
||||
timeout,
|
||||
self.extra_timeout).run()
|
||||
|
||||
def _screenshot(self, session, url, timeout):
|
||||
session.url = url
|
||||
session.execute_async_script(self.wait_script)
|
||||
return session.screenshot()
|
||||
WebDriverRefTestExecutor.__init__(self, logger, browser, server_config,
|
||||
timeout_multiplier, screenshot_cache,
|
||||
capabilities=capabilities,
|
||||
debug_info=debug_info)
|
||||
|
||||
def on_environment_change(self, new_environment):
|
||||
self.protocol.session.extension.change_prefs(
|
||||
self.protocol.webdriver.extension.change_prefs(
|
||||
self.last_environment.get("prefs", {}),
|
||||
new_environment.get("prefs", {})
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
setup({output:%(output)d, debug: %(debug)s});
|
||||
|
||||
add_completion_callback(function() {
|
||||
add_completion_callback(function (tests, status) {
|
||||
var subtest_results = tests.map(function(x) {
|
||||
return [x.name, x.status, x.message, x.stack]
|
||||
});
|
||||
var id = location.pathname + location.search + location.hash;
|
||||
var results = JSON.stringify([id,
|
||||
status.status,
|
||||
status.message,
|
||||
status.stack,
|
||||
subtest_results]);
|
||||
(function done() {
|
||||
if (window.__wd_results_callback__) {
|
||||
clearTimeout(__wd_results_timer__);
|
||||
__wd_results_callback__(results)
|
||||
} else {
|
||||
setTimeout(done, 20);
|
||||
}
|
||||
})()
|
||||
})
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue