mirror of
https://github.com/servo/servo.git
synced 2025-08-03 04:30:10 +01:00
Update to latest wptrunner (036c9931).
This commit is contained in:
James Graham
parent
88d9c1b257
commit
f837e575fe
33 changed files with 709 additions and 229 deletions
|
|
@ -1,5 +1,4 @@
|
|||
html5lib >= 0.99
|
||||
mozinfo >= 0.7
|
||||
mozlog >= 2.8
|
||||
# Unfortunately, just for gdb flags
|
||||
mozrunner >= 6.1
|
||||
mozdebug >= 0.1
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
marionette_client >= 0.7.10
|
||||
marionette_driver >= 0.4
|
||||
mozprofile >= 0.21
|
||||
mozprocess >= 0.19
|
||||
mozcrash >= 0.13
|
||||
mozrunner >= 6.7
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
[test_pref_set.html]
|
||||
prefs: ["browser.display.foreground_color:#00FF00",
|
||||
"browser.display.background_color:#000000"]
|
||||
|
|
@ -139,7 +139,7 @@ def get_parser():
|
|||
help="Specific product to include in test run")
|
||||
parser.add_argument("--pdb", action="store_true",
|
||||
help="Invoke pdb on uncaught exception")
|
||||
parser.add_argument("test", nargs="*", type=wptcommandline.slash_prefixed,
|
||||
parser.add_argument("test", nargs="*",
|
||||
help="Specific tests to include in test run")
|
||||
return parser
|
||||
|
||||
|
|
|
|||
10
tests/wpt/harness/test/testdata/testharness/firefox/test_pref_set.html
vendored
Normal file
10
tests/wpt/harness/test/testdata/testharness/firefox/test_pref_set.html
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
<!doctype html>
|
||||
<title>Example https test</title>
|
||||
<script src="/resources/testharness.js"></script>
|
||||
<script src="/resources/testharnessreport.js"></script>
|
||||
<p>Test requires the pref browser.display.foreground_color to be set to #00FF00</p>
|
||||
<script>
|
||||
test(function() {
|
||||
assert_equals(getComputedStyle(document.body).color, "rgb(0, 255, 0)");
|
||||
}, "Test that pref was set");
|
||||
</script>
|
||||
|
|
@ -20,6 +20,7 @@ from mozprofile import FirefoxProfile, Preferences
|
|||
from .base import get_free_port, BrowserError, Browser, ExecutorBrowser
|
||||
from ..executors.executormarionette import MarionetteTestharnessExecutor
|
||||
from ..hosts import HostsFile, HostsLine
|
||||
from ..environment import hostnames
|
||||
|
||||
here = os.path.split(__file__)[0]
|
||||
|
||||
|
|
@ -115,13 +116,6 @@ class B2GBrowser(Browser):
|
|||
self.logger.debug("Device runner started")
|
||||
|
||||
def setup_hosts(self):
|
||||
hostnames = ["web-platform.test",
|
||||
"www.web-platform.test",
|
||||
"www1.web-platform.test",
|
||||
"www2.web-platform.test",
|
||||
"xn--n8j6ds53lwwkrqhv28a.web-platform.test",
|
||||
"xn--lve-6lad.web-platform.test"]
|
||||
|
||||
host_ip = moznetwork.get_ip()
|
||||
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -41,6 +41,18 @@ def get_free_port(start_port, exclude=None):
|
|||
finally:
|
||||
s.close()
|
||||
|
||||
def browser_command(binary, args, debug_info):
|
||||
if debug_info:
|
||||
if debug_info.requiresEscapedArgs:
|
||||
args = [item.replace("&", "\\&") for item in args]
|
||||
debug_args = [debug_info.path] + debug_info.args
|
||||
else:
|
||||
debug_args = []
|
||||
|
||||
command = [binary] + args
|
||||
|
||||
return debug_args, command
|
||||
|
||||
|
||||
class BrowserError(Exception):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -12,9 +12,10 @@ from mozprofile.permissions import ServerLocations
|
|||
from mozrunner import FirefoxRunner
|
||||
from mozcrash import mozcrash
|
||||
|
||||
from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg
|
||||
from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg, browser_command
|
||||
from ..executors import executor_kwargs as base_executor_kwargs
|
||||
from ..executors.executormarionette import MarionetteTestharnessExecutor, MarionetteRefTestExecutor
|
||||
from ..environment import hostnames
|
||||
|
||||
here = os.path.join(os.path.split(__file__)[0])
|
||||
|
||||
|
|
@ -37,8 +38,7 @@ def check_args(**kwargs):
|
|||
def browser_kwargs(**kwargs):
|
||||
return {"binary": kwargs["binary"],
|
||||
"prefs_root": kwargs["prefs_root"],
|
||||
"debug_args": kwargs["debug_args"],
|
||||
"interactive": kwargs["interactive"],
|
||||
"debug_info": kwargs["debug_info"],
|
||||
"symbols_path": kwargs["symbols_path"],
|
||||
"stackwalk_binary": kwargs["stackwalk_binary"],
|
||||
"certutil_binary": kwargs["certutil_binary"],
|
||||
|
|
@ -57,13 +57,13 @@ def env_options():
|
|||
"external_host": "web-platform.test",
|
||||
"bind_hostname": "false",
|
||||
"certificate_domain": "web-platform.test",
|
||||
"encrypt_after_connect": True}
|
||||
"supports_debugger": True}
|
||||
|
||||
|
||||
class FirefoxBrowser(Browser):
|
||||
used_ports = set()
|
||||
|
||||
def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
|
||||
def __init__(self, logger, binary, prefs_root, debug_info=None,
|
||||
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
|
||||
ca_certificate_path=None):
|
||||
Browser.__init__(self, logger)
|
||||
|
|
@ -72,8 +72,7 @@ class FirefoxBrowser(Browser):
|
|||
self.marionette_port = None
|
||||
self.used_ports.add(self.marionette_port)
|
||||
self.runner = None
|
||||
self.debug_args = debug_args
|
||||
self.interactive = interactive
|
||||
self.debug_info = debug_info
|
||||
self.profile = None
|
||||
self.symbols_path = symbols_path
|
||||
self.stackwalk_binary = stackwalk_binary
|
||||
|
|
@ -84,38 +83,35 @@ class FirefoxBrowser(Browser):
|
|||
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["MOZ_CRASHREPORTER"] = "1"
|
||||
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
|
||||
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
|
||||
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
|
||||
|
||||
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
|
||||
|
||||
preferences = self.load_prefs()
|
||||
|
||||
ports = {"http": "8000",
|
||||
"https": "8443",
|
||||
"ws": "8888"}
|
||||
|
||||
self.profile = FirefoxProfile(locations=locations,
|
||||
proxy=ports,
|
||||
preferences=preferences)
|
||||
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
|
||||
"marionette.defaultPrefs.port": self.marionette_port,
|
||||
"dom.disable_open_during_load": False})
|
||||
"dom.disable_open_during_load": False,
|
||||
"network.dns.localDomains": ",".join(hostnames)})
|
||||
|
||||
if self.ca_certificate_path is not None:
|
||||
self.setup_ssl()
|
||||
|
||||
debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
|
||||
self.debug_info)
|
||||
|
||||
self.runner = FirefoxRunner(profile=self.profile,
|
||||
binary=self.binary,
|
||||
cmdargs=[cmd_arg("marionette"), "about:blank"],
|
||||
binary=cmd[0],
|
||||
cmdargs=cmd[1:],
|
||||
env=env,
|
||||
process_class=ProcessHandler,
|
||||
process_args={"processOutputLine": [self.on_output]})
|
||||
|
||||
self.logger.debug("Starting Firefox")
|
||||
self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
|
||||
|
||||
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
|
||||
self.logger.debug("Firefox Started")
|
||||
|
||||
def load_prefs(self):
|
||||
|
|
|
|||
|
|
@ -26,8 +26,7 @@ def check_args(**kwargs):
|
|||
|
||||
def browser_kwargs(**kwargs):
|
||||
return {"binary": kwargs["binary"],
|
||||
"debug_args": kwargs["debug_args"],
|
||||
"interactive": kwargs["interactive"]}
|
||||
"debug_info": kwargs["debug_info"]}
|
||||
|
||||
|
||||
def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
|
||||
|
|
@ -39,17 +38,16 @@ def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
|
|||
def env_options():
|
||||
return {"host": "localhost",
|
||||
"bind_hostname": "true",
|
||||
"testharnessreport": "testharnessreport-servo.js"}
|
||||
"testharnessreport": "testharnessreport-servo.js",
|
||||
"supports_debugger": True}
|
||||
|
||||
|
||||
class ServoBrowser(NullBrowser):
|
||||
def __init__(self, logger, binary, debug_args=None, interactive=False):
|
||||
def __init__(self, logger, binary, debug_info=None):
|
||||
NullBrowser.__init__(self, logger)
|
||||
self.binary = binary
|
||||
self.debug_args = debug_args
|
||||
self.interactive = interactive
|
||||
self.debug_info = debug_info
|
||||
|
||||
def executor_browser(self):
|
||||
return ExecutorBrowser, {"binary": self.binary,
|
||||
"debug_args": self.debug_args,
|
||||
"interactive": self.interactive}
|
||||
"debug_info": self.debug_info}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
import json
|
||||
import os
|
||||
import multiprocessing
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
|
@ -18,6 +19,15 @@ here = os.path.split(__file__)[0]
|
|||
serve = None
|
||||
sslutils = None
|
||||
|
||||
|
||||
hostnames = ["web-platform.test",
|
||||
"www.web-platform.test",
|
||||
"www1.web-platform.test",
|
||||
"www2.web-platform.test",
|
||||
"xn--n8j6ds53lwwkrqhv28a.web-platform.test",
|
||||
"xn--lve-6lad.web-platform.test"]
|
||||
|
||||
|
||||
def do_delayed_imports(logger, test_paths):
|
||||
global serve, sslutils
|
||||
|
||||
|
|
@ -90,7 +100,7 @@ class StaticHandler(object):
|
|||
|
||||
|
||||
class TestEnvironment(object):
|
||||
def __init__(self, test_paths, ssl_env, pause_after_test, options):
|
||||
def __init__(self, test_paths, ssl_env, pause_after_test, debug_info, options):
|
||||
"""Context manager that owns the test environment i.e. the http and
|
||||
websockets servers"""
|
||||
self.test_paths = test_paths
|
||||
|
|
@ -100,11 +110,13 @@ class TestEnvironment(object):
|
|||
self.external_config = None
|
||||
self.pause_after_test = pause_after_test
|
||||
self.test_server_port = options.pop("test_server_port", True)
|
||||
self.debug_info = debug_info
|
||||
self.options = options if options is not None else {}
|
||||
|
||||
self.cache_manager = multiprocessing.Manager()
|
||||
self.routes = self.get_routes()
|
||||
|
||||
|
||||
def __enter__(self):
|
||||
self.ssl_env.__enter__()
|
||||
self.cache_manager.__enter__()
|
||||
|
|
@ -113,9 +125,12 @@ class TestEnvironment(object):
|
|||
serve.set_computed_defaults(self.config)
|
||||
self.external_config, self.servers = serve.start(self.config, self.ssl_env,
|
||||
self.routes)
|
||||
if self.options.get("supports_debugger") and self.debug_info and self.debug_info.interactive:
|
||||
self.ignore_interrupts()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.process_interrupts()
|
||||
self.cache_manager.__exit__(exc_type, exc_val, exc_tb)
|
||||
self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
|
||||
|
||||
|
|
@ -123,6 +138,12 @@ class TestEnvironment(object):
|
|||
for port, server in servers:
|
||||
server.kill()
|
||||
|
||||
def ignore_interrupts(self):
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
|
||||
def process_interrupts(self):
|
||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||
|
||||
def load_config(self):
|
||||
default_config_path = os.path.join(serve_path(self.test_paths), "config.default.json")
|
||||
local_config_path = os.path.join(here, "config.json")
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
|
|||
|
||||
executor_kwargs = {"server_config": server_config,
|
||||
"timeout_multiplier": timeout_multiplier,
|
||||
"debug_args": kwargs["debug_args"]}
|
||||
"debug_info": kwargs["debug_info"]}
|
||||
|
||||
if test_type == "reftest":
|
||||
executor_kwargs["screenshot_cache"] = cache_manager.dict()
|
||||
|
|
@ -81,7 +81,7 @@ class TestExecutor(object):
|
|||
convert_result = None
|
||||
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1,
|
||||
debug_args=None):
|
||||
debug_info=None):
|
||||
"""Abstract Base class for object that actually executes the tests in a
|
||||
specific browser. Typically there will be a different TestExecutor
|
||||
subclass for each test type and method of executing tests.
|
||||
|
|
@ -97,8 +97,9 @@ class TestExecutor(object):
|
|||
self.browser = browser
|
||||
self.server_config = server_config
|
||||
self.timeout_multiplier = timeout_multiplier
|
||||
self.debug_args = debug_args
|
||||
self.last_protocol = "http"
|
||||
self.debug_info = debug_info
|
||||
self.last_environment = {"protocol": "http",
|
||||
"prefs": []}
|
||||
self.protocol = None # This must be set in subclasses
|
||||
|
||||
@property
|
||||
|
|
@ -123,9 +124,8 @@ class TestExecutor(object):
|
|||
"""Run a particular test.
|
||||
|
||||
:param test: The test to run"""
|
||||
|
||||
if test.protocol != self.last_protocol:
|
||||
self.on_protocol_change(test.protocol)
|
||||
if test.environment != self.last_environment:
|
||||
self.on_environment_change(test.environment)
|
||||
|
||||
try:
|
||||
result = self.do_test(test)
|
||||
|
|
@ -138,7 +138,7 @@ class TestExecutor(object):
|
|||
if result[0].status == "ERROR":
|
||||
self.logger.debug(result[0].message)
|
||||
|
||||
self.last_protocol = test.protocol
|
||||
self.last_environment = test.environment
|
||||
|
||||
self.runner.send_message("test_ended", test, result)
|
||||
|
||||
|
|
@ -149,17 +149,17 @@ class TestExecutor(object):
|
|||
self.server_config["ports"][protocol][0])
|
||||
|
||||
def test_url(self, test):
|
||||
return urlparse.urljoin(self.server_url(test.protocol), test.url)
|
||||
return urlparse.urljoin(self.server_url(test.environment["protocol"]), test.url)
|
||||
|
||||
@abstractmethod
|
||||
def do_test(self, test):
|
||||
"""Test-type and protocol specific implmentation of running a
|
||||
"""Test-type and protocol specific implementation of running a
|
||||
specific test.
|
||||
|
||||
:param test: The test to run."""
|
||||
pass
|
||||
|
||||
def on_protocol_change(self, new_protocol):
|
||||
def on_environment_change(self, new_environment):
|
||||
pass
|
||||
|
||||
def result_from_exception(self, test, e):
|
||||
|
|
@ -182,10 +182,10 @@ class RefTestExecutor(TestExecutor):
|
|||
convert_result = reftest_result_converter
|
||||
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1, screenshot_cache=None,
|
||||
debug_args=None):
|
||||
debug_info=None):
|
||||
TestExecutor.__init__(self, browser, server_config,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_args=debug_args)
|
||||
debug_info=debug_info)
|
||||
|
||||
self.screenshot_cache = screenshot_cache
|
||||
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ class MarionetteProtocol(Protocol):
|
|||
while True:
|
||||
success = self.marionette.wait_for_port(60)
|
||||
#When running in a debugger wait indefinitely for firefox to start
|
||||
if success or self.executor.debug_args is None:
|
||||
if success or self.executor.debug_info is None:
|
||||
break
|
||||
|
||||
session_started = False
|
||||
|
|
@ -131,12 +131,80 @@ class MarionetteProtocol(Protocol):
|
|||
self.marionette.execute_async_script("");
|
||||
except errors.ScriptTimeoutException:
|
||||
pass
|
||||
except (socket.timeout, errors.InvalidResponseException, IOError):
|
||||
except (socket.timeout, IOError):
|
||||
break
|
||||
except Exception as e:
|
||||
self.logger.error(traceback.format_exc(e))
|
||||
break
|
||||
|
||||
def on_environment_change(self, old_environment, new_environment):
|
||||
#Unset all the old prefs
|
||||
for name, _ in old_environment.get("prefs", []):
|
||||
value = self.executor.original_pref_values[name]
|
||||
if value is None:
|
||||
self.clear_user_pref(name)
|
||||
else:
|
||||
self.set_pref(name, value)
|
||||
|
||||
for name, value in new_environment.get("prefs", []):
|
||||
self.executor.original_pref_values[name] = self.get_pref(name)
|
||||
self.set_pref(name, value)
|
||||
|
||||
def set_pref(self, name, value):
|
||||
self.logger.info("Setting pref %s (%s)" % (name, value))
|
||||
self.marionette.set_context(self.marionette.CONTEXT_CHROME)
|
||||
script = """
|
||||
let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
|
||||
.getService(Components.interfaces.nsIPrefBranch);
|
||||
let pref = '%s';
|
||||
let value = '%s';
|
||||
let type = prefInterface.getPrefType(pref);
|
||||
switch(type) {
|
||||
case prefInterface.PREF_STRING:
|
||||
prefInterface.setCharPref(pref, value);
|
||||
break;
|
||||
case prefInterface.PREF_BOOL:
|
||||
prefInterface.setBoolPref(pref, value);
|
||||
break;
|
||||
case prefInterface.PREF_INT:
|
||||
prefInterface.setIntPref(pref, value);
|
||||
break;
|
||||
}
|
||||
""" % (name, value)
|
||||
self.marionette.execute_script(script)
|
||||
self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
|
||||
|
||||
def clear_user_pref(self, name):
|
||||
self.logger.info("Clearing pref %s" % (name))
|
||||
self.marionette.set_context(self.marionette.CONTEXT_CHROME)
|
||||
script = """
|
||||
let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
|
||||
.getService(Components.interfaces.nsIPrefBranch);
|
||||
let pref = '%s';
|
||||
prefInterface.clearUserPref(pref);
|
||||
""" % name
|
||||
self.marionette.execute_script(script)
|
||||
self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
|
||||
|
||||
def get_pref(self, name):
|
||||
self.marionette.set_context(self.marionette.CONTEXT_CHROME)
|
||||
self.marionette.execute_script("""
|
||||
let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
|
||||
.getService(Components.interfaces.nsIPrefBranch);
|
||||
let pref = '%s';
|
||||
let type = prefInterface.getPrefType(pref);
|
||||
switch(type) {
|
||||
case prefInterface.PREF_STRING:
|
||||
return prefInterface.getCharPref(pref);
|
||||
case prefInterface.PREF_BOOL:
|
||||
return prefInterface.getBoolPref(pref);
|
||||
case prefInterface.PREF_INT:
|
||||
return prefInterface.getIntPref(pref);
|
||||
case prefInterface.PREF_INVALID:
|
||||
return null;
|
||||
}
|
||||
""" % (name))
|
||||
self.marionette.set_context(self.marionette.CONTEXT_CONTENT)
|
||||
|
||||
class MarionetteRun(object):
|
||||
def __init__(self, logger, func, marionette, url, timeout):
|
||||
|
|
@ -159,7 +227,7 @@ class MarionetteRun(object):
|
|||
# make that possible. It also seems to time out immediately if the
|
||||
# timeout is set too high. This works at least.
|
||||
self.marionette.set_script_timeout(2**31 - 1)
|
||||
except (IOError, errors.InvalidResponseException):
|
||||
except IOError:
|
||||
self.logger.error("Lost marionette connection before starting test")
|
||||
return Stop
|
||||
|
||||
|
|
@ -185,7 +253,7 @@ class MarionetteRun(object):
|
|||
except errors.ScriptTimeoutException:
|
||||
self.logger.debug("Got a marionette timeout")
|
||||
self.result = False, ("EXTERNAL-TIMEOUT", None)
|
||||
except (socket.timeout, errors.InvalidResponseException, IOError):
|
||||
except (socket.timeout, IOError):
|
||||
# This can happen on a crash
|
||||
# Also, should check after the test if the firefox process is still running
|
||||
# and otherwise ignore any other result and set it to crash
|
||||
|
|
@ -203,28 +271,33 @@ class MarionetteRun(object):
|
|||
|
||||
class MarionetteTestharnessExecutor(TestharnessExecutor):
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1, close_after_done=True,
|
||||
debug_args=None):
|
||||
debug_info=None):
|
||||
"""Marionette-based executor for testharness.js tests"""
|
||||
TestharnessExecutor.__init__(self, browser, server_config,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_args=debug_args)
|
||||
debug_info=debug_info)
|
||||
|
||||
self.protocol = MarionetteProtocol(self, browser)
|
||||
self.script = open(os.path.join(here, "testharness_marionette.js")).read()
|
||||
self.close_after_done = close_after_done
|
||||
self.window_id = str(uuid.uuid4())
|
||||
|
||||
self.original_pref_values = {}
|
||||
|
||||
if marionette is None:
|
||||
do_delayed_imports()
|
||||
|
||||
def is_alive(self):
|
||||
return self.protocol.is_alive()
|
||||
|
||||
def on_protocol_change(self, new_protocol):
|
||||
self.protocol.load_runner(new_protocol)
|
||||
def on_environment_change(self, new_environment):
|
||||
self.protocol.on_environment_change(self.last_environment, new_environment)
|
||||
|
||||
if new_environment["protocol"] != self.last_environment["protocol"]:
|
||||
self.protocol.load_runner(new_environment["protocol"])
|
||||
|
||||
def do_test(self, test):
|
||||
timeout = (test.timeout * self.timeout_multiplier if self.debug_args is None
|
||||
timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
|
||||
else None)
|
||||
|
||||
success, data = MarionetteRun(self.logger,
|
||||
|
|
@ -258,18 +331,19 @@ class MarionetteTestharnessExecutor(TestharnessExecutor):
|
|||
|
||||
class MarionetteRefTestExecutor(RefTestExecutor):
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1,
|
||||
screenshot_cache=None, close_after_done=True, debug_args=None):
|
||||
screenshot_cache=None, close_after_done=True, debug_info=None):
|
||||
"""Marionette-based executor for reftests"""
|
||||
RefTestExecutor.__init__(self,
|
||||
browser,
|
||||
server_config,
|
||||
screenshot_cache=screenshot_cache,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_args=debug_args)
|
||||
debug_info=debug_info)
|
||||
self.protocol = MarionetteProtocol(self, browser)
|
||||
self.implementation = RefTestImplementation(self)
|
||||
self.close_after_done = close_after_done
|
||||
self.has_window = False
|
||||
self.original_pref_values = {}
|
||||
|
||||
with open(os.path.join(here, "reftest.js")) as f:
|
||||
self.script = f.read()
|
||||
|
|
@ -279,6 +353,9 @@ class MarionetteRefTestExecutor(RefTestExecutor):
|
|||
def is_alive(self):
|
||||
return self.protocol.is_alive()
|
||||
|
||||
def on_environment_change(self, new_environment):
|
||||
self.protocol.on_environment_change(self.last_environment, new_environment)
|
||||
|
||||
def do_test(self, test):
|
||||
if self.close_after_done and self.has_window:
|
||||
self.protocol.marionette.close()
|
||||
|
|
@ -296,7 +373,7 @@ class MarionetteRefTestExecutor(RefTestExecutor):
|
|||
return self.convert_result(test, result)
|
||||
|
||||
def screenshot(self, test):
|
||||
timeout = test.timeout if self.debug_args is None else None
|
||||
timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None
|
||||
|
||||
test_url = self.test_url(test)
|
||||
|
||||
|
|
|
|||
|
|
@ -165,11 +165,11 @@ class SeleniumRun(object):
|
|||
|
||||
class SeleniumTestharnessExecutor(TestharnessExecutor):
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1,
|
||||
close_after_done=True, capabilities=None, debug_args=None):
|
||||
close_after_done=True, capabilities=None, debug_info=None):
|
||||
"""Selenium-based executor for testharness.js tests"""
|
||||
TestharnessExecutor.__init__(self, browser, server_config,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_args=debug_args)
|
||||
debug_info=debug_info)
|
||||
self.protocol = SeleniumProtocol(self, browser, capabilities)
|
||||
with open(os.path.join(here, "testharness_webdriver.js")) as f:
|
||||
self.script = f.read()
|
||||
|
|
@ -206,14 +206,14 @@ class SeleniumTestharnessExecutor(TestharnessExecutor):
|
|||
class SeleniumRefTestExecutor(RefTestExecutor):
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1,
|
||||
screenshot_cache=None, close_after_done=True,
|
||||
debug_args=None, capabilities=None):
|
||||
debug_info=None, capabilities=None):
|
||||
"""Selenium WebDriver-based executor for reftests"""
|
||||
RefTestExecutor.__init__(self,
|
||||
browser,
|
||||
server_config,
|
||||
screenshot_cache=screenshot_cache,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_args=debug_args)
|
||||
debug_info=debug_info)
|
||||
self.protocol = SeleniumProtocol(self, browser,
|
||||
capabilities=capabilities)
|
||||
self.implementation = RefTestImplementation(self)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from .base import (ExecutorException,
|
|||
testharness_result_converter,
|
||||
reftest_result_converter)
|
||||
from .process import ProcessTestExecutor
|
||||
from ..browsers.base import browser_command
|
||||
|
||||
hosts_text = """127.0.0.1 web-platform.test
|
||||
127.0.0.1 www.web-platform.test
|
||||
|
|
@ -39,11 +40,11 @@ def make_hosts_file():
|
|||
class ServoTestharnessExecutor(ProcessTestExecutor):
|
||||
convert_result = testharness_result_converter
|
||||
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1, debug_args=None,
|
||||
def __init__(self, browser, server_config, timeout_multiplier=1, debug_info=None,
|
||||
pause_after_test=False):
|
||||
ProcessTestExecutor.__init__(self, browser, server_config,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_args=debug_args)
|
||||
debug_info=debug_info)
|
||||
self.pause_after_test = pause_after_test
|
||||
self.result_data = None
|
||||
self.result_flag = None
|
||||
|
|
@ -61,40 +62,48 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
|
|||
self.result_data = None
|
||||
self.result_flag = threading.Event()
|
||||
|
||||
self.command = [self.binary, "--cpu", "--hard-fail", "-z", self.test_url(test)]
|
||||
debug_args, command = browser_command(self.binary, ["--cpu", "--hard-fail", "-z", self.test_url(test)],
|
||||
self.debug_info)
|
||||
|
||||
self.command = command
|
||||
|
||||
if self.pause_after_test:
|
||||
self.command.remove("-z")
|
||||
|
||||
if self.debug_args:
|
||||
self.command = list(self.debug_args) + self.command
|
||||
self.command = debug_args + self.command
|
||||
|
||||
env = os.environ.copy()
|
||||
env["HOST_FILE"] = self.hosts_path
|
||||
|
||||
self.proc = ProcessHandler(self.command,
|
||||
processOutputLine=[self.on_output],
|
||||
onFinish=self.on_finish,
|
||||
env=env)
|
||||
|
||||
|
||||
if not self.interactive:
|
||||
self.proc = ProcessHandler(self.command,
|
||||
processOutputLine=[self.on_output],
|
||||
onFinish=self.on_finish,
|
||||
env=env,
|
||||
storeOutput=False)
|
||||
self.proc.run()
|
||||
else:
|
||||
self.proc = subprocess.Popen(self.command, env=env)
|
||||
|
||||
try:
|
||||
self.proc.run()
|
||||
|
||||
timeout = test.timeout * self.timeout_multiplier
|
||||
|
||||
# Now wait to get the output we expect, or until we reach the timeout
|
||||
if self.debug_args is None and not self.pause_after_test:
|
||||
if not self.interactive and not self.pause_after_test:
|
||||
wait_timeout = timeout + 5
|
||||
self.result_flag.wait(wait_timeout)
|
||||
else:
|
||||
wait_timeout = None
|
||||
self.result_flag.wait(wait_timeout)
|
||||
self.proc.wait()
|
||||
|
||||
proc_is_running = True
|
||||
if self.result_flag.is_set() and self.result_data is not None:
|
||||
self.result_data["test"] = test.url
|
||||
result = self.convert_result(test, self.result_data)
|
||||
else:
|
||||
if self.proc.proc.poll() is not None:
|
||||
if self.proc.poll() is not None:
|
||||
result = (test.result_cls("CRASH", None), [])
|
||||
proc_is_running = False
|
||||
else:
|
||||
|
|
@ -150,13 +159,13 @@ class ServoRefTestExecutor(ProcessTestExecutor):
|
|||
convert_result = reftest_result_converter
|
||||
|
||||
def __init__(self, browser, server_config, binary=None, timeout_multiplier=1,
|
||||
screenshot_cache=None, debug_args=None, pause_after_test=False):
|
||||
screenshot_cache=None, debug_info=None, pause_after_test=False):
|
||||
|
||||
ProcessTestExecutor.__init__(self,
|
||||
browser,
|
||||
server_config,
|
||||
timeout_multiplier=timeout_multiplier,
|
||||
debug_args=debug_args)
|
||||
debug_info=debug_info)
|
||||
|
||||
self.protocol = Protocol(self, browser)
|
||||
self.screenshot_cache = screenshot_cache
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ class ProcessTestExecutor(TestExecutor):
|
|||
def __init__(self, *args, **kwargs):
|
||||
TestExecutor.__init__(self, *args, **kwargs)
|
||||
self.binary = self.browser.binary
|
||||
self.interactive = self.browser.interactive
|
||||
self.interactive = (False if self.debug_info is None
|
||||
else self.debug_info.interactive)
|
||||
|
||||
def setup(self, runner):
|
||||
self.runner = runner
|
||||
|
|
|
|||
|
|
@ -107,6 +107,15 @@ class TestNode(ManifestItem):
|
|||
except KeyError:
|
||||
return False
|
||||
|
||||
def prefs(self):
|
||||
try:
|
||||
prefs = self.get("prefs")
|
||||
if type(prefs) in (str, unicode):
|
||||
prefs = [prefs]
|
||||
return [item.split(":", 1) for item in prefs]
|
||||
except KeyError:
|
||||
return []
|
||||
|
||||
def append(self, node):
|
||||
"""Add a subtest to the current test
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ representing the file and each subnode representing a subdirectory that should
|
|||
be included or excluded.
|
||||
"""
|
||||
import os
|
||||
import urlparse
|
||||
|
||||
from wptmanifest.node import DataNode
|
||||
from wptmanifest.backends import conditional
|
||||
|
|
@ -42,7 +43,7 @@ class IncludeManifest(ManifestItem):
|
|||
this object.
|
||||
|
||||
:param test: The test object"""
|
||||
path_components = self._get_path_components(test)
|
||||
path_components = self._get_components(test.url)
|
||||
return self._include(test, path_components)
|
||||
|
||||
def _include(self, test, path_components):
|
||||
|
|
@ -64,21 +65,41 @@ class IncludeManifest(ManifestItem):
|
|||
# Include by default
|
||||
return True
|
||||
|
||||
def _get_path_components(self, test):
|
||||
test_url = test.url
|
||||
assert test_url[0] == "/"
|
||||
return [item for item in reversed(test_url.split("/")) if item]
|
||||
def _get_components(self, url):
|
||||
rv = []
|
||||
url_parts = urlparse.urlsplit(url)
|
||||
variant = ""
|
||||
if url_parts.query:
|
||||
variant += "?" + url_parts.query
|
||||
if url_parts.fragment:
|
||||
variant += "#" + url_parts.fragment
|
||||
if variant:
|
||||
rv.append(variant)
|
||||
rv.extend([item for item in reversed(url_parts.path.split("/")) if item])
|
||||
return rv
|
||||
|
||||
def _add_rule(self, test_manifests, url, direction):
|
||||
maybe_path = os.path.abspath(os.path.join(os.curdir, url))
|
||||
maybe_path = os.path.join(os.path.abspath(os.curdir), url)
|
||||
rest, last = os.path.split(maybe_path)
|
||||
variant = ""
|
||||
if "#" in last:
|
||||
last, fragment = last.rsplit("#", 1)
|
||||
variant += "#" + fragment
|
||||
if "?" in last:
|
||||
last, query = last.rsplit("?", 1)
|
||||
variant += "?" + query
|
||||
|
||||
maybe_path = os.path.join(rest, last)
|
||||
|
||||
if os.path.exists(maybe_path):
|
||||
for manifest, data in test_manifests.iteritems():
|
||||
rel_path = os.path.relpath(maybe_path, data["tests_path"])
|
||||
if ".." not in rel_path.split(os.sep):
|
||||
url = rel_path
|
||||
url = data["url_base"] + rel_path.replace(os.path.sep, "/") + variant
|
||||
break
|
||||
|
||||
assert direction in ("include", "exclude")
|
||||
components = [item for item in reversed(url.split("/")) if item]
|
||||
components = self._get_components(url)
|
||||
|
||||
node = self
|
||||
while components:
|
||||
|
|
|
|||
|
|
@ -186,7 +186,7 @@ def write_new_expected(metadata_path, expected_map):
|
|||
if not os.path.exists(dir):
|
||||
os.makedirs(dir)
|
||||
with open(path, "w") as f:
|
||||
f.write(manifest_str.encode("utf8"))
|
||||
f.write(manifest_str)
|
||||
|
||||
|
||||
class ExpectedUpdater(object):
|
||||
|
|
|
|||
|
|
@ -190,8 +190,6 @@ class EqualTimeChunker(TestChunker):
|
|||
|
||||
class TestFilter(object):
|
||||
def __init__(self, test_manifests, include=None, exclude=None, manifest_path=None):
|
||||
test_manifests = test_manifests
|
||||
|
||||
if manifest_path is not None and include is None:
|
||||
self.manifest = manifestinclude.get_manifest(manifest_path)
|
||||
else:
|
||||
|
|
@ -355,7 +353,7 @@ class TestLoader(object):
|
|||
|
||||
for test_path, test_type, test in self.iter_tests():
|
||||
enabled = not test.disabled()
|
||||
if not self.include_https and test.protocol == "https":
|
||||
if not self.include_https and test.environment["protocol"] == "https":
|
||||
enabled = False
|
||||
key = "enabled" if enabled else "disabled"
|
||||
tests[key][test_type].append(test)
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ class TestRunnerManager(threading.Thread):
|
|||
|
||||
def __init__(self, suite_name, test_queue, test_source_cls, browser_cls, browser_kwargs,
|
||||
executor_cls, executor_kwargs, stop_flag, pause_after_test=False,
|
||||
pause_on_unexpected=False, debug_args=None):
|
||||
pause_on_unexpected=False, debug_info=None):
|
||||
"""Thread that owns a single TestRunner process and any processes required
|
||||
by the TestRunner (e.g. the Firefox binary).
|
||||
|
||||
|
|
@ -206,7 +206,7 @@ class TestRunnerManager(threading.Thread):
|
|||
|
||||
self.pause_after_test = pause_after_test
|
||||
self.pause_on_unexpected = pause_on_unexpected
|
||||
self.debug_args = debug_args
|
||||
self.debug_info = debug_info
|
||||
|
||||
self.manager_number = next_manager_number()
|
||||
|
||||
|
|
@ -333,7 +333,7 @@ class TestRunnerManager(threading.Thread):
|
|||
with self.init_lock:
|
||||
# Guard against problems initialising the browser or the browser
|
||||
# remote control method
|
||||
if self.debug_args is None:
|
||||
if self.debug_info is None:
|
||||
self.init_timer = threading.Timer(self.browser.init_timeout, init_failed)
|
||||
|
||||
test_queue = self.test_source.get_queue()
|
||||
|
|
@ -560,7 +560,6 @@ class TestQueue(object):
|
|||
self.test_type = test_type
|
||||
self.tests = tests
|
||||
self.kwargs = kwargs
|
||||
self.queue = None
|
||||
|
||||
def __enter__(self):
|
||||
if not self.tests[self.test_type]:
|
||||
|
|
@ -590,7 +589,7 @@ class ManagerGroup(object):
|
|||
executor_cls, executor_kwargs,
|
||||
pause_after_test=False,
|
||||
pause_on_unexpected=False,
|
||||
debug_args=None):
|
||||
debug_info=None):
|
||||
"""Main thread object that owns all the TestManager threads."""
|
||||
self.suite_name = suite_name
|
||||
self.size = size
|
||||
|
|
@ -602,7 +601,7 @@ class ManagerGroup(object):
|
|||
self.executor_kwargs = executor_kwargs
|
||||
self.pause_after_test = pause_after_test
|
||||
self.pause_on_unexpected = pause_on_unexpected
|
||||
self.debug_args = debug_args
|
||||
self.debug_info = debug_info
|
||||
|
||||
self.pool = set()
|
||||
# Event that is polled by threads so that they can gracefully exit in the face
|
||||
|
|
@ -640,7 +639,7 @@ class ManagerGroup(object):
|
|||
self.stop_flag,
|
||||
self.pause_after_test,
|
||||
self.pause_on_unexpected,
|
||||
self.debug_args)
|
||||
self.debug_info)
|
||||
manager.start()
|
||||
self.pool.add(manager)
|
||||
self.wait()
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from .. import testloader
|
||||
|
|
@ -93,6 +94,14 @@ class UpdateCheckout(Step):
|
|||
sync_tree.update(state.sync["remote_url"],
|
||||
state.sync["branch"],
|
||||
state.local_branch)
|
||||
sync_path = os.path.abspath(sync_tree.root)
|
||||
if not sync_path in sys.path:
|
||||
from update import setup_paths
|
||||
setup_paths(sync_path)
|
||||
|
||||
def restore(self, state):
|
||||
assert os.path.abspath(state.sync_tree.root) in sys.path
|
||||
Step.restore(self, state)
|
||||
|
||||
|
||||
class GetSyncTargetCommit(Step):
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from base import Step, StepRunner, exit_clean, exit_unclean
|
|||
from state import State
|
||||
|
||||
def setup_paths(sync_path):
|
||||
sys.path.insert(0, sync_path)
|
||||
sys.path.insert(0, os.path.abspath(sync_path))
|
||||
from tools import localpaths
|
||||
|
||||
class LoadConfig(Step):
|
||||
|
|
@ -117,7 +117,9 @@ class WPTUpdate(object):
|
|||
if not kwargs["sync"]:
|
||||
setup_paths(self.serve_root)
|
||||
else:
|
||||
setup_paths(kwargs["sync_path"])
|
||||
if os.path.exists(kwargs["sync_path"]):
|
||||
# If the sync path doesn't exist we defer this until it does
|
||||
setup_paths(kwargs["sync_path"])
|
||||
|
||||
self.state = State(logger)
|
||||
self.kwargs = kwargs
|
||||
|
|
|
|||
|
|
@ -25,12 +25,6 @@ def url_or_path(path):
|
|||
else:
|
||||
return abs_path(path)
|
||||
|
||||
def slash_prefixed(url):
|
||||
if not url.startswith("/"):
|
||||
url = "/" + url
|
||||
return url
|
||||
|
||||
|
||||
def require_arg(kwargs, name, value_func=None):
|
||||
if value_func is None:
|
||||
value_func = lambda x: x is not None
|
||||
|
|
@ -97,15 +91,15 @@ def create_parser(product_choices=None):
|
|||
nargs="*", default=["testharness", "reftest"],
|
||||
choices=["testharness", "reftest"],
|
||||
help="Test types to run")
|
||||
test_selection_group.add_argument("--include", action="append", type=slash_prefixed,
|
||||
test_selection_group.add_argument("--include", action="append",
|
||||
help="URL prefix to include")
|
||||
test_selection_group.add_argument("--exclude", action="append", type=slash_prefixed,
|
||||
test_selection_group.add_argument("--exclude", action="append",
|
||||
help="URL prefix to exclude")
|
||||
test_selection_group.add_argument("--include-manifest", type=abs_path,
|
||||
help="Path to manifest listing tests to include")
|
||||
|
||||
debugging_group = parser.add_argument_group("Debugging")
|
||||
debugging_group.add_argument('--debugger',
|
||||
debugging_group.add_argument('--debugger', const="__default__", nargs="?",
|
||||
help="run under a debugger, e.g. gdb or valgrind")
|
||||
debugging_group.add_argument('--debugger-args', help="arguments to the debugger")
|
||||
|
||||
|
|
@ -233,8 +227,6 @@ def exe_path(name):
|
|||
|
||||
|
||||
def check_args(kwargs):
|
||||
from mozrunner import debugger_arguments
|
||||
|
||||
set_from_config(kwargs)
|
||||
|
||||
for test_paths in kwargs["test_paths"].itervalues():
|
||||
|
|
@ -278,16 +270,18 @@ def check_args(kwargs):
|
|||
kwargs["processes"] = 1
|
||||
|
||||
if kwargs["debugger"] is not None:
|
||||
debug_args, interactive = debugger_arguments(kwargs["debugger"],
|
||||
kwargs["debugger_args"])
|
||||
if interactive:
|
||||
require_arg(kwargs, "processes", lambda x: x == 1)
|
||||
import mozdebug
|
||||
if kwargs["debugger"] == "__default__":
|
||||
kwargs["debugger"] = mozdebug.get_default_debugger_name()
|
||||
debug_info = mozdebug.get_debugger_info(kwargs["debugger"],
|
||||
kwargs["debugger_args"])
|
||||
if debug_info.interactive:
|
||||
if kwargs["processes"] != 1:
|
||||
kwargs["processes"] = 1
|
||||
kwargs["no_capture_stdio"] = True
|
||||
kwargs["interactive"] = interactive
|
||||
kwargs["debug_args"] = debug_args
|
||||
kwargs["debug_info"] = debug_info
|
||||
else:
|
||||
kwargs["interactive"] = False
|
||||
kwargs["debug_args"] = None
|
||||
kwargs["debug_info"] = None
|
||||
|
||||
if kwargs["binary"] is not None:
|
||||
if not os.path.exists(kwargs["binary"]):
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
import operator
|
||||
|
||||
from ..node import NodeVisitor, DataNode, ConditionalNode, KeyValueNode, ValueNode
|
||||
from ..node import NodeVisitor, DataNode, ConditionalNode, KeyValueNode, ListNode, ValueNode
|
||||
from ..parser import parse
|
||||
|
||||
|
||||
|
|
@ -17,13 +17,16 @@ class ConditionalValue(object):
|
|||
self.condition_node = self.node.children[0]
|
||||
self.value_node = self.node.children[1]
|
||||
else:
|
||||
assert isinstance(node, ValueNode)
|
||||
assert isinstance(node, (ValueNode, ListNode))
|
||||
self.condition_node = None
|
||||
self.value_node = self.node
|
||||
|
||||
@property
|
||||
def value(self):
|
||||
return self.value_node.data
|
||||
if isinstance(self.value_node, ValueNode):
|
||||
return self.value_node.data
|
||||
else:
|
||||
return [item.data for item in self.value_node.children]
|
||||
|
||||
@value.setter
|
||||
def value(self, value):
|
||||
|
|
@ -106,6 +109,9 @@ class Compiler(NodeVisitor):
|
|||
|
||||
self.output_node._add_key_value(node, key_values)
|
||||
|
||||
def visit_ListNode(self, node):
|
||||
return (lambda x:True, [self.visit(child) for child in node.children])
|
||||
|
||||
def visit_ValueNode(self, node):
|
||||
return (lambda x: True, node.data)
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,9 @@ class Compiler(NodeVisitor):
|
|||
def visit_ValueNode(self, node):
|
||||
return node.data
|
||||
|
||||
def visit_ListNode(self, node):
|
||||
return [self.visit(child) for child in node.children]
|
||||
|
||||
def visit_ConditionalNode(self, node):
|
||||
assert len(node.children) == 2
|
||||
if self.visit(node.children[0]):
|
||||
|
|
|
|||
|
|
@ -82,6 +82,12 @@ class KeyValueNode(Node):
|
|||
self.children.append(other)
|
||||
|
||||
|
||||
class ListNode(Node):
|
||||
def append(self, other):
|
||||
other.parent = self
|
||||
self.children.append(other)
|
||||
|
||||
|
||||
class ValueNode(Node):
|
||||
def append(self, other):
|
||||
raise TypeError
|
||||
|
|
|
|||
|
|
@ -23,7 +23,12 @@ from node import *
|
|||
|
||||
|
||||
class ParseError(Exception):
|
||||
pass
|
||||
def __init__(self, filename, line, detail):
|
||||
self.line = line
|
||||
self.filename = filename
|
||||
self.detail = detail
|
||||
self.message = "%s: %s line %s" % (self.detail, self.filename, self.line)
|
||||
Exception.__init__(self, self.message)
|
||||
|
||||
eol = object
|
||||
group_start = object
|
||||
|
|
@ -41,7 +46,7 @@ operators = ["==", "!=", "not", "and", "or"]
|
|||
|
||||
|
||||
def decode(byte_str):
|
||||
return byte_str.decode("string_escape").decode("utf8")
|
||||
return byte_str.decode("utf8")
|
||||
|
||||
|
||||
def precedence(operator_node):
|
||||
|
|
@ -50,7 +55,7 @@ def precedence(operator_node):
|
|||
|
||||
class TokenTypes(object):
|
||||
def __init__(self):
|
||||
for type in ["group_start", "group_end", "paren", "separator", "ident", "string", "number", "eof"]:
|
||||
for type in ["group_start", "group_end", "paren", "list_start", "list_end", "separator", "ident", "string", "number", "eof"]:
|
||||
setattr(self, type, type)
|
||||
|
||||
token_types = TokenTypes()
|
||||
|
|
@ -70,18 +75,27 @@ class Tokenizer(object):
|
|||
self.reset()
|
||||
if type(stream) in types.StringTypes:
|
||||
stream = StringIO(stream)
|
||||
if not hasattr(stream, "name"):
|
||||
self.filename = ""
|
||||
else:
|
||||
self.filename = stream.name
|
||||
|
||||
self.next_line_state = self.line_start_state
|
||||
for i, line in enumerate(stream):
|
||||
self.state = self.line_start_state
|
||||
self.state = self.next_line_state
|
||||
assert self.state is not None
|
||||
states = []
|
||||
self.next_line_state = None
|
||||
self.line_number = i + 1
|
||||
self.index = 0
|
||||
self.line = line.rstrip()
|
||||
if self.line:
|
||||
while self.state != self.eol_state:
|
||||
tokens = self.state()
|
||||
if tokens:
|
||||
for token in tokens:
|
||||
yield token
|
||||
while self.state != self.eol_state:
|
||||
states.append(self.state)
|
||||
tokens = self.state()
|
||||
if tokens:
|
||||
for token in tokens:
|
||||
yield token
|
||||
self.state()
|
||||
while True:
|
||||
yield (token_types.eof, None)
|
||||
|
||||
|
|
@ -102,11 +116,14 @@ class Tokenizer(object):
|
|||
self.consume()
|
||||
|
||||
def eol_state(self):
|
||||
pass
|
||||
if self.next_line_state is None:
|
||||
self.next_line_state = self.line_start_state
|
||||
|
||||
def line_start_state(self):
|
||||
self.skip_whitespace()
|
||||
assert self.char() != eol
|
||||
if self.char() == eol:
|
||||
self.state = self.eol_state
|
||||
return
|
||||
if self.index > self.indent_levels[-1]:
|
||||
self.indent_levels.append(self.index)
|
||||
yield (token_types.group_start, None)
|
||||
|
|
@ -119,7 +136,7 @@ class Tokenizer(object):
|
|||
# it must always be a heading or key next so we go back to data_line_state
|
||||
self.next_state = self.data_line_state
|
||||
if self.index != self.indent_levels[-1]:
|
||||
raise ParseError("Unexpected indent")
|
||||
raise ParseError(self.filename, self.line_number, "Unexpected indent")
|
||||
|
||||
self.state = self.next_state
|
||||
|
||||
|
|
@ -132,57 +149,44 @@ class Tokenizer(object):
|
|||
self.state = self.key_state
|
||||
|
||||
def heading_state(self):
|
||||
index_0 = self.index
|
||||
skip_indexes = []
|
||||
rv = ""
|
||||
while True:
|
||||
c = self.char()
|
||||
if c == "\\":
|
||||
self.consume()
|
||||
c = self.char()
|
||||
if c == eol:
|
||||
raise ParseError("Unexpected EOL in heading")
|
||||
elif c == "]":
|
||||
skip_indexes.append(self.index - 1)
|
||||
self.consume()
|
||||
rv += self.consume_escape()
|
||||
elif c == "]":
|
||||
break
|
||||
elif c == eol:
|
||||
raise ParseError("EOL in heading")
|
||||
raise ParseError(self.filename, self.line_number, "EOL in heading")
|
||||
else:
|
||||
rv += c
|
||||
self.consume()
|
||||
|
||||
self.state = self.line_end_state
|
||||
index_1 = self.index
|
||||
parts = []
|
||||
min_index = index_0
|
||||
for index in skip_indexes:
|
||||
parts.append(self.line[min_index:index])
|
||||
min_index = index + 1
|
||||
parts.append(self.line[min_index:index_1])
|
||||
yield (token_types.string, decode("".join(parts)))
|
||||
yield (token_types.string, decode(rv))
|
||||
yield (token_types.paren, "]")
|
||||
self.consume()
|
||||
self.state = self.line_end_state
|
||||
self.next_state = self.data_line_state
|
||||
|
||||
def key_state(self):
|
||||
index_0 = self.index
|
||||
rv = ""
|
||||
while True:
|
||||
c = self.char()
|
||||
if c == " ":
|
||||
index_1 = self.index
|
||||
self.skip_whitespace()
|
||||
if self.char() != ":":
|
||||
raise ParseError("Space in key name")
|
||||
raise ParseError(self.filename, self.line_number, "Space in key name")
|
||||
break
|
||||
elif c == ":":
|
||||
index_1 = self.index
|
||||
break
|
||||
elif c == eol:
|
||||
raise ParseError("EOL in key name (missing ':'?)")
|
||||
raise ParseError(self.filename, self.line_number, "EOL in key name (missing ':'?)")
|
||||
elif c == "\\":
|
||||
rv += self.consume_escape()
|
||||
else:
|
||||
rv += c
|
||||
self.consume()
|
||||
yield (token_types.string, decode(self.line[index_0:index_1]))
|
||||
yield (token_types.string, decode(rv))
|
||||
yield (token_types.separator, ":")
|
||||
self.consume()
|
||||
self.state = self.after_key_state
|
||||
|
|
@ -196,39 +200,115 @@ class Tokenizer(object):
|
|||
elif c == eol:
|
||||
self.next_state = self.expr_or_value_state
|
||||
self.state = self.eol_state
|
||||
elif c == "[":
|
||||
self.state = self.list_start_state
|
||||
else:
|
||||
self.state = self.value_state
|
||||
|
||||
def list_start_state(self):
|
||||
yield (token_types.list_start, "[")
|
||||
self.consume()
|
||||
self.state = self.list_value_start_state
|
||||
|
||||
def list_value_start_state(self):
|
||||
self.skip_whitespace()
|
||||
if self.char() == "]":
|
||||
self.state = self.list_end_state
|
||||
elif self.char() in ("'", '"'):
|
||||
quote_char = self.char()
|
||||
self.consume()
|
||||
yield (token_types.string, self.consume_string(quote_char))
|
||||
self.skip_whitespace()
|
||||
if self.char() == "]":
|
||||
self.state = self.list_end_state
|
||||
elif self.char() != ",":
|
||||
raise ParseError(self.filename, self.line_number, "Junk after quoted string")
|
||||
self.consume()
|
||||
elif self.char() == "#":
|
||||
self.state = self.comment_state
|
||||
self.next_line_state = self.list_value_start_state
|
||||
elif self.char() == eol:
|
||||
self.next_line_state = self.list_value_start_state
|
||||
self.state = self.eol_state
|
||||
elif self.char() == ",":
|
||||
raise ParseError(self.filename, self.line_number, "List item started with separator")
|
||||
else:
|
||||
self.state = self.list_value_state
|
||||
|
||||
def list_value_state(self):
|
||||
rv = ""
|
||||
spaces = 0
|
||||
while True:
|
||||
c = self.char()
|
||||
if c == "\\":
|
||||
escape = self.consume_escape()
|
||||
rv += escape
|
||||
elif c == eol:
|
||||
raise ParseError(self.filename, self.line_number, "EOL in list value")
|
||||
elif c == "#":
|
||||
raise ParseError(self.filename, self.line_number, "EOL in list value (comment)")
|
||||
elif c == ",":
|
||||
self.state = self.list_value_start_state
|
||||
self.consume()
|
||||
break
|
||||
elif c == " ":
|
||||
spaces += 1
|
||||
self.consume()
|
||||
elif c == "]":
|
||||
self.state = self.list_end_state
|
||||
self.consume()
|
||||
break
|
||||
else:
|
||||
rv += " " * spaces
|
||||
spaces = 0
|
||||
rv += c
|
||||
self.consume()
|
||||
|
||||
if rv:
|
||||
yield (token_types.string, decode(rv))
|
||||
|
||||
def list_end_state(self):
|
||||
self.consume()
|
||||
yield (token_types.list_end, "]")
|
||||
self.state = self.line_end_state
|
||||
|
||||
def value_state(self):
|
||||
self.skip_whitespace()
|
||||
index_0 = self.index
|
||||
if self.char() in ("'", '"'):
|
||||
quote_char = self.char()
|
||||
self.consume()
|
||||
yield (token_types.string, decode(self.read_string(quote_char)))
|
||||
yield (token_types.string, self.consume_string(quote_char))
|
||||
if self.char() == "#":
|
||||
self.state = self.comment_state
|
||||
else:
|
||||
self.state = self.line_end_state
|
||||
else:
|
||||
index_1 = self.index
|
||||
rv = ""
|
||||
spaces = 0
|
||||
while True:
|
||||
c = self.char()
|
||||
if c == "\\":
|
||||
self.consume()
|
||||
if self.char() == eol:
|
||||
raise ParseError("EOL in character escape")
|
||||
rv += self.consume_escape()
|
||||
elif c == "#":
|
||||
self.state = self.comment_state
|
||||
break
|
||||
elif c == " ":
|
||||
# prevent whitespace before comments from being included in the value
|
||||
pass
|
||||
spaces += 1
|
||||
self.consume()
|
||||
elif c == eol:
|
||||
self.state = self.line_end_state
|
||||
break
|
||||
else:
|
||||
index_1 = self.index
|
||||
self.consume()
|
||||
yield (token_types.string, decode(self.line[index_0:index_1 + 1]))
|
||||
self.state = self.line_end_state
|
||||
rv += " " * spaces
|
||||
spaces = 0
|
||||
rv += c
|
||||
self.consume()
|
||||
yield (token_types.string, decode(rv))
|
||||
|
||||
def comment_state(self):
|
||||
while self.char() is not eol:
|
||||
self.consume()
|
||||
self.state = self.eol_state
|
||||
|
||||
def line_end_state(self):
|
||||
|
|
@ -239,26 +319,24 @@ class Tokenizer(object):
|
|||
elif c == eol:
|
||||
self.state = self.eol_state
|
||||
else:
|
||||
raise ParseError("Junk before EOL c")
|
||||
raise ParseError(self.filename, self.line_number, "Junk before EOL %s" % c)
|
||||
|
||||
def read_string(self, quote_char):
|
||||
index_0 = self.index
|
||||
def consume_string(self, quote_char):
|
||||
rv = ""
|
||||
while True:
|
||||
c = self.char()
|
||||
if c == "\\":
|
||||
self.consume()
|
||||
if self.char == eol:
|
||||
raise ParseError("EOL following quote")
|
||||
self.consume()
|
||||
rv += self.consume_escape()
|
||||
elif c == quote_char:
|
||||
self.consume()
|
||||
break
|
||||
elif c == eol:
|
||||
raise ParseError("EOL in quoted string")
|
||||
raise ParseError(self.filename, self.line_number, "EOL in quoted string")
|
||||
else:
|
||||
rv += c
|
||||
self.consume()
|
||||
rv = self.line[index_0:self.index]
|
||||
self.consume()
|
||||
return rv
|
||||
|
||||
return decode(rv)
|
||||
|
||||
def expr_or_value_state(self):
|
||||
if self.peek(3) == "if ":
|
||||
|
|
@ -270,12 +348,12 @@ class Tokenizer(object):
|
|||
self.skip_whitespace()
|
||||
c = self.char()
|
||||
if c == eol:
|
||||
raise ParseError("EOL in expression")
|
||||
raise ParseError(self.filename, self.line_number, "EOL in expression")
|
||||
elif c in "'\"":
|
||||
self.consume()
|
||||
yield (token_types.string, decode(self.read_string(c)))
|
||||
yield (token_types.string, self.consume_string(c))
|
||||
elif c == "#":
|
||||
raise ParseError("Comment before end of expression")
|
||||
raise ParseError(self.filename, self.line_number, "Comment before end of expression")
|
||||
elif c == ":":
|
||||
yield (token_types.separator, c)
|
||||
self.consume()
|
||||
|
|
@ -315,7 +393,7 @@ class Tokenizer(object):
|
|||
self.consume()
|
||||
elif c == ".":
|
||||
if seen_dot:
|
||||
raise ParseError("Invalid number")
|
||||
raise ParseError(self.filename, self.line_number, "Invalid number")
|
||||
self.consume()
|
||||
seen_dot = True
|
||||
elif c in parens:
|
||||
|
|
@ -327,7 +405,7 @@ class Tokenizer(object):
|
|||
elif c == ":":
|
||||
break
|
||||
else:
|
||||
raise ParseError("Invalid character in number")
|
||||
raise ParseError(self.filename, self.line_number, "Invalid character in number")
|
||||
|
||||
self.state = self.expr_state
|
||||
yield (token_types.number, self.line[index_0:self.index])
|
||||
|
|
@ -353,6 +431,44 @@ class Tokenizer(object):
|
|||
self.state = self.expr_state
|
||||
yield (token_types.ident, self.line[index_0:self.index])
|
||||
|
||||
def consume_escape(self):
|
||||
assert self.char() == "\\"
|
||||
self.consume()
|
||||
c = self.char()
|
||||
self.consume()
|
||||
if c == "x":
|
||||
return self.decode_escape(2)
|
||||
elif c == "u":
|
||||
return self.decode_escape(4)
|
||||
elif c == "U":
|
||||
return self.decode_escape(6)
|
||||
elif c in ["a", "b", "f", "n", "r", "t", "v"]:
|
||||
return eval("'\%s'" % c)
|
||||
elif c is eol:
|
||||
raise ParseError(self.filename, self.line_number, "EOL in escape")
|
||||
else:
|
||||
return c
|
||||
|
||||
def decode_escape(self, length):
|
||||
value = 0
|
||||
for i in xrange(length):
|
||||
c = self.char()
|
||||
value *= 16
|
||||
value += self.escape_value(c)
|
||||
self.consume()
|
||||
|
||||
return unichr(value).encode("utf8")
|
||||
|
||||
def escape_value(self, c):
|
||||
if '0' <= c <= '9':
|
||||
return ord(c) - ord('0')
|
||||
elif 'a' <= c <= 'f':
|
||||
return ord(c) - ord('a') + 10
|
||||
elif 'A' <= c <= 'F':
|
||||
return ord(c) - ord('A') + 10
|
||||
else:
|
||||
raise ParseError(self.filename, self.line_number, "Invalid character escape")
|
||||
|
||||
|
||||
class Parser(object):
|
||||
def __init__(self):
|
||||
|
|
@ -417,7 +533,10 @@ class Parser(object):
|
|||
self.expect(token_types.group_end)
|
||||
|
||||
def value_block(self):
|
||||
if self.token[0] == token_types.string:
|
||||
if self.token[0] == token_types.list_start:
|
||||
self.consume()
|
||||
self.list_value()
|
||||
elif self.token[0] == token_types.string:
|
||||
self.value()
|
||||
elif self.token[0] == token_types.group_start:
|
||||
self.consume()
|
||||
|
|
@ -428,6 +547,13 @@ class Parser(object):
|
|||
else:
|
||||
raise ParseError
|
||||
|
||||
def list_value(self):
|
||||
self.tree.append(ListNode())
|
||||
while self.token[0] == token_types.string:
|
||||
self.value()
|
||||
self.expect(token_types.list_end)
|
||||
self.tree.pop()
|
||||
|
||||
def expression_values(self):
|
||||
while self.token == (token_types.ident, "if"):
|
||||
self.consume()
|
||||
|
|
@ -446,7 +572,7 @@ class Parser(object):
|
|||
self.tree.pop()
|
||||
|
||||
def expr_start(self):
|
||||
self.expr_builder = ExpressionBuilder()
|
||||
self.expr_builder = ExpressionBuilder(self.tokenizer)
|
||||
self.expr_builders.append(self.expr_builder)
|
||||
self.expr()
|
||||
expression = self.expr_builder.finish()
|
||||
|
|
@ -486,14 +612,14 @@ class Parser(object):
|
|||
self.expr_builder.push_operator(UnaryOperatorNode(self.token[1]))
|
||||
self.consume()
|
||||
else:
|
||||
raise ParseError()
|
||||
raise ParseError(self.filename, self.tokenizer.line_number, "Expected unary operator")
|
||||
|
||||
def expr_bin_op(self):
|
||||
if self.token[1] in binary_operators:
|
||||
self.expr_builder.push_operator(BinaryOperatorNode(self.token[1]))
|
||||
self.consume()
|
||||
else:
|
||||
raise ParseError()
|
||||
raise ParseError(self.filename, self.tokenizer.line_number, "Expected binary operator")
|
||||
|
||||
def expr_value(self):
|
||||
node_type = {token_types.string: StringNode,
|
||||
|
|
@ -528,9 +654,10 @@ class Treebuilder(object):
|
|||
|
||||
|
||||
class ExpressionBuilder(object):
|
||||
def __init__(self):
|
||||
def __init__(self, tokenizer):
|
||||
self.operands = []
|
||||
self.operators = [None]
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def finish(self):
|
||||
while self.operators[-1] is not None:
|
||||
|
|
@ -546,7 +673,8 @@ class ExpressionBuilder(object):
|
|||
while self.operators[-1] is not None:
|
||||
self.pop_operator()
|
||||
if not self.operators:
|
||||
raise ParseError("Unbalanced parens")
|
||||
raise ParseError(self.tokenizer.filename, self.tokenizer.line,
|
||||
"Unbalanced parens")
|
||||
|
||||
assert self.operators.pop() is None
|
||||
|
||||
|
|
|
|||
|
|
@ -2,15 +2,25 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from node import NodeVisitor, ValueNode, BinaryExpressionNode
|
||||
from node import NodeVisitor, ValueNode, ListNode, BinaryExpressionNode
|
||||
from parser import precedence
|
||||
|
||||
named_escapes = set(["\a", "\b", "\f", "\n", "\r", "\t", "\v"])
|
||||
|
||||
def escape(string, extras=""):
|
||||
rv = string.encode("utf8").encode("string_escape")
|
||||
for extra in extras:
|
||||
rv = rv.replace(extra, "\\" + extra)
|
||||
return rv
|
||||
rv = ""
|
||||
for c in string:
|
||||
if c in named_escapes:
|
||||
rv += c.encode("unicode_escape")
|
||||
elif c == "\\":
|
||||
rv += "\\\\"
|
||||
elif c < '\x20':
|
||||
rv += "\\x%02x" % ord(c)
|
||||
elif c in extras:
|
||||
rv += "\\" + c
|
||||
else:
|
||||
rv += c
|
||||
return rv.encode("utf8")
|
||||
|
||||
|
||||
class ManifestSerializer(NodeVisitor):
|
||||
|
|
@ -42,34 +52,48 @@ class ManifestSerializer(NodeVisitor):
|
|||
return rv
|
||||
|
||||
def visit_KeyValueNode(self, node):
|
||||
rv = [node.data + ":"]
|
||||
rv = [escape(node.data, ":") + ":"]
|
||||
indent = " " * self.indent
|
||||
|
||||
if len(node.children) == 1 and isinstance(node.children[0], ValueNode):
|
||||
rv[0] += " %s" % escape(self.visit(node.children[0])[0])
|
||||
if len(node.children) == 1 and isinstance(node.children[0], (ValueNode, ListNode)):
|
||||
rv[0] += " %s" % self.visit(node.children[0])[0]
|
||||
else:
|
||||
for child in node.children:
|
||||
rv.append(indent + self.visit(child)[0])
|
||||
|
||||
return rv
|
||||
|
||||
def visit_ListNode(self, node):
|
||||
rv = ["["]
|
||||
rv.extend(", ".join(self.visit(child)[0] for child in node.children))
|
||||
rv.append("]")
|
||||
return ["".join(rv)]
|
||||
|
||||
def visit_ValueNode(self, node):
|
||||
return [escape(node.data)]
|
||||
if "#" in node.data or (isinstance(node.parent, ListNode) and
|
||||
("," in node.data or "]" in node.data)):
|
||||
if "\"" in node.data:
|
||||
quote = "'"
|
||||
else:
|
||||
quote = "\""
|
||||
else:
|
||||
quote = ""
|
||||
return [quote + escape(node.data, extras=quote) + quote]
|
||||
|
||||
def visit_ConditionalNode(self, node):
|
||||
return ["if %s: %s" % tuple(self.visit(item)[0] for item in node.children)]
|
||||
|
||||
def visit_StringNode(self, node):
|
||||
rv = ["\"%s\"" % node.data]
|
||||
rv = ["\"%s\"" % escape(node.data, extras="\"")]
|
||||
for child in node.children:
|
||||
rv[0] += self.visit(child)[0]
|
||||
return rv
|
||||
|
||||
def visit_NumberNode(self, node):
|
||||
return [node.data]
|
||||
return [str(node.data)]
|
||||
|
||||
def visit_VariableNode(self, node):
|
||||
rv = node.data
|
||||
rv = escape(node.data)
|
||||
for child in node.children:
|
||||
rv += self.visit(child)
|
||||
return [rv]
|
||||
|
|
@ -100,10 +124,10 @@ class ManifestSerializer(NodeVisitor):
|
|||
return [" ".join(children)]
|
||||
|
||||
def visit_UnaryOperatorNode(self, node):
|
||||
return [node.data]
|
||||
return [str(node.data)]
|
||||
|
||||
def visit_BinaryOperatorNode(self, node):
|
||||
return [node.data]
|
||||
return [str(node.data)]
|
||||
|
||||
|
||||
def serialize(tree, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -15,12 +15,12 @@ class TokenizerTest(unittest.TestCase):
|
|||
self.parser = parser.Parser()
|
||||
|
||||
def serialize(self, input_str):
|
||||
return self.serializer.serialize(self.parser.parse(StringIO(input_str)))
|
||||
return self.serializer.serialize(self.parser.parse(input_str))
|
||||
|
||||
def compare(self, input_str, expected=None):
|
||||
if expected is None:
|
||||
expected = input_str
|
||||
|
||||
expected = expected.encode("utf8")
|
||||
actual = self.serialize(input_str)
|
||||
self.assertEquals(actual, expected)
|
||||
|
||||
|
|
@ -114,11 +114,98 @@ class TokenizerTest(unittest.TestCase):
|
|||
|
||||
[Heading 2]
|
||||
other_key: other_value
|
||||
"""
|
||||
)
|
||||
""")
|
||||
|
||||
def test_11(self):
|
||||
self.compare("""key:
|
||||
if not a and b and c and d: true
|
||||
"""
|
||||
)
|
||||
""")
|
||||
|
||||
def test_12(self):
|
||||
self.compare("""[Heading 1]
|
||||
key: [a:1, b:2]
|
||||
""")
|
||||
|
||||
def test_13(self):
|
||||
self.compare("""key: [a:1, "b:#"]
|
||||
""")
|
||||
|
||||
def test_14(self):
|
||||
self.compare("""key: [","]
|
||||
""")
|
||||
|
||||
def test_15(self):
|
||||
self.compare("""key: ,
|
||||
""")
|
||||
|
||||
def test_16(self):
|
||||
self.compare("""key: ["]", b]
|
||||
""")
|
||||
|
||||
def test_17(self):
|
||||
self.compare("""key: ]
|
||||
""")
|
||||
|
||||
def test_18(self):
|
||||
self.compare("""key: \]
|
||||
""", """key: ]
|
||||
""")
|
||||
|
||||
def test_escape_0(self):
|
||||
self.compare(r"""k\t\:y: \a\b\f\n\r\t\v""",
|
||||
r"""k\t\:y: \x07\x08\x0c\n\r\t\x0b
|
||||
""")
|
||||
|
||||
def test_escape_1(self):
|
||||
self.compare(r"""k\x00: \x12A\x45""",
|
||||
r"""k\x00: \x12AE
|
||||
""")
|
||||
|
||||
def test_escape_2(self):
|
||||
self.compare(r"""k\u0045y: \u1234A\uABc6""",
|
||||
u"""kEy: \u1234A\uabc6
|
||||
""")
|
||||
|
||||
def test_escape_3(self):
|
||||
self.compare(r"""k\u0045y: \u1234A\uABc6""",
|
||||
u"""kEy: \u1234A\uabc6
|
||||
""")
|
||||
|
||||
def test_escape_4(self):
|
||||
self.compare(r"""key: '\u1234A\uABc6'""",
|
||||
u"""key: \u1234A\uabc6
|
||||
""")
|
||||
|
||||
def test_escape_5(self):
|
||||
self.compare(r"""key: [\u1234A\uABc6]""",
|
||||
u"""key: [\u1234A\uabc6]
|
||||
""")
|
||||
|
||||
def test_escape_6(self):
|
||||
self.compare(r"""key: [\u1234A\uABc6\,]""",
|
||||
u"""key: ["\u1234A\uabc6,"]
|
||||
""")
|
||||
|
||||
def test_escape_7(self):
|
||||
self.compare(r"""key: [\,\]\#]""",
|
||||
r"""key: [",]#"]
|
||||
""")
|
||||
|
||||
def test_escape_8(self):
|
||||
self.compare(r"""key: \#""",
|
||||
r"""key: "#"
|
||||
""")
|
||||
|
||||
def test_escape_9(self):
|
||||
self.compare(r"""key: \U10FFFFabc""",
|
||||
u"""key: \U0010FFFFabc
|
||||
""")
|
||||
|
||||
def test_escape_10(self):
|
||||
self.compare(r"""key: \u10FFab""",
|
||||
u"""key: \u10FFab
|
||||
""")
|
||||
|
||||
def test_escape_11(self):
|
||||
self.compare(r"""key: \\ab
|
||||
""")
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ class TokenizerTest(unittest.TestCase):
|
|||
(token_types.paren, "]")])
|
||||
|
||||
def test_heading_6(self):
|
||||
self.compare("""[Heading \\ttext]""",
|
||||
self.compare(r"""[Heading \ttext]""",
|
||||
[(token_types.paren, "["),
|
||||
(token_types.string, "Heading \ttext"),
|
||||
(token_types.paren, "]")])
|
||||
|
|
@ -142,6 +142,76 @@ class TokenizerTest(unittest.TestCase):
|
|||
with self.assertRaises(parser.ParseError):
|
||||
self.tokenize("""key: 'value' abc""")
|
||||
|
||||
def test_key_14(self):
|
||||
self.compare(r"""key: \\nb""",
|
||||
[(token_types.string, "key"),
|
||||
(token_types.separator, ":"),
|
||||
(token_types.string, r"\nb")])
|
||||
|
||||
def test_list_0(self):
|
||||
self.compare(
|
||||
"""
|
||||
key: []""",
|
||||
[(token_types.string, "key"),
|
||||
(token_types.separator, ":"),
|
||||
(token_types.list_start, "["),
|
||||
(token_types.list_end, "]")])
|
||||
|
||||
def test_list_1(self):
|
||||
self.compare(
|
||||
"""
|
||||
key: [a, "b"]""",
|
||||
[(token_types.string, "key"),
|
||||
(token_types.separator, ":"),
|
||||
(token_types.list_start, "["),
|
||||
(token_types.string, "a"),
|
||||
(token_types.string, "b"),
|
||||
(token_types.list_end, "]")])
|
||||
|
||||
def test_list_2(self):
|
||||
self.compare(
|
||||
"""
|
||||
key: [a,
|
||||
b]""",
|
||||
[(token_types.string, "key"),
|
||||
(token_types.separator, ":"),
|
||||
(token_types.list_start, "["),
|
||||
(token_types.string, "a"),
|
||||
(token_types.string, "b"),
|
||||
(token_types.list_end, "]")])
|
||||
|
||||
def test_list_3(self):
|
||||
self.compare(
|
||||
"""
|
||||
key: [a, #b]
|
||||
c]""",
|
||||
[(token_types.string, "key"),
|
||||
(token_types.separator, ":"),
|
||||
(token_types.list_start, "["),
|
||||
(token_types.string, "a"),
|
||||
(token_types.string, "c"),
|
||||
(token_types.list_end, "]")])
|
||||
|
||||
def test_list_4(self):
|
||||
with self.assertRaises(parser.ParseError):
|
||||
self.tokenize("""key: [a #b]
|
||||
c]""")
|
||||
|
||||
def test_list_5(self):
|
||||
with self.assertRaises(parser.ParseError):
|
||||
self.tokenize("""key: [a \\
|
||||
c]""")
|
||||
|
||||
def test_list_6(self):
|
||||
self.compare(
|
||||
"""key: [a , b]""",
|
||||
[(token_types.string, "key"),
|
||||
(token_types.separator, ":"),
|
||||
(token_types.list_start, "["),
|
||||
(token_types.string, "a"),
|
||||
(token_types.string, "b"),
|
||||
(token_types.list_end, "]")])
|
||||
|
||||
def test_expr_0(self):
|
||||
self.compare(
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -134,6 +134,7 @@ def run_tests(config, test_paths, product, **kwargs):
|
|||
with env.TestEnvironment(test_paths,
|
||||
ssl_env,
|
||||
kwargs["pause_after_test"],
|
||||
kwargs["debug_info"],
|
||||
env_options) as test_environment:
|
||||
try:
|
||||
test_environment.ensure_started()
|
||||
|
|
@ -180,7 +181,7 @@ def run_tests(config, test_paths, product, **kwargs):
|
|||
executor_kwargs,
|
||||
kwargs["pause_after_test"],
|
||||
kwargs["pause_on_unexpected"],
|
||||
kwargs["debug_args"]) as manager_group:
|
||||
kwargs["debug_info"]) as manager_group:
|
||||
try:
|
||||
manager_group.run(test_type, test_loader.tests)
|
||||
except KeyboardInterrupt:
|
||||
|
|
|
|||
|
|
@ -90,7 +90,11 @@ class Test(object):
|
|||
self._expected_metadata = expected_metadata
|
||||
self.timeout = timeout
|
||||
self.path = path
|
||||
self.protocol = protocol
|
||||
if expected_metadata:
|
||||
prefs = expected_metadata.prefs()
|
||||
else:
|
||||
prefs = []
|
||||
self.environment = {"protocol": protocol, "prefs": prefs}
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.id == other.id
|
||||
|
|
@ -102,7 +106,7 @@ class Test(object):
|
|||
expected_metadata,
|
||||
timeout=timeout,
|
||||
path=manifest_item.path,
|
||||
protocol="https" if manifest_item.https else "http")
|
||||
protocol="https" if hasattr(manifest_item, "https") and manifest_item.https else "http")
|
||||
|
||||
|
||||
@property
|
||||
|
|
@ -165,14 +169,12 @@ class ReftestTest(Test):
|
|||
result_cls = ReftestResult
|
||||
|
||||
def __init__(self, url, expected, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
|
||||
self.url = url
|
||||
Test.__init__(self, url, expected, timeout, path, protocol)
|
||||
|
||||
for _, ref_type in references:
|
||||
if ref_type not in ("==", "!="):
|
||||
raise ValueError
|
||||
self._expected_metadata = expected
|
||||
self.timeout = timeout
|
||||
self.path = path
|
||||
self.protocol = protocol
|
||||
|
||||
self.references = references
|
||||
|
||||
@classmethod
|
||||
|
|
@ -196,7 +198,7 @@ class ReftestTest(Test):
|
|||
[],
|
||||
timeout=timeout,
|
||||
path=manifest_test.path,
|
||||
protocol="https" if manifest_test.https else "http")
|
||||
protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http")
|
||||
|
||||
nodes[url] = node
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue