diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 483f9c7c2c9..c98311ad9c5 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -427,10 +427,10 @@ class MachCommands(CommandBase): def _test_wpt(self, android=False, **kwargs): self.set_run_env(android) - hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts') - os.environ["HOST_FILE"] = hosts_file_path - run_file = path.abspath(path.join(self.context.topdir, "tests", "wpt", "run.py")) - return self.wptrunner(run_file, **kwargs) + + sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt')) + import servowpt + return servowpt.run_tests(**kwargs) # Helper to ensure all specified paths are handled, otherwise dispatch to appropriate test suite. def run_test_list_or_dispatch(self, requested_paths, correct_suite, correct_function, **kwargs): @@ -448,42 +448,6 @@ class MachCommands(CommandBase): # Dispatch each test to the correct suite via test() Registrar.dispatch("test", context=self.context, params=requested_paths) - # Helper for test_css and test_wpt: - def wptrunner(self, run_file, **kwargs): - # By default, Rayon selects the number of worker threads - # based on the available CPU count. This doesn't work very - # well when running tests on CI, since we run so many - # Servo processes in parallel. The result is a lot of - # extra timeouts. Instead, force Rayon to assume we are - # running on a 2 CPU environment. - os.environ['RAYON_RS_NUM_CPUS'] = "2" - - os.environ["RUST_BACKTRACE"] = "1" - kwargs["debug"] = not kwargs["release"] - if kwargs.pop("rr_chaos"): - kwargs["debugger"] = "rr" - kwargs["debugger_args"] = "record --chaos" - kwargs["repeat_until_unexpected"] = True - # TODO: Delete rr traces from green test runs? - prefs = kwargs.pop("prefs") - if prefs: - binary_args = [] - for pref in prefs: - binary_args.append("--pref=" + pref) - kwargs["binary_args"] = binary_args - - if not kwargs.get('no_default_test_types'): - test_types = { - "servo": ["testharness", "reftest", "wdspec"], - "servodriver": ["testharness", "reftest"], - } - product = kwargs.get("product") or "servo" - kwargs["test_types"] = test_types[product] - - run_globals = {"__file__": run_file} - exec(compile(open(run_file).read(), run_file, 'exec'), run_globals) - return run_globals["run_tests"](**kwargs) - @Command('update-manifest', description='Run test-wpt --manifest-update SKIP_TESTS to regenerate MANIFEST.json', category='testing', @@ -509,16 +473,14 @@ class MachCommands(CommandBase): category='testing', parser=updatecommandline.create_parser()) def update_wpt(self, **kwargs): - run_file = path.abspath(path.join("tests", "wpt", "update.py")) patch = kwargs.get("patch", False) - if not patch and kwargs["sync"]: print("Are you sure you don't want a patch?") return 1 - run_globals = {"__file__": run_file} - exec(compile(open(run_file).read(), run_file, 'exec'), run_globals) - return run_globals["update_tests"](**kwargs) + sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt')) + import servowpt + return servowpt.update_tests(**kwargs) @Command('filter-intermittents', description='Given a WPT error summary file, filter out intermittents and other cruft.', diff --git a/tests/wpt/README.md b/tests/wpt/README.md index 9b1328ceafa..7966faacb0e 100644 --- a/tests/wpt/README.md +++ b/tests/wpt/README.md @@ -9,7 +9,7 @@ In particular, this folder contains: * `config.ini`: some configuration for the web-platform-tests. * `include.ini`: the subset of web-platform-tests we currently run. -* `run.py`: run the web-platform-tests in Servo. +* `servowpt.py`: run the web-platform-tests in Servo. * `web-platform-tests`: copy of the web-platform-tests. * `metadata`: expected failures for the web-platform-tests we run. * `mozilla`: web-platform-tests that cannot be upstreamed. @@ -81,7 +81,7 @@ for more details). Running the tests without mach ------------------------------ -When avoiding `mach` for some reason, one can run `run.py` +When avoiding `mach` for some reason, one can run `servowpt.py` directly. However, this requires that all the dependencies for `wptrunner` are available in the current python environment. diff --git a/tests/wpt/run.py b/tests/wpt/run.py deleted file mode 100644 index 84048d4ea0e..00000000000 --- a/tests/wpt/run.py +++ /dev/null @@ -1,116 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import multiprocessing -import os -from os import path -import sys -import mozlog -import grouping_formatter - -here = os.path.split(__file__)[0] -servo_root = os.path.abspath(os.path.join(here, "..", "..")) - - -def wpt_path(*args): - return os.path.join(here, *args) - - -def servo_path(*args): - return os.path.join(servo_root, *args) - - -paths = {"include_manifest": wpt_path("include.ini"), - "config": wpt_path("config.ini"), - "ca-cert-path": wpt_path("web-platform-tests/tools/certs/cacert.pem"), - "host-key-path": wpt_path("web-platform-tests/tools/certs/web-platform.test.key"), - "host-cert-path": wpt_path("web-platform-tests/tools/certs/web-platform.test.pem")} -# Imports -sys.path.append(wpt_path("web-platform-tests", "tools")) -import localpaths # noqa: F401,E402 -from wptrunner import wptrunner, wptcommandline # noqa: E402 - - -def run_tests(**kwargs): - set_defaults(kwargs) - - mozlog.commandline.log_formatters["servo"] = \ - (grouping_formatter.ServoFormatter, "Servo's grouping output formatter") - mozlog.commandline.log_formatters["servojson"] = \ - (grouping_formatter.ServoJsonFormatter, "Servo's JSON logger of unexpected results") - - use_mach_logging = False - if len(kwargs["test_list"]) == 1: - file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower() - if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]: - use_mach_logging = True - - if use_mach_logging: - wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) - else: - wptrunner.setup_logging(kwargs, {"servo": sys.stdout}) - - success = wptrunner.run_tests(**kwargs) - return 0 if success else 1 - - -def set_defaults(kwargs): - if kwargs["product"] is None: - kwargs["product"] = "servo" - - if kwargs["config"] is None and "config" in paths: - kwargs["config"] = paths["config"] - - if kwargs["include_manifest"] is None and "include_manifest" in paths: - kwargs["include_manifest"] = paths["include_manifest"] - - if kwargs["manifest_update"] is None: - kwargs["manifest_update"] = False - - if kwargs["binary"] is None: - bin_dir = "release" if kwargs["release"] else "debug" - bin_name = "servo" - if sys.platform == "win32": - bin_name += ".exe" - if "CARGO_TARGET_DIR" in os.environ: - bin_path = path.join(os.environ["CARGO_TARGET_DIR"], bin_dir, bin_name) - else: - bin_path = servo_path("target", bin_dir, bin_name) - - kwargs["binary"] = bin_path - kwargs["webdriver_binary"] = bin_path - - if kwargs["processes"] is None: - kwargs["processes"] = multiprocessing.cpu_count() - - if kwargs["ca_cert_path"] is None: - kwargs["ca_cert_path"] = paths["ca-cert-path"] - - if kwargs["host_key_path"] is None: - kwargs["host_key_path"] = paths["host-key-path"] - - if kwargs["host_cert_path"] is None: - kwargs["host_cert_path"] = paths["host-cert-path"] - - if kwargs["ssl_type"] is None: - kwargs["ssl_type"] = "pregenerated" - - kwargs["user_stylesheets"].append(servo_path("resources", "ahem.css")) - - wptcommandline.check_args(kwargs) - - if kwargs.pop("layout_2020"): - kwargs["test_paths"]["/"]["metadata_path"] = wpt_path("metadata-layout-2020") - kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = wpt_path("mozilla/meta-layout-2020") - kwargs["include_manifest"] = wpt_path("include-layout-2020.ini") - - -def main(): - parser = wptcommandline.create_parser() - kwargs = vars(parser.parse_args()) - return run_tests(**kwargs) - - -if __name__ == "__main__": - sys.exit(0 if main() else 1) diff --git a/tests/wpt/servowpt.py b/tests/wpt/servowpt.py new file mode 100644 index 00000000000..6364d1645e0 --- /dev/null +++ b/tests/wpt/servowpt.py @@ -0,0 +1,162 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +import sys + +import grouping_formatter +import mozlog +import multiprocessing + +SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) +SERVO_ROOT = os.path.abspath(os.path.join(SCRIPT_PATH, "..", "..")) +WPT_TOOLS_PATH = os.path.join(SCRIPT_PATH, "web-platform-tests", "tools") +CERTS_PATH = os.path.join(WPT_TOOLS_PATH, "certs") + +sys.path.insert(0, WPT_TOOLS_PATH) +import update # noqa: F401,E402 +import localpaths # noqa: F401,E402 + + +def determine_build_type(kwargs: dict, target_dir: str): + if kwargs["release"]: + return "release" + elif kwargs["debug"]: + return "debug" + elif os.path.exists(os.path.join(target_dir, "debug")): + return "debug" + elif os.path.exists(os.path.join(target_dir, "release")): + return "release" + return "debug" + + +def set_if_none(args: dict, key: str, value): + if key not in args or args[key] is None: + args[key] = value + + +def update_args_for_layout_2020(kwargs: dict): + if kwargs.pop("layout_2020"): + kwargs["test_paths"]["/"]["metadata_path"] = os.path.join( + SCRIPT_PATH, "metadata-layout-2020" + ) + kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = os.path.join( + SCRIPT_PATH, "mozilla", "meta-layout-2020" + ) + kwargs["include_manifest"] = os.path.join( + SCRIPT_PATH, "include-layout-2020.ini" + ) + + +def run_tests(**kwargs): + from wptrunner import wptrunner + from wptrunner import wptcommandline + + # By default, Rayon selects the number of worker threads based on the + # available CPU count. This doesn't work very well when running tests on CI, + # since we run so many Servo processes in parallel. The result is a lot of + # extra timeouts. Instead, force Rayon to assume we are running on a 2 CPU + # environment. + os.environ["RAYON_RS_NUM_CPUS"] = "2" + os.environ["RUST_BACKTRACE"] = "1" + os.environ["HOST_FILE"] = os.path.join(SERVO_ROOT, "tests", "wpt", "hosts") + + set_if_none(kwargs, "product", "servo") + set_if_none(kwargs, "config", os.path.join(SCRIPT_PATH, "config.ini")) + set_if_none(kwargs, "include_manifest", os.path.join(SCRIPT_PATH, "include.ini")) + set_if_none(kwargs, "manifest_update", False) + set_if_none(kwargs, "processes", multiprocessing.cpu_count()) + + set_if_none(kwargs, "ca_cert_path", os.path.join(CERTS_PATH, "cacert.pem")) + set_if_none( + kwargs, "host_key_path", os.path.join(CERTS_PATH, "web-platform.test.key") + ) + set_if_none( + kwargs, "host_cert_path", os.path.join(CERTS_PATH, "web-platform.test.pem") + ) + + kwargs["user_stylesheets"].append(os.path.join(SERVO_ROOT, "resources", "ahem.css")) + + if "CARGO_TARGET_DIR" in os.environ: + target_dir = os.path.join(os.environ["CARGO_TARGET_DIR"]) + else: + target_dir = os.path.join(SERVO_ROOT, "target") + default_binary_path = os.path.join( + target_dir, determine_build_type(kwargs, target_dir), "servo" + ) + if sys.platform == "win32": + target_dir += ".exe" + + set_if_none(kwargs, "binary", default_binary_path) + set_if_none(kwargs, "webdriver_binary", default_binary_path) + + if kwargs.pop("rr_chaos"): + kwargs["debugger"] = "rr" + kwargs["debugger_args"] = "record --chaos" + kwargs["repeat_until_unexpected"] = True + # TODO: Delete rr traces from green test runs? + + prefs = kwargs.pop("prefs") + if prefs: + kwargs["binary_args"] = ["--pref=" + pref for pref in prefs] + + if not kwargs.get("no_default_test_types"): + test_types = { + "servo": ["testharness", "reftest", "wdspec"], + "servodriver": ["testharness", "reftest"], + } + product = kwargs.get("product") or "servo" + kwargs["test_types"] = test_types[product] + + wptcommandline.check_args(kwargs) + update_args_for_layout_2020(kwargs) + + mozlog.commandline.log_formatters["servo"] = ( + grouping_formatter.ServoFormatter, + "Servo's grouping output formatter", + ) + mozlog.commandline.log_formatters["servojson"] = ( + grouping_formatter.ServoJsonFormatter, + "Servo's JSON logger of unexpected results", + ) + + use_mach_logging = False + if len(kwargs["test_list"]) == 1: + file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower() + if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]: + use_mach_logging = True + + if use_mach_logging: + wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) + else: + wptrunner.setup_logging(kwargs, {"servo": sys.stdout}) + + success = wptrunner.run_tests(**kwargs) + return 0 if success else 1 + + +def update_tests(**kwargs): + from update import updatecommandline + + set_if_none(kwargs, "product", "servo") + set_if_none(kwargs, "config", os.path.join(SCRIPT_PATH, "config.ini")) + kwargs["store_state"] = False + + updatecommandline.check_args(kwargs) + + logger = update.setup_logging(kwargs, {"mach": sys.stdout}) + return_value = update.run_update(logger, **kwargs) + return 1 if return_value is update.exit_unclean else 0 + + +def main(): + from wptrunner import wptcommandline + + parser = wptcommandline.create_parser() + kwargs = vars(parser.parse_args()) + return run_tests(**kwargs) + + +if __name__ == "__main__": + sys.exit(0 if main() else 1) diff --git a/tests/wpt/update.py b/tests/wpt/update.py deleted file mode 100644 index 8e7100a13d7..00000000000 --- a/tests/wpt/update.py +++ /dev/null @@ -1,48 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import os -import sys -from wptrunner import wptcommandline -from update import updatecommandline - -here = os.path.split(__file__)[0] - - -def wpt_path(*args): - return os.path.join(here, *args) - - -def update_tests(**kwargs): - import update - - set_defaults(kwargs) - logger = update.setup_logging(kwargs, {"mach": sys.stdout}) - - rv = update.run_update(logger, **kwargs) - return 1 if rv is update.exit_unclean else 0 - - -def set_defaults(kwargs): - if kwargs["product"] is None: - kwargs["product"] = "servo" - if kwargs["config"] is None: - kwargs["config"] = wpt_path('config.ini') - kwargs["store_state"] = False - updatecommandline.check_args(kwargs) - - if kwargs.pop("layout_2020"): - kwargs["test_paths"]["/"]["metadata_path"] = wpt_path("metadata-layout-2020") - kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = wpt_path("mozilla/meta-layout-2020") - kwargs["include_manifest"] = wpt_path("include-layout-2020.ini") - - -def main(): - parser = wptcommandline.create_parser() - kwargs = vars(parser.parse_args()) - return update_tests(**kwargs) - - -if __name__ == "__main__": - sys.exit(0 if main() else 1)