mirror of
https://github.com/servo/servo.git
synced 2025-06-06 16:45:39 +00:00
Simplify Python code for running WPT tests
Combine `run.py` and `update.py` into `servowpt.py` in order to allow them to share code. Import them directly into the mach script to avoid having to call `compile` and `exec` on the code. This makes it clearer how they are executed. In addition, move all of the setup into `setupwpt.py` to avoid differences between tests executed via mach and not. Finally, be more ambitious when detecting the build to use. If none was specified, try to use the one that exists between "release" and "debug."
This commit is contained in:
Martin Robinson
parent
633f14df11
commit
42c3d05d2b
5 changed files with 171 additions and 211 deletions
|
|
@ -427,10 +427,10 @@ class MachCommands(CommandBase):
|
|||
|
||||
def _test_wpt(self, android=False, **kwargs):
|
||||
self.set_run_env(android)
|
||||
hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts')
|
||||
os.environ["HOST_FILE"] = hosts_file_path
|
||||
run_file = path.abspath(path.join(self.context.topdir, "tests", "wpt", "run.py"))
|
||||
return self.wptrunner(run_file, **kwargs)
|
||||
|
||||
sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt'))
|
||||
import servowpt
|
||||
return servowpt.run_tests(**kwargs)
|
||||
|
||||
# Helper to ensure all specified paths are handled, otherwise dispatch to appropriate test suite.
|
||||
def run_test_list_or_dispatch(self, requested_paths, correct_suite, correct_function, **kwargs):
|
||||
|
|
@ -448,42 +448,6 @@ class MachCommands(CommandBase):
|
|||
# Dispatch each test to the correct suite via test()
|
||||
Registrar.dispatch("test", context=self.context, params=requested_paths)
|
||||
|
||||
# Helper for test_css and test_wpt:
|
||||
def wptrunner(self, run_file, **kwargs):
|
||||
# By default, Rayon selects the number of worker threads
|
||||
# based on the available CPU count. This doesn't work very
|
||||
# well when running tests on CI, since we run so many
|
||||
# Servo processes in parallel. The result is a lot of
|
||||
# extra timeouts. Instead, force Rayon to assume we are
|
||||
# running on a 2 CPU environment.
|
||||
os.environ['RAYON_RS_NUM_CPUS'] = "2"
|
||||
|
||||
os.environ["RUST_BACKTRACE"] = "1"
|
||||
kwargs["debug"] = not kwargs["release"]
|
||||
if kwargs.pop("rr_chaos"):
|
||||
kwargs["debugger"] = "rr"
|
||||
kwargs["debugger_args"] = "record --chaos"
|
||||
kwargs["repeat_until_unexpected"] = True
|
||||
# TODO: Delete rr traces from green test runs?
|
||||
prefs = kwargs.pop("prefs")
|
||||
if prefs:
|
||||
binary_args = []
|
||||
for pref in prefs:
|
||||
binary_args.append("--pref=" + pref)
|
||||
kwargs["binary_args"] = binary_args
|
||||
|
||||
if not kwargs.get('no_default_test_types'):
|
||||
test_types = {
|
||||
"servo": ["testharness", "reftest", "wdspec"],
|
||||
"servodriver": ["testharness", "reftest"],
|
||||
}
|
||||
product = kwargs.get("product") or "servo"
|
||||
kwargs["test_types"] = test_types[product]
|
||||
|
||||
run_globals = {"__file__": run_file}
|
||||
exec(compile(open(run_file).read(), run_file, 'exec'), run_globals)
|
||||
return run_globals["run_tests"](**kwargs)
|
||||
|
||||
@Command('update-manifest',
|
||||
description='Run test-wpt --manifest-update SKIP_TESTS to regenerate MANIFEST.json',
|
||||
category='testing',
|
||||
|
|
@ -509,16 +473,14 @@ class MachCommands(CommandBase):
|
|||
category='testing',
|
||||
parser=updatecommandline.create_parser())
|
||||
def update_wpt(self, **kwargs):
|
||||
run_file = path.abspath(path.join("tests", "wpt", "update.py"))
|
||||
patch = kwargs.get("patch", False)
|
||||
|
||||
if not patch and kwargs["sync"]:
|
||||
print("Are you sure you don't want a patch?")
|
||||
return 1
|
||||
|
||||
run_globals = {"__file__": run_file}
|
||||
exec(compile(open(run_file).read(), run_file, 'exec'), run_globals)
|
||||
return run_globals["update_tests"](**kwargs)
|
||||
sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt'))
|
||||
import servowpt
|
||||
return servowpt.update_tests(**kwargs)
|
||||
|
||||
@Command('filter-intermittents',
|
||||
description='Given a WPT error summary file, filter out intermittents and other cruft.',
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ In particular, this folder contains:
|
|||
|
||||
* `config.ini`: some configuration for the web-platform-tests.
|
||||
* `include.ini`: the subset of web-platform-tests we currently run.
|
||||
* `run.py`: run the web-platform-tests in Servo.
|
||||
* `servowpt.py`: run the web-platform-tests in Servo.
|
||||
* `web-platform-tests`: copy of the web-platform-tests.
|
||||
* `metadata`: expected failures for the web-platform-tests we run.
|
||||
* `mozilla`: web-platform-tests that cannot be upstreamed.
|
||||
|
|
@ -81,7 +81,7 @@ for more details).
|
|||
Running the tests without mach
|
||||
------------------------------
|
||||
|
||||
When avoiding `mach` for some reason, one can run `run.py`
|
||||
When avoiding `mach` for some reason, one can run `servowpt.py`
|
||||
directly. However, this requires that all the dependencies for
|
||||
`wptrunner` are available in the current python environment.
|
||||
|
||||
|
|
|
|||
116
tests/wpt/run.py
116
tests/wpt/run.py
|
|
@ -1,116 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
from os import path
|
||||
import sys
|
||||
import mozlog
|
||||
import grouping_formatter
|
||||
|
||||
here = os.path.split(__file__)[0]
|
||||
servo_root = os.path.abspath(os.path.join(here, "..", ".."))
|
||||
|
||||
|
||||
def wpt_path(*args):
|
||||
return os.path.join(here, *args)
|
||||
|
||||
|
||||
def servo_path(*args):
|
||||
return os.path.join(servo_root, *args)
|
||||
|
||||
|
||||
paths = {"include_manifest": wpt_path("include.ini"),
|
||||
"config": wpt_path("config.ini"),
|
||||
"ca-cert-path": wpt_path("web-platform-tests/tools/certs/cacert.pem"),
|
||||
"host-key-path": wpt_path("web-platform-tests/tools/certs/web-platform.test.key"),
|
||||
"host-cert-path": wpt_path("web-platform-tests/tools/certs/web-platform.test.pem")}
|
||||
# Imports
|
||||
sys.path.append(wpt_path("web-platform-tests", "tools"))
|
||||
import localpaths # noqa: F401,E402
|
||||
from wptrunner import wptrunner, wptcommandline # noqa: E402
|
||||
|
||||
|
||||
def run_tests(**kwargs):
|
||||
set_defaults(kwargs)
|
||||
|
||||
mozlog.commandline.log_formatters["servo"] = \
|
||||
(grouping_formatter.ServoFormatter, "Servo's grouping output formatter")
|
||||
mozlog.commandline.log_formatters["servojson"] = \
|
||||
(grouping_formatter.ServoJsonFormatter, "Servo's JSON logger of unexpected results")
|
||||
|
||||
use_mach_logging = False
|
||||
if len(kwargs["test_list"]) == 1:
|
||||
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
|
||||
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
|
||||
use_mach_logging = True
|
||||
|
||||
if use_mach_logging:
|
||||
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
else:
|
||||
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
|
||||
|
||||
success = wptrunner.run_tests(**kwargs)
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
def set_defaults(kwargs):
|
||||
if kwargs["product"] is None:
|
||||
kwargs["product"] = "servo"
|
||||
|
||||
if kwargs["config"] is None and "config" in paths:
|
||||
kwargs["config"] = paths["config"]
|
||||
|
||||
if kwargs["include_manifest"] is None and "include_manifest" in paths:
|
||||
kwargs["include_manifest"] = paths["include_manifest"]
|
||||
|
||||
if kwargs["manifest_update"] is None:
|
||||
kwargs["manifest_update"] = False
|
||||
|
||||
if kwargs["binary"] is None:
|
||||
bin_dir = "release" if kwargs["release"] else "debug"
|
||||
bin_name = "servo"
|
||||
if sys.platform == "win32":
|
||||
bin_name += ".exe"
|
||||
if "CARGO_TARGET_DIR" in os.environ:
|
||||
bin_path = path.join(os.environ["CARGO_TARGET_DIR"], bin_dir, bin_name)
|
||||
else:
|
||||
bin_path = servo_path("target", bin_dir, bin_name)
|
||||
|
||||
kwargs["binary"] = bin_path
|
||||
kwargs["webdriver_binary"] = bin_path
|
||||
|
||||
if kwargs["processes"] is None:
|
||||
kwargs["processes"] = multiprocessing.cpu_count()
|
||||
|
||||
if kwargs["ca_cert_path"] is None:
|
||||
kwargs["ca_cert_path"] = paths["ca-cert-path"]
|
||||
|
||||
if kwargs["host_key_path"] is None:
|
||||
kwargs["host_key_path"] = paths["host-key-path"]
|
||||
|
||||
if kwargs["host_cert_path"] is None:
|
||||
kwargs["host_cert_path"] = paths["host-cert-path"]
|
||||
|
||||
if kwargs["ssl_type"] is None:
|
||||
kwargs["ssl_type"] = "pregenerated"
|
||||
|
||||
kwargs["user_stylesheets"].append(servo_path("resources", "ahem.css"))
|
||||
|
||||
wptcommandline.check_args(kwargs)
|
||||
|
||||
if kwargs.pop("layout_2020"):
|
||||
kwargs["test_paths"]["/"]["metadata_path"] = wpt_path("metadata-layout-2020")
|
||||
kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = wpt_path("mozilla/meta-layout-2020")
|
||||
kwargs["include_manifest"] = wpt_path("include-layout-2020.ini")
|
||||
|
||||
|
||||
def main():
|
||||
parser = wptcommandline.create_parser()
|
||||
kwargs = vars(parser.parse_args())
|
||||
return run_tests(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if main() else 1)
|
||||
162
tests/wpt/servowpt.py
Normal file
162
tests/wpt/servowpt.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import grouping_formatter
|
||||
import mozlog
|
||||
import multiprocessing
|
||||
|
||||
SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
|
||||
SERVO_ROOT = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
|
||||
WPT_TOOLS_PATH = os.path.join(SCRIPT_PATH, "web-platform-tests", "tools")
|
||||
CERTS_PATH = os.path.join(WPT_TOOLS_PATH, "certs")
|
||||
|
||||
sys.path.insert(0, WPT_TOOLS_PATH)
|
||||
import update # noqa: F401,E402
|
||||
import localpaths # noqa: F401,E402
|
||||
|
||||
|
||||
def determine_build_type(kwargs: dict, target_dir: str):
|
||||
if kwargs["release"]:
|
||||
return "release"
|
||||
elif kwargs["debug"]:
|
||||
return "debug"
|
||||
elif os.path.exists(os.path.join(target_dir, "debug")):
|
||||
return "debug"
|
||||
elif os.path.exists(os.path.join(target_dir, "release")):
|
||||
return "release"
|
||||
return "debug"
|
||||
|
||||
|
||||
def set_if_none(args: dict, key: str, value):
|
||||
if key not in args or args[key] is None:
|
||||
args[key] = value
|
||||
|
||||
|
||||
def update_args_for_layout_2020(kwargs: dict):
|
||||
if kwargs.pop("layout_2020"):
|
||||
kwargs["test_paths"]["/"]["metadata_path"] = os.path.join(
|
||||
SCRIPT_PATH, "metadata-layout-2020"
|
||||
)
|
||||
kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = os.path.join(
|
||||
SCRIPT_PATH, "mozilla", "meta-layout-2020"
|
||||
)
|
||||
kwargs["include_manifest"] = os.path.join(
|
||||
SCRIPT_PATH, "include-layout-2020.ini"
|
||||
)
|
||||
|
||||
|
||||
def run_tests(**kwargs):
|
||||
from wptrunner import wptrunner
|
||||
from wptrunner import wptcommandline
|
||||
|
||||
# By default, Rayon selects the number of worker threads based on the
|
||||
# available CPU count. This doesn't work very well when running tests on CI,
|
||||
# since we run so many Servo processes in parallel. The result is a lot of
|
||||
# extra timeouts. Instead, force Rayon to assume we are running on a 2 CPU
|
||||
# environment.
|
||||
os.environ["RAYON_RS_NUM_CPUS"] = "2"
|
||||
os.environ["RUST_BACKTRACE"] = "1"
|
||||
os.environ["HOST_FILE"] = os.path.join(SERVO_ROOT, "tests", "wpt", "hosts")
|
||||
|
||||
set_if_none(kwargs, "product", "servo")
|
||||
set_if_none(kwargs, "config", os.path.join(SCRIPT_PATH, "config.ini"))
|
||||
set_if_none(kwargs, "include_manifest", os.path.join(SCRIPT_PATH, "include.ini"))
|
||||
set_if_none(kwargs, "manifest_update", False)
|
||||
set_if_none(kwargs, "processes", multiprocessing.cpu_count())
|
||||
|
||||
set_if_none(kwargs, "ca_cert_path", os.path.join(CERTS_PATH, "cacert.pem"))
|
||||
set_if_none(
|
||||
kwargs, "host_key_path", os.path.join(CERTS_PATH, "web-platform.test.key")
|
||||
)
|
||||
set_if_none(
|
||||
kwargs, "host_cert_path", os.path.join(CERTS_PATH, "web-platform.test.pem")
|
||||
)
|
||||
|
||||
kwargs["user_stylesheets"].append(os.path.join(SERVO_ROOT, "resources", "ahem.css"))
|
||||
|
||||
if "CARGO_TARGET_DIR" in os.environ:
|
||||
target_dir = os.path.join(os.environ["CARGO_TARGET_DIR"])
|
||||
else:
|
||||
target_dir = os.path.join(SERVO_ROOT, "target")
|
||||
default_binary_path = os.path.join(
|
||||
target_dir, determine_build_type(kwargs, target_dir), "servo"
|
||||
)
|
||||
if sys.platform == "win32":
|
||||
target_dir += ".exe"
|
||||
|
||||
set_if_none(kwargs, "binary", default_binary_path)
|
||||
set_if_none(kwargs, "webdriver_binary", default_binary_path)
|
||||
|
||||
if kwargs.pop("rr_chaos"):
|
||||
kwargs["debugger"] = "rr"
|
||||
kwargs["debugger_args"] = "record --chaos"
|
||||
kwargs["repeat_until_unexpected"] = True
|
||||
# TODO: Delete rr traces from green test runs?
|
||||
|
||||
prefs = kwargs.pop("prefs")
|
||||
if prefs:
|
||||
kwargs["binary_args"] = ["--pref=" + pref for pref in prefs]
|
||||
|
||||
if not kwargs.get("no_default_test_types"):
|
||||
test_types = {
|
||||
"servo": ["testharness", "reftest", "wdspec"],
|
||||
"servodriver": ["testharness", "reftest"],
|
||||
}
|
||||
product = kwargs.get("product") or "servo"
|
||||
kwargs["test_types"] = test_types[product]
|
||||
|
||||
wptcommandline.check_args(kwargs)
|
||||
update_args_for_layout_2020(kwargs)
|
||||
|
||||
mozlog.commandline.log_formatters["servo"] = (
|
||||
grouping_formatter.ServoFormatter,
|
||||
"Servo's grouping output formatter",
|
||||
)
|
||||
mozlog.commandline.log_formatters["servojson"] = (
|
||||
grouping_formatter.ServoJsonFormatter,
|
||||
"Servo's JSON logger of unexpected results",
|
||||
)
|
||||
|
||||
use_mach_logging = False
|
||||
if len(kwargs["test_list"]) == 1:
|
||||
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
|
||||
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
|
||||
use_mach_logging = True
|
||||
|
||||
if use_mach_logging:
|
||||
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
else:
|
||||
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
|
||||
|
||||
success = wptrunner.run_tests(**kwargs)
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
def update_tests(**kwargs):
|
||||
from update import updatecommandline
|
||||
|
||||
set_if_none(kwargs, "product", "servo")
|
||||
set_if_none(kwargs, "config", os.path.join(SCRIPT_PATH, "config.ini"))
|
||||
kwargs["store_state"] = False
|
||||
|
||||
updatecommandline.check_args(kwargs)
|
||||
|
||||
logger = update.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
return_value = update.run_update(logger, **kwargs)
|
||||
return 1 if return_value is update.exit_unclean else 0
|
||||
|
||||
|
||||
def main():
|
||||
from wptrunner import wptcommandline
|
||||
|
||||
parser = wptcommandline.create_parser()
|
||||
kwargs = vars(parser.parse_args())
|
||||
return run_tests(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if main() else 1)
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import sys
|
||||
from wptrunner import wptcommandline
|
||||
from update import updatecommandline
|
||||
|
||||
here = os.path.split(__file__)[0]
|
||||
|
||||
|
||||
def wpt_path(*args):
|
||||
return os.path.join(here, *args)
|
||||
|
||||
|
||||
def update_tests(**kwargs):
|
||||
import update
|
||||
|
||||
set_defaults(kwargs)
|
||||
logger = update.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
|
||||
rv = update.run_update(logger, **kwargs)
|
||||
return 1 if rv is update.exit_unclean else 0
|
||||
|
||||
|
||||
def set_defaults(kwargs):
|
||||
if kwargs["product"] is None:
|
||||
kwargs["product"] = "servo"
|
||||
if kwargs["config"] is None:
|
||||
kwargs["config"] = wpt_path('config.ini')
|
||||
kwargs["store_state"] = False
|
||||
updatecommandline.check_args(kwargs)
|
||||
|
||||
if kwargs.pop("layout_2020"):
|
||||
kwargs["test_paths"]["/"]["metadata_path"] = wpt_path("metadata-layout-2020")
|
||||
kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = wpt_path("mozilla/meta-layout-2020")
|
||||
kwargs["include_manifest"] = wpt_path("include-layout-2020.ini")
|
||||
|
||||
|
||||
def main():
|
||||
parser = wptcommandline.create_parser()
|
||||
kwargs = vars(parser.parse_args())
|
||||
return update_tests(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if main() else 1)
|
||||
Loading…
Add table
Add a link
Reference in a new issue