mirror of
https://github.com/servo/servo.git
synced 2025-08-06 14:10:11 +01:00
Auto merge of #29237 - servo:simplify-wpt-run, r=jdm
Simplify Python code for running WPT tests Combine `run.py` and `update.py` into `servowpt.py` in order to allow them to share code. Import them directly into the mach script to avoid having to call `compile` and `exec` on the code. This makes it clearer how they are executed. In addition, move all of the setup into `setupwpt.py` to avoid differences between tests executed via mach and not. Finally, be more ambitious when detecting the build to use. If none was specified, try to use the one that exists between "release" and "debug." <!-- Please describe your changes on the following line: --> --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: --> - [x] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [x] These changes do not require tests because they only change a small script behavior. <!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.--> <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->
This commit is contained in:
bors-servo
GitHub
commit
76a5c5d7d2
5 changed files with 171 additions and 211 deletions
|
|
@ -427,10 +427,10 @@ class MachCommands(CommandBase):
|
|||
|
||||
def _test_wpt(self, android=False, **kwargs):
|
||||
self.set_run_env(android)
|
||||
hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts')
|
||||
os.environ["HOST_FILE"] = hosts_file_path
|
||||
run_file = path.abspath(path.join(self.context.topdir, "tests", "wpt", "run.py"))
|
||||
return self.wptrunner(run_file, **kwargs)
|
||||
|
||||
sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt'))
|
||||
import servowpt
|
||||
return servowpt.run_tests(**kwargs)
|
||||
|
||||
# Helper to ensure all specified paths are handled, otherwise dispatch to appropriate test suite.
|
||||
def run_test_list_or_dispatch(self, requested_paths, correct_suite, correct_function, **kwargs):
|
||||
|
|
@ -448,42 +448,6 @@ class MachCommands(CommandBase):
|
|||
# Dispatch each test to the correct suite via test()
|
||||
Registrar.dispatch("test", context=self.context, params=requested_paths)
|
||||
|
||||
# Helper for test_css and test_wpt:
|
||||
def wptrunner(self, run_file, **kwargs):
|
||||
# By default, Rayon selects the number of worker threads
|
||||
# based on the available CPU count. This doesn't work very
|
||||
# well when running tests on CI, since we run so many
|
||||
# Servo processes in parallel. The result is a lot of
|
||||
# extra timeouts. Instead, force Rayon to assume we are
|
||||
# running on a 2 CPU environment.
|
||||
os.environ['RAYON_RS_NUM_CPUS'] = "2"
|
||||
|
||||
os.environ["RUST_BACKTRACE"] = "1"
|
||||
kwargs["debug"] = not kwargs["release"]
|
||||
if kwargs.pop("rr_chaos"):
|
||||
kwargs["debugger"] = "rr"
|
||||
kwargs["debugger_args"] = "record --chaos"
|
||||
kwargs["repeat_until_unexpected"] = True
|
||||
# TODO: Delete rr traces from green test runs?
|
||||
prefs = kwargs.pop("prefs")
|
||||
if prefs:
|
||||
binary_args = []
|
||||
for pref in prefs:
|
||||
binary_args.append("--pref=" + pref)
|
||||
kwargs["binary_args"] = binary_args
|
||||
|
||||
if not kwargs.get('no_default_test_types'):
|
||||
test_types = {
|
||||
"servo": ["testharness", "reftest", "wdspec"],
|
||||
"servodriver": ["testharness", "reftest"],
|
||||
}
|
||||
product = kwargs.get("product") or "servo"
|
||||
kwargs["test_types"] = test_types[product]
|
||||
|
||||
run_globals = {"__file__": run_file}
|
||||
exec(compile(open(run_file).read(), run_file, 'exec'), run_globals)
|
||||
return run_globals["run_tests"](**kwargs)
|
||||
|
||||
@Command('update-manifest',
|
||||
description='Run test-wpt --manifest-update SKIP_TESTS to regenerate MANIFEST.json',
|
||||
category='testing',
|
||||
|
|
@ -509,16 +473,14 @@ class MachCommands(CommandBase):
|
|||
category='testing',
|
||||
parser=updatecommandline.create_parser())
|
||||
def update_wpt(self, **kwargs):
|
||||
run_file = path.abspath(path.join("tests", "wpt", "update.py"))
|
||||
patch = kwargs.get("patch", False)
|
||||
|
||||
if not patch and kwargs["sync"]:
|
||||
print("Are you sure you don't want a patch?")
|
||||
return 1
|
||||
|
||||
run_globals = {"__file__": run_file}
|
||||
exec(compile(open(run_file).read(), run_file, 'exec'), run_globals)
|
||||
return run_globals["update_tests"](**kwargs)
|
||||
sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt'))
|
||||
import servowpt
|
||||
return servowpt.update_tests(**kwargs)
|
||||
|
||||
@Command('filter-intermittents',
|
||||
description='Given a WPT error summary file, filter out intermittents and other cruft.',
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ In particular, this folder contains:
|
|||
|
||||
* `config.ini`: some configuration for the web-platform-tests.
|
||||
* `include.ini`: the subset of web-platform-tests we currently run.
|
||||
* `run.py`: run the web-platform-tests in Servo.
|
||||
* `servowpt.py`: run the web-platform-tests in Servo.
|
||||
* `web-platform-tests`: copy of the web-platform-tests.
|
||||
* `metadata`: expected failures for the web-platform-tests we run.
|
||||
* `mozilla`: web-platform-tests that cannot be upstreamed.
|
||||
|
|
@ -81,7 +81,7 @@ for more details).
|
|||
Running the tests without mach
|
||||
------------------------------
|
||||
|
||||
When avoiding `mach` for some reason, one can run `run.py`
|
||||
When avoiding `mach` for some reason, one can run `servowpt.py`
|
||||
directly. However, this requires that all the dependencies for
|
||||
`wptrunner` are available in the current python environment.
|
||||
|
||||
|
|
|
|||
116
tests/wpt/run.py
116
tests/wpt/run.py
|
|
@ -1,116 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
from os import path
|
||||
import sys
|
||||
import mozlog
|
||||
import grouping_formatter
|
||||
|
||||
here = os.path.split(__file__)[0]
|
||||
servo_root = os.path.abspath(os.path.join(here, "..", ".."))
|
||||
|
||||
|
||||
def wpt_path(*args):
|
||||
return os.path.join(here, *args)
|
||||
|
||||
|
||||
def servo_path(*args):
|
||||
return os.path.join(servo_root, *args)
|
||||
|
||||
|
||||
paths = {"include_manifest": wpt_path("include.ini"),
|
||||
"config": wpt_path("config.ini"),
|
||||
"ca-cert-path": wpt_path("web-platform-tests/tools/certs/cacert.pem"),
|
||||
"host-key-path": wpt_path("web-platform-tests/tools/certs/web-platform.test.key"),
|
||||
"host-cert-path": wpt_path("web-platform-tests/tools/certs/web-platform.test.pem")}
|
||||
# Imports
|
||||
sys.path.append(wpt_path("web-platform-tests", "tools"))
|
||||
import localpaths # noqa: F401,E402
|
||||
from wptrunner import wptrunner, wptcommandline # noqa: E402
|
||||
|
||||
|
||||
def run_tests(**kwargs):
|
||||
set_defaults(kwargs)
|
||||
|
||||
mozlog.commandline.log_formatters["servo"] = \
|
||||
(grouping_formatter.ServoFormatter, "Servo's grouping output formatter")
|
||||
mozlog.commandline.log_formatters["servojson"] = \
|
||||
(grouping_formatter.ServoJsonFormatter, "Servo's JSON logger of unexpected results")
|
||||
|
||||
use_mach_logging = False
|
||||
if len(kwargs["test_list"]) == 1:
|
||||
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
|
||||
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
|
||||
use_mach_logging = True
|
||||
|
||||
if use_mach_logging:
|
||||
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
else:
|
||||
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
|
||||
|
||||
success = wptrunner.run_tests(**kwargs)
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
def set_defaults(kwargs):
|
||||
if kwargs["product"] is None:
|
||||
kwargs["product"] = "servo"
|
||||
|
||||
if kwargs["config"] is None and "config" in paths:
|
||||
kwargs["config"] = paths["config"]
|
||||
|
||||
if kwargs["include_manifest"] is None and "include_manifest" in paths:
|
||||
kwargs["include_manifest"] = paths["include_manifest"]
|
||||
|
||||
if kwargs["manifest_update"] is None:
|
||||
kwargs["manifest_update"] = False
|
||||
|
||||
if kwargs["binary"] is None:
|
||||
bin_dir = "release" if kwargs["release"] else "debug"
|
||||
bin_name = "servo"
|
||||
if sys.platform == "win32":
|
||||
bin_name += ".exe"
|
||||
if "CARGO_TARGET_DIR" in os.environ:
|
||||
bin_path = path.join(os.environ["CARGO_TARGET_DIR"], bin_dir, bin_name)
|
||||
else:
|
||||
bin_path = servo_path("target", bin_dir, bin_name)
|
||||
|
||||
kwargs["binary"] = bin_path
|
||||
kwargs["webdriver_binary"] = bin_path
|
||||
|
||||
if kwargs["processes"] is None:
|
||||
kwargs["processes"] = multiprocessing.cpu_count()
|
||||
|
||||
if kwargs["ca_cert_path"] is None:
|
||||
kwargs["ca_cert_path"] = paths["ca-cert-path"]
|
||||
|
||||
if kwargs["host_key_path"] is None:
|
||||
kwargs["host_key_path"] = paths["host-key-path"]
|
||||
|
||||
if kwargs["host_cert_path"] is None:
|
||||
kwargs["host_cert_path"] = paths["host-cert-path"]
|
||||
|
||||
if kwargs["ssl_type"] is None:
|
||||
kwargs["ssl_type"] = "pregenerated"
|
||||
|
||||
kwargs["user_stylesheets"].append(servo_path("resources", "ahem.css"))
|
||||
|
||||
wptcommandline.check_args(kwargs)
|
||||
|
||||
if kwargs.pop("layout_2020"):
|
||||
kwargs["test_paths"]["/"]["metadata_path"] = wpt_path("metadata-layout-2020")
|
||||
kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = wpt_path("mozilla/meta-layout-2020")
|
||||
kwargs["include_manifest"] = wpt_path("include-layout-2020.ini")
|
||||
|
||||
|
||||
def main():
|
||||
parser = wptcommandline.create_parser()
|
||||
kwargs = vars(parser.parse_args())
|
||||
return run_tests(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if main() else 1)
|
||||
162
tests/wpt/servowpt.py
Normal file
162
tests/wpt/servowpt.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import grouping_formatter
|
||||
import mozlog
|
||||
import multiprocessing
|
||||
|
||||
SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
|
||||
SERVO_ROOT = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
|
||||
WPT_TOOLS_PATH = os.path.join(SCRIPT_PATH, "web-platform-tests", "tools")
|
||||
CERTS_PATH = os.path.join(WPT_TOOLS_PATH, "certs")
|
||||
|
||||
sys.path.insert(0, WPT_TOOLS_PATH)
|
||||
import update # noqa: F401,E402
|
||||
import localpaths # noqa: F401,E402
|
||||
|
||||
|
||||
def determine_build_type(kwargs: dict, target_dir: str):
|
||||
if kwargs["release"]:
|
||||
return "release"
|
||||
elif kwargs["debug"]:
|
||||
return "debug"
|
||||
elif os.path.exists(os.path.join(target_dir, "debug")):
|
||||
return "debug"
|
||||
elif os.path.exists(os.path.join(target_dir, "release")):
|
||||
return "release"
|
||||
return "debug"
|
||||
|
||||
|
||||
def set_if_none(args: dict, key: str, value):
|
||||
if key not in args or args[key] is None:
|
||||
args[key] = value
|
||||
|
||||
|
||||
def update_args_for_layout_2020(kwargs: dict):
|
||||
if kwargs.pop("layout_2020"):
|
||||
kwargs["test_paths"]["/"]["metadata_path"] = os.path.join(
|
||||
SCRIPT_PATH, "metadata-layout-2020"
|
||||
)
|
||||
kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = os.path.join(
|
||||
SCRIPT_PATH, "mozilla", "meta-layout-2020"
|
||||
)
|
||||
kwargs["include_manifest"] = os.path.join(
|
||||
SCRIPT_PATH, "include-layout-2020.ini"
|
||||
)
|
||||
|
||||
|
||||
def run_tests(**kwargs):
|
||||
from wptrunner import wptrunner
|
||||
from wptrunner import wptcommandline
|
||||
|
||||
# By default, Rayon selects the number of worker threads based on the
|
||||
# available CPU count. This doesn't work very well when running tests on CI,
|
||||
# since we run so many Servo processes in parallel. The result is a lot of
|
||||
# extra timeouts. Instead, force Rayon to assume we are running on a 2 CPU
|
||||
# environment.
|
||||
os.environ["RAYON_RS_NUM_CPUS"] = "2"
|
||||
os.environ["RUST_BACKTRACE"] = "1"
|
||||
os.environ["HOST_FILE"] = os.path.join(SERVO_ROOT, "tests", "wpt", "hosts")
|
||||
|
||||
set_if_none(kwargs, "product", "servo")
|
||||
set_if_none(kwargs, "config", os.path.join(SCRIPT_PATH, "config.ini"))
|
||||
set_if_none(kwargs, "include_manifest", os.path.join(SCRIPT_PATH, "include.ini"))
|
||||
set_if_none(kwargs, "manifest_update", False)
|
||||
set_if_none(kwargs, "processes", multiprocessing.cpu_count())
|
||||
|
||||
set_if_none(kwargs, "ca_cert_path", os.path.join(CERTS_PATH, "cacert.pem"))
|
||||
set_if_none(
|
||||
kwargs, "host_key_path", os.path.join(CERTS_PATH, "web-platform.test.key")
|
||||
)
|
||||
set_if_none(
|
||||
kwargs, "host_cert_path", os.path.join(CERTS_PATH, "web-platform.test.pem")
|
||||
)
|
||||
|
||||
kwargs["user_stylesheets"].append(os.path.join(SERVO_ROOT, "resources", "ahem.css"))
|
||||
|
||||
if "CARGO_TARGET_DIR" in os.environ:
|
||||
target_dir = os.path.join(os.environ["CARGO_TARGET_DIR"])
|
||||
else:
|
||||
target_dir = os.path.join(SERVO_ROOT, "target")
|
||||
default_binary_path = os.path.join(
|
||||
target_dir, determine_build_type(kwargs, target_dir), "servo"
|
||||
)
|
||||
if sys.platform == "win32":
|
||||
target_dir += ".exe"
|
||||
|
||||
set_if_none(kwargs, "binary", default_binary_path)
|
||||
set_if_none(kwargs, "webdriver_binary", default_binary_path)
|
||||
|
||||
if kwargs.pop("rr_chaos"):
|
||||
kwargs["debugger"] = "rr"
|
||||
kwargs["debugger_args"] = "record --chaos"
|
||||
kwargs["repeat_until_unexpected"] = True
|
||||
# TODO: Delete rr traces from green test runs?
|
||||
|
||||
prefs = kwargs.pop("prefs")
|
||||
if prefs:
|
||||
kwargs["binary_args"] = ["--pref=" + pref for pref in prefs]
|
||||
|
||||
if not kwargs.get("no_default_test_types"):
|
||||
test_types = {
|
||||
"servo": ["testharness", "reftest", "wdspec"],
|
||||
"servodriver": ["testharness", "reftest"],
|
||||
}
|
||||
product = kwargs.get("product") or "servo"
|
||||
kwargs["test_types"] = test_types[product]
|
||||
|
||||
wptcommandline.check_args(kwargs)
|
||||
update_args_for_layout_2020(kwargs)
|
||||
|
||||
mozlog.commandline.log_formatters["servo"] = (
|
||||
grouping_formatter.ServoFormatter,
|
||||
"Servo's grouping output formatter",
|
||||
)
|
||||
mozlog.commandline.log_formatters["servojson"] = (
|
||||
grouping_formatter.ServoJsonFormatter,
|
||||
"Servo's JSON logger of unexpected results",
|
||||
)
|
||||
|
||||
use_mach_logging = False
|
||||
if len(kwargs["test_list"]) == 1:
|
||||
file_ext = os.path.splitext(kwargs["test_list"][0])[1].lower()
|
||||
if file_ext in [".htm", ".html", ".js", ".xhtml", ".xht", ".py"]:
|
||||
use_mach_logging = True
|
||||
|
||||
if use_mach_logging:
|
||||
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
else:
|
||||
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
|
||||
|
||||
success = wptrunner.run_tests(**kwargs)
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
def update_tests(**kwargs):
|
||||
from update import updatecommandline
|
||||
|
||||
set_if_none(kwargs, "product", "servo")
|
||||
set_if_none(kwargs, "config", os.path.join(SCRIPT_PATH, "config.ini"))
|
||||
kwargs["store_state"] = False
|
||||
|
||||
updatecommandline.check_args(kwargs)
|
||||
|
||||
logger = update.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
return_value = update.run_update(logger, **kwargs)
|
||||
return 1 if return_value is update.exit_unclean else 0
|
||||
|
||||
|
||||
def main():
|
||||
from wptrunner import wptcommandline
|
||||
|
||||
parser = wptcommandline.create_parser()
|
||||
kwargs = vars(parser.parse_args())
|
||||
return run_tests(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if main() else 1)
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import sys
|
||||
from wptrunner import wptcommandline
|
||||
from update import updatecommandline
|
||||
|
||||
here = os.path.split(__file__)[0]
|
||||
|
||||
|
||||
def wpt_path(*args):
|
||||
return os.path.join(here, *args)
|
||||
|
||||
|
||||
def update_tests(**kwargs):
|
||||
import update
|
||||
|
||||
set_defaults(kwargs)
|
||||
logger = update.setup_logging(kwargs, {"mach": sys.stdout})
|
||||
|
||||
rv = update.run_update(logger, **kwargs)
|
||||
return 1 if rv is update.exit_unclean else 0
|
||||
|
||||
|
||||
def set_defaults(kwargs):
|
||||
if kwargs["product"] is None:
|
||||
kwargs["product"] = "servo"
|
||||
if kwargs["config"] is None:
|
||||
kwargs["config"] = wpt_path('config.ini')
|
||||
kwargs["store_state"] = False
|
||||
updatecommandline.check_args(kwargs)
|
||||
|
||||
if kwargs.pop("layout_2020"):
|
||||
kwargs["test_paths"]["/"]["metadata_path"] = wpt_path("metadata-layout-2020")
|
||||
kwargs["test_paths"]["/_mozilla/"]["metadata_path"] = wpt_path("mozilla/meta-layout-2020")
|
||||
kwargs["include_manifest"] = wpt_path("include-layout-2020.ini")
|
||||
|
||||
|
||||
def main():
|
||||
parser = wptcommandline.create_parser()
|
||||
kwargs = vars(parser.parse_args())
|
||||
return update_tests(**kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(0 if main() else 1)
|
||||
Loading…
Add table
Add a link
Reference in a new issue