diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index d9d26366d20..5e55ca18925 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -41,6 +41,8 @@ def create_parser_wpt(): parser = wptcommandline.create_parser() parser.add_argument('--release', default=False, action="store_true", help="Run with a release build of servo") + parser.add_argument('--chaos', default=False, action="store_true", + help="Run under chaos mode in rr until a failure is captured") return parser @@ -332,19 +334,24 @@ class MachCommands(CommandBase): @Command('test-wpt', description='Run the web platform tests', category='testing', - parser=wptcommandline.create_parser) - @CommandArgument('--release', default=False, action="store_true", - help="Run with a release build of servo") + parser=create_parser_wpt) def test_wpt(self, **kwargs): self.ensure_bootstrapped() hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts') - os.environ["hosts_file_path"] = hosts_file_path - os.environ["RUST_BACKTRACE"] = "1" - - kwargs["debug"] = not kwargs["release"] - run_file = path.abspath(path.join(self.context.topdir, "tests", "wpt", "run_wpt.py")) + return self.wptrunner(run_file, **kwargs) + + # Helper for test_css and test_wpt: + def wptrunner(self, run_file, **kwargs): + os.environ["RUST_BACKTRACE"] = "1" + kwargs["debug"] = not kwargs["release"] + if kwargs.pop("chaos"): + kwargs["debugger"] = "rr" + kwargs["debugger_args"] = "record --chaos" + kwargs["repeat_until_unexpected"] = True + # TODO: Delete rr traces from green test runs? + run_globals = {"__file__": run_file} execfile(run_file, run_globals) return run_globals["run_tests"](**kwargs) @@ -398,11 +405,8 @@ class MachCommands(CommandBase): parser=create_parser_wpt) def test_css(self, **kwargs): self.ensure_bootstrapped() - run_file = path.abspath(path.join("tests", "wpt", "run_css.py")) - run_globals = {"__file__": run_file} - execfile(run_file, run_globals) - return run_globals["run_tests"](**kwargs) + return self.wptrunner(run_file, **kwargs) @Command('update-css', description='Update the web platform tests', diff --git a/tests/wpt/harness/wptrunner/wptcommandline.py b/tests/wpt/harness/wptrunner/wptcommandline.py index 96b69060767..5b110094e5c 100644 --- a/tests/wpt/harness/wptrunner/wptcommandline.py +++ b/tests/wpt/harness/wptrunner/wptcommandline.py @@ -72,6 +72,8 @@ def create_parser(product_choices=None): help="Multiplier relative to standard test timeout to use") parser.add_argument("--repeat", action="store", type=int, default=1, help="Number of times to run the tests") + parser.add_argument("--repeat-until-unexpected", action="store_true", default=None, + help="Run tests in a loop until one returns an unexpected result") parser.add_argument("--no-capture-stdio", action="store_true", default=False, help="Don't capture stdio and write to logging") diff --git a/tests/wpt/harness/wptrunner/wptrunner.py b/tests/wpt/harness/wptrunner/wptrunner.py index 29fd06a3619..65b02cc2431 100644 --- a/tests/wpt/harness/wptrunner/wptrunner.py +++ b/tests/wpt/harness/wptrunner/wptrunner.py @@ -160,10 +160,15 @@ def run_tests(config, test_paths, product, **kwargs): browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs) repeat = kwargs["repeat"] - for repeat_count in xrange(repeat): - if repeat > 1: - logger.info("Repetition %i / %i" % (repeat_count + 1, repeat)) + repeat_count = 0 + repeat_until_unexpected = kwargs["repeat_until_unexpected"] + while repeat_count < repeat or repeat_until_unexpected: + repeat_count += 1 + if repeat_until_unexpected: + logger.info("Repetition %i" % (repeat_count)) + elif repeat > 1: + logger.info("Repetition %i / %i" % (repeat_count, repeat)) unexpected_count = 0 logger.suite_start(test_loader.test_ids, run_info) @@ -208,6 +213,8 @@ def run_tests(config, test_paths, product, **kwargs): unexpected_total += unexpected_count logger.info("Got %i unexpected results" % unexpected_count) + if repeat_until_unexpected and unexpected_total > 0: + break logger.suite_end() return unexpected_total == 0