diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c17d4110c1..a2e46bf9548 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -212,12 +212,7 @@ jobs: # --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ # --log-raw test-wpt.${{ matrix.chunk_id }}.log \ # --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \ - # --always-succeed - # python3 ./mach filter-intermittents wpt-jsonsummary.${{ matrix.chunk_id }}.log \ - # --log-intermittents=intermittents.${{ matrix.chunk_id }}.log \ - # --log-filteredsummary=filtered-wpt-summary.${{ matrix.chunk_id }}.log \ - # --tracker-api=default --reporter-api=default - + # --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log # - name: Archive logs # uses: actions/upload-artifact@v3 # if: ${{ failure() }} @@ -227,7 +222,6 @@ jobs: # test-wpt.${{ matrix.chunk_id }}.log # wpt-jsonsummary.${{ matrix.chunk_id }}.log # filtered-wpt-summary.${{ matrix.chunk_id }}.log - # intermittents.${{ matrix.chunk_id }}.log build-linux: name: Build (Linux) @@ -290,11 +284,7 @@ jobs: --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \ --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \ - --always-succeed - python3 ./mach filter-intermittents wpt-jsonsummary.${{ matrix.chunk_id }}.log \ - --log-intermittents=intermittents.${{ matrix.chunk_id }}.log \ - --log-filteredsummary=filtered-wpt-summary.${{ matrix.chunk_id }}.log \ - --tracker-api=default --reporter-api=default + --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log - name: Archive logs uses: actions/upload-artifact@v3 if: ${{ failure() }} @@ -304,7 +294,6 @@ jobs: test-wpt.${{ matrix.chunk_id }}.log wpt-jsonsummary.${{ matrix.chunk_id }}.log filtered-wpt-summary.${{ matrix.chunk_id }}.log - intermittents.${{ matrix.chunk_id }}.log build_result: name: homu build finished diff --git a/.github/workflows/wpt-nightly.yml b/.github/workflows/wpt-nightly.yml index 7197191bcce..b558c1a2281 100644 --- a/.github/workflows/wpt-nightly.yml +++ b/.github/workflows/wpt-nightly.yml @@ -80,8 +80,6 @@ jobs: path: | test-wpt.${{ matrix.chunk_id }}.log wpt-jsonsummary.${{ matrix.chunk_id }}.log - filtered-wpt-summary.${{ matrix.chunk_id }}.log - intermittents.${{ matrix.chunk_id }}.log sync: name: Synchronize WPT Nightly diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index c98311ad9c5..8e4e4858aee 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -17,9 +17,6 @@ import os.path as path import copy from collections import OrderedDict import time -import json -import six.moves.urllib as urllib -import base64 import shutil import subprocess from xml.etree.ElementTree import XML @@ -48,6 +45,9 @@ PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", "..")) WEB_PLATFORM_TESTS_PATH = os.path.join("tests", "wpt", "web-platform-tests") SERVO_TESTS_PATH = os.path.join("tests", "wpt", "mozilla", "tests") +sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt')) +import servowpt # noqa: E402 + CLANGFMT_CPP_DIRS = ["support/hololens/"] CLANGFMT_VERSION = "14" @@ -83,7 +83,10 @@ def create_parser_wpt(): parser.add_argument('--always-succeed', default=False, action="store_true", help="Always yield exit code of zero") parser.add_argument('--no-default-test-types', default=False, action="store_true", - help="Run all of the test types provided by wptrunner or specified explicitly by --test-types"), + help="Run all of the test types provided by wptrunner or specified explicitly by --test-types") + parser.add_argument('--filter-intermittents', default=None, action="store", + help="Filter intermittents against known intermittents " + "and save the filtered output to the given file.") return parser @@ -427,9 +430,6 @@ class MachCommands(CommandBase): def _test_wpt(self, android=False, **kwargs): self.set_run_env(android) - - sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt')) - import servowpt return servowpt.run_tests(**kwargs) # Helper to ensure all specified paths are handled, otherwise dispatch to appropriate test suite. @@ -477,115 +477,8 @@ class MachCommands(CommandBase): if not patch and kwargs["sync"]: print("Are you sure you don't want a patch?") return 1 - - sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt')) - import servowpt return servowpt.update_tests(**kwargs) - @Command('filter-intermittents', - description='Given a WPT error summary file, filter out intermittents and other cruft.', - category='testing') - @CommandArgument('summary', - help="Error summary log to take in") - @CommandArgument('--log-filteredsummary', default=None, - help='Print filtered log to file') - @CommandArgument('--log-intermittents', default=None, - help='Print intermittents to file') - @CommandArgument('--json', dest="json_mode", default=False, action="store_true", - help='Output filtered and intermittents as JSON') - @CommandArgument('--auth', default=None, - help='File containing basic authorization credentials for Github API (format `username:password`)') - @CommandArgument('--tracker-api', default=None, action='store', - help='The API endpoint for tracking known intermittent failures.') - @CommandArgument('--reporter-api', default=None, action='store', - help='The API endpoint for reporting tracked intermittent failures.') - def filter_intermittents(self, - summary, - log_filteredsummary, - log_intermittents, - json_mode, - auth, - tracker_api, - reporter_api): - encoded_auth = None - if auth: - with open(auth, "r") as file: - encoded_auth = base64.encodestring(file.read().strip()).replace('\n', '') - failures = [] - with open(summary, "r") as file: - failures = [json.loads(line) for line in file] - actual_failures = [] - intermittents = [] - progress = 0 - for failure in failures: - if tracker_api: - if tracker_api == 'default': - tracker_api = "https://build.servo.org/intermittent-tracker" - elif tracker_api.endswith('/'): - tracker_api = tracker_api[0:-1] - - if 'test' not in failure: - continue - query = urllib.parse.quote(failure['test'], safe='') - request = urllib.request.Request("%s/query.py?name=%s" % (tracker_api, query)) - search = urllib.request.urlopen(request) - data = json.load(search) - is_intermittent = len(data) > 0 - else: - qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test'] - # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+` - query = urllib.parse.quote(qstr, safe='+') - request = urllib.request.Request("https://api.github.com/search/issues?q=%s" % query) - if encoded_auth: - request.add_header("Authorization", "Basic %s" % encoded_auth) - search = urllib.request.urlopen(request) - data = json.load(search) - is_intermittent = data['total_count'] > 0 - - progress += 1 - print(f" [{progress}/{len(failures)}]", file=sys.stderr, end="\r") - - if is_intermittent: - if json_mode: - intermittents.append(failure) - elif 'output' in failure: - intermittents.append(failure["output"]) - else: - intermittents.append("%s [expected %s] %s \n" - % (failure["status"], failure["expected"], failure['test'])) - else: - if json_mode: - actual_failures.append(failure) - elif 'output' in failure: - actual_failures.append(failure["output"]) - else: - actual_failures.append("%s [expected %s] %s \n" - % (failure["status"], failure["expected"], failure['test'])) - - def format(outputs, description, file=sys.stdout): - if json_mode: - formatted = json.dumps(outputs) - else: - formatted = "%s %s:\n%s" % (len(outputs), description, "\n".join(outputs)) - if file == sys.stdout: - file.write(formatted) - else: - file.write(formatted.encode("utf-8")) - - if log_intermittents: - with open(log_intermittents, "wb") as file: - format(intermittents, "known-intermittent unexpected results", file) - - description = "unexpected results that are NOT known-intermittents" - if log_filteredsummary: - with open(log_filteredsummary, "wb") as file: - format(actual_failures, description, file) - - if actual_failures: - format(actual_failures, description) - - return bool(actual_failures) - @Command('test-android-startup', description='Extremely minimal testing of Servo for Android', category='testing') diff --git a/tests/wpt/grouping_formatter.py b/tests/wpt/grouping_formatter.py index 49cf8ece626..7217992e83c 100644 --- a/tests/wpt/grouping_formatter.py +++ b/tests/wpt/grouping_formatter.py @@ -2,42 +2,41 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from mozlog.formatters import base import collections -import json import os import sys -import subprocess -import platform +import mozlog +import mozlog.formatters.base +import mozlog.reader + +from typing import Dict, List, NamedTuple from six import itervalues, iteritems DEFAULT_MOVE_UP_CODE = u"\x1b[A" DEFAULT_CLEAR_EOL_CODE = u"\x1b[K" -class ServoFormatter(base.BaseFormatter): - """Formatter designed to produce unexpected test results grouped - together in a readable format.""" +class UnexpectedResult(NamedTuple): + test_name: str + test_status: str + output: str + + +class ServoHandler(mozlog.reader.LogHandler): + """LogHandler designed to collect unexpected results for use by + script or by the ServoFormatter output formatter.""" def __init__(self): + self.reset_state() + + def reset_state(self): self.number_of_tests = 0 self.completed_tests = 0 self.need_to_erase_last_line = False - self.current_display = "" - self.running_tests = {} + self.running_tests: Dict[str, str] = {} self.test_output = collections.defaultdict(str) self.subtest_failures = collections.defaultdict(list) - self.test_failure_text = "" self.tests_with_failing_subtests = [] - self.interactive = os.isatty(sys.stdout.fileno()) - - # TODO(mrobinson, 8313): We need to add support for Windows terminals here. - if self.interactive: - self.move_up, self.clear_eol = self.get_move_up_and_clear_eol_codes() - if platform.system() != "Windows": - self.line_width = int(subprocess.check_output(['stty', 'size']).split()[1]) - else: - # Until we figure out proper Windows support, this makes things work well enough to run. - self.line_width = 80 + self.unexpected_results: List[UnexpectedResult] = [] self.expected = { 'OK': 0, @@ -60,19 +59,161 @@ class ServoFormatter(base.BaseFormatter): 'PRECONDITION_FAILED': [], } - def get_move_up_and_clear_eol_codes(self): - try: - import blessings - except ImportError: - return DEFAULT_MOVE_UP_CODE, DEFAULT_CLEAR_EOL_CODE + def suite_start(self, data): + self.reset_state() + self.number_of_tests = sum(len(tests) for tests in itervalues(data["tests"])) + self.suite_start_time = data["time"] - try: - self.terminal = blessings.Terminal() - return self.terminal.move_up, self.terminal.clear_eol - except Exception as exception: - sys.stderr.write("GroupingFormatter: Could not get terminal " - "control characters: %s\n" % exception) - return DEFAULT_MOVE_UP_CODE, DEFAULT_CLEAR_EOL_CODE + def suite_end(self, _): + pass + + def test_start(self, data): + self.running_tests[data['thread']] = data['test'] + + def wrap_and_indent_lines(self, lines, indent): + assert(len(lines) > 0) + + output = indent + u"\u25B6 %s\n" % lines[0] + for line in lines[1:-1]: + output += indent + u"\u2502 %s\n" % line + if len(lines) > 1: + output += indent + u"\u2514 %s\n" % lines[-1] + return output + + def get_lines_for_unexpected_result(self, + test_name, + status, + expected, + message, + stack): + # Test names sometimes contain control characters, which we want + # to be printed in their raw form, and not their interpreted form. + test_name = test_name.encode('unicode-escape') + + if expected: + expected_text = f" [expected {expected}]" + else: + expected_text = u"" + + lines = [f"{status}{expected_text} {test_name}"] + if message: + for message_line in message.splitlines(): + lines.append(f" \u2192 {message_line}") + if stack: + lines.append("") + lines.extend(stack.splitlines()) + return lines + + def get_output_for_unexpected_subtests(self, test_name, unexpected_subtests): + if not unexpected_subtests: + return "" + + def add_subtest_failure(lines, subtest, stack=None): + lines += self.get_lines_for_unexpected_result( + subtest.get('subtest', None), + subtest.get('status', None), + subtest.get('expected', None), + subtest.get('message', None), + stack) + + def make_subtests_failure(test_name, subtests, stack=None): + lines = [u"Unexpected subtest result in %s:" % test_name] + for subtest in subtests[:-1]: + add_subtest_failure(lines, subtest, None) + add_subtest_failure(lines, subtests[-1], stack) + return self.wrap_and_indent_lines(lines, " ") + + # Organize the failures by stack trace so we don't print the same stack trace + # more than once. They are really tall and we don't want to flood the screen + # with duplicate information. + output = "" + failures_by_stack = collections.defaultdict(list) + for failure in unexpected_subtests: + # Print stackless results first. They are all separate. + if 'stack' not in failure: + output += make_subtests_failure(test_name, [failure], None) + else: + failures_by_stack[failure['stack']].append(failure) + + for (stack, failures) in iteritems(failures_by_stack): + output += make_subtests_failure(test_name, failures, stack) + return output + + def test_end(self, data): + self.completed_tests += 1 + test_status = data["status"] + test_name = data["test"] + had_unexpected_test_result = "expected" in data + subtest_failures = self.subtest_failures.pop(test_name, []) + + del self.running_tests[data['thread']] + + if not had_unexpected_test_result and not subtest_failures: + self.expected[test_status] += 1 + return None + + # If the test crashed or timed out, we also include any process output, + # because there is a good chance that the test produced a stack trace + # or other error messages. + if test_status in ("CRASH", "TIMEOUT"): + stack = self.test_output[test_name] + data.get('stack', "") + else: + stack = data.get('stack', None) + + output = "" + if had_unexpected_test_result: + self.unexpected_tests[test_status].append(data) + lines = self.get_lines_for_unexpected_result( + test_name, + test_status, + data.get('expected', None), + data.get('message', None), + stack) + output += self.wrap_and_indent_lines(lines, " ") + + if subtest_failures: + self.tests_with_failing_subtests.append(test_name) + output += self.get_output_for_unexpected_subtests(test_name, + subtest_failures) + self.unexpected_results.append( + UnexpectedResult(test_name, test_status, output)) + return output + + def test_status(self, data): + if "expected" in data: + self.subtest_failures[data["test"]].append(data) + + def process_output(self, data): + if data['thread'] not in self.running_tests: + return + test_name = self.running_tests[data['thread']] + self.test_output[test_name] += data['data'] + "\n" + + def log(self, _): + pass + + +class ServoFormatter(mozlog.formatters.base.BaseFormatter, ServoHandler): + """Formatter designed to produce unexpected test results grouped + together in a readable format.""" + def __init__(self): + ServoHandler.__init__(self) + self.current_display = "" + self.interactive = os.isatty(sys.stdout.fileno()) + + if self.interactive: + self.line_width = os.get_terminal_size().columns + self.move_up = DEFAULT_MOVE_UP_CODE + self.clear_eol = DEFAULT_CLEAR_EOL_CODE + + try: + import blessings + self.terminal = blessings.Terminal() + self.move_up = self.terminal.move_up + self.clear_eol = self.terminal.clear_eol + except Exception as exception: + sys.stderr.write("GroupingFormatter: Could not get terminal " + "control characters: %s\n" % exception) def text_to_erase_display(self): if not self.interactive or not self.current_display: @@ -80,7 +221,7 @@ class ServoFormatter(base.BaseFormatter): return ((self.move_up + self.clear_eol) * self.current_display.count('\n')) - def generate_output(self, text=None, new_display=None, unexpected_in_test=None): + def generate_output(self, text=None, new_display=None): if not self.interactive: return text @@ -112,148 +253,42 @@ class ServoFormatter(base.BaseFormatter): return new_display + "No tests running.\n" def suite_start(self, data): - self.number_of_tests = sum(len(tests) for tests in itervalues(data["tests"])) - self.start_time = data["time"] - + ServoHandler.suite_start(self, data) if self.number_of_tests == 0: return "Running tests in %s\n\n" % data[u'source'] else: return "Running %i tests in %s\n\n" % (self.number_of_tests, data[u'source']) def test_start(self, data): - self.running_tests[data['thread']] = data['test'] + ServoHandler.test_start(self, data) if self.interactive: return self.generate_output(new_display=self.build_status_line()) - def wrap_and_indent_lines(self, lines, indent): - assert(len(lines) > 0) - - output = indent + u"\u25B6 %s\n" % lines[0] - for line in lines[1:-1]: - output += indent + u"\u2502 %s\n" % line - if len(lines) > 1: - output += indent + u"\u2514 %s\n" % lines[-1] - return output - - def get_lines_for_unexpected_result(self, - test_name, - status, - expected, - message, - stack): - # Test names sometimes contain control characters, which we want - # to be printed in their raw form, and not their interpreted form. - test_name = test_name.encode('unicode-escape') - - if expected: - expected_text = u" [expected %s]" % expected - else: - expected_text = u"" - - lines = [u"%s%s %s" % (status, expected_text, test_name)] - if message: - for message_line in message.splitlines(): - lines.append(u" \u2192 %s" % message_line) - if stack: - lines.append("") - lines.extend(stack.splitlines()) - return lines - - def get_output_for_unexpected_subtests(self, test_name, unexpected_subtests): - if not unexpected_subtests: - return "" - - def add_subtest_failure(lines, subtest, stack=None): - lines += self.get_lines_for_unexpected_result( - subtest.get('subtest', None), - subtest.get('status', None), - subtest.get('expected', None), - subtest.get('message', None), - stack) - - def make_subtests_failure(test_name, subtests, stack=None): - lines = [u"Unexpected subtest result in %s:" % test_name] - for subtest in subtests[:-1]: - add_subtest_failure(lines, subtest, None) - add_subtest_failure(lines, subtests[-1], stack) - return self.wrap_and_indent_lines(lines, " ") + "\n" - - # Organize the failures by stack trace so we don't print the same stack trace - # more than once. They are really tall and we don't want to flood the screen - # with duplicate information. - output = "" - failures_by_stack = collections.defaultdict(list) - for failure in unexpected_subtests: - # Print stackless results first. They are all separate. - if 'stack' not in failure: - output += make_subtests_failure(test_name, [failure], None) - else: - failures_by_stack[failure['stack']].append(failure) - - for (stack, failures) in iteritems(failures_by_stack): - output += make_subtests_failure(test_name, failures, stack) - return output - def test_end(self, data): - self.completed_tests += 1 - test_status = data["status"] - test_name = data["test"] - had_unexpected_test_result = "expected" in data - subtest_failures = self.subtest_failures.pop(test_name, []) - - del self.running_tests[data['thread']] - - if not had_unexpected_test_result and not subtest_failures: - self.expected[test_status] += 1 + output_for_unexpected_test = ServoHandler.test_end(self, data) + if not output_for_unexpected_test: if self.interactive: - new_display = self.build_status_line() - return self.generate_output(new_display=new_display) + return self.generate_output(new_display=self.build_status_line()) else: - return self.generate_output(text="%s%s\n" % (self.test_counter(), test_name)) + return self.generate_output(text="%s%s\n" % (self.test_counter(), data["test"])) - # If the test crashed or timed out, we also include any process output, - # because there is a good chance that the test produced a stack trace - # or other error messages. - if test_status in ("CRASH", "TIMEOUT"): - stack = self.test_output[test_name] + data.get('stack', "") - else: - stack = data.get('stack', None) - - output = "" - if had_unexpected_test_result: - self.unexpected_tests[test_status].append(data) - lines = self.get_lines_for_unexpected_result( - test_name, - test_status, - data.get('expected', None), - data.get('message', None), - stack) - output += self.wrap_and_indent_lines(lines, " ") + "\n" - - if subtest_failures: - self.tests_with_failing_subtests.append(test_name) - output += self.get_output_for_unexpected_subtests(test_name, - subtest_failures) - self.test_failure_text += output - - new_display = self.build_status_line() - return self.generate_output(text=output, new_display=new_display, - unexpected_in_test=test_name) + # Surround test output by newlines so that it is easier to read. + output_for_unexpected_test = f"{output_for_unexpected_test}\n" + return self.generate_output(text=output_for_unexpected_test, + new_display=self.build_status_line()) def test_status(self, data): - if "expected" in data: - self.subtest_failures[data["test"]].append(data) + ServoHandler.test_status(self, data) def suite_end(self, data): - self.end_time = data["time"] - + ServoHandler.suite_end(self, data) if not self.interactive: output = u"\n" else: output = "" output += u"Ran %i tests finished in %.1f seconds.\n" % ( - self.completed_tests, (self.end_time - self.start_time) / 1000) + self.completed_tests, (data["time"] - self.suite_start_time) / 1000) output += u" \u2022 %i ran as expected. %i tests skipped.\n" % ( sum(self.expected.values()), self.expected['SKIP']) @@ -279,18 +314,18 @@ class ServoFormatter(base.BaseFormatter): # Repeat failing test output, so that it is easier to find, since the # non-interactive version prints all the test names. - if not self.interactive and self.test_failure_text: - output += u"Tests with unexpected results:\n" + self.test_failure_text + if not self.interactive and self.unexpected_results: + output += u"Tests with unexpected results:\n" + output += "".join([result.output for result in self.unexpected_results]) return self.generate_output(text=output, new_display="") def process_output(self, data): - if data['thread'] not in self.running_tests: - return - test_name = self.running_tests[data['thread']] - self.test_output[test_name] += data['data'] + "\n" + ServoHandler.process_output(self, data) def log(self, data): + ServoHandler.log(self, data) + # We are logging messages that begin with STDERR, because that is how exceptions # in this formatter are indicated. if data['message'].startswith('STDERR'): @@ -298,16 +333,3 @@ class ServoFormatter(base.BaseFormatter): if data['level'] in ('CRITICAL', 'ERROR'): return self.generate_output(text=data['message'] + "\n") - - -class ServoJsonFormatter(ServoFormatter): - def suite_start(self, data): - ServoFormatter.suite_start(self, data) - # Don't forward the return value - - def generate_output(self, text=None, new_display=None, unexpected_in_test=None): - if unexpected_in_test: - return "%s\n" % json.dumps({"test": unexpected_in_test, "output": text}) - - def log(self, _): - return diff --git a/tests/wpt/servowpt.py b/tests/wpt/servowpt.py index 6364d1645e0..fee6dcf2dfb 100644 --- a/tests/wpt/servowpt.py +++ b/tests/wpt/servowpt.py @@ -2,21 +2,32 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import grouping_formatter +import json import os import sys +import urllib.parse +import urllib.request -import grouping_formatter import mozlog +import mozlog.formatters import multiprocessing +from typing import List +from grouping_formatter import UnexpectedResult + SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) SERVO_ROOT = os.path.abspath(os.path.join(SCRIPT_PATH, "..", "..")) WPT_TOOLS_PATH = os.path.join(SCRIPT_PATH, "web-platform-tests", "tools") CERTS_PATH = os.path.join(WPT_TOOLS_PATH, "certs") sys.path.insert(0, WPT_TOOLS_PATH) -import update # noqa: F401,E402 import localpaths # noqa: F401,E402 +import update # noqa: F401,E402 + +TRACKER_API = "https://build.servo.org/intermittent-tracker" +TRACKER_API_ENV_VAR = "INTERMITTENT_TRACKER_API" +GITHUB_API_TOKEN_ENV_VAR = "INTERMITTENT_TRACKER_GITHUB_API_TOKEN" def determine_build_type(kwargs: dict, target_dir: str): @@ -109,6 +120,8 @@ def run_tests(**kwargs): product = kwargs.get("product") or "servo" kwargs["test_types"] = test_types[product] + filter_intermittents_output = kwargs.pop("filter_intermittents", None) + wptcommandline.check_args(kwargs) update_args_for_layout_2020(kwargs) @@ -116,10 +129,6 @@ def run_tests(**kwargs): grouping_formatter.ServoFormatter, "Servo's grouping output formatter", ) - mozlog.commandline.log_formatters["servojson"] = ( - grouping_formatter.ServoJsonFormatter, - "Servo's JSON logger of unexpected results", - ) use_mach_logging = False if len(kwargs["test_list"]) == 1: @@ -128,12 +137,22 @@ def run_tests(**kwargs): use_mach_logging = True if use_mach_logging: - wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) + logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout}) else: - wptrunner.setup_logging(kwargs, {"servo": sys.stdout}) + logger = wptrunner.setup_logging(kwargs, {"servo": sys.stdout}) - success = wptrunner.run_tests(**kwargs) - return 0 if success else 1 + handler = grouping_formatter.ServoHandler() + logger.add_handler(handler) + + wptrunner.run_tests(**kwargs) + if handler.unexpected_results and filter_intermittents_output: + all_filtered = filter_intermittents( + handler.unexpected_results, + filter_intermittents_output, + ) + return 0 if all_filtered else 1 + else: + return 0 if not handler.unexpected_results else 1 def update_tests(**kwargs): @@ -150,6 +169,81 @@ def update_tests(**kwargs): return 1 if return_value is update.exit_unclean else 0 +class TrackerFilter(): + def __init__(self): + self.url = os.environ.get(TRACKER_API_ENV_VAR, TRACKER_API) + if self.url.endswith("/"): + self.url = self.url[0:-1] + + def is_failure_intermittent(self, test_name): + query = urllib.parse.quote(test_name, safe='') + request = urllib.request.Request("%s/query.py?name=%s" % (self.url, query)) + search = urllib.request.urlopen(request) + return len(json.load(search)) > 0 + + +class GitHubQueryFilter(): + def __init__(self, token): + self.token = token + + def is_failure_intermittent(self, test_name): + url = "https://api.github.com/search/issues?q=" + query = "repo:servo/servo+" + \ + "label:I-intermittent+" + \ + "type:issue+" + \ + "state:open+" + \ + test_name + + # we want `/` to get quoted, but not `+` (github's API doesn't like + # that), so we set `safe` to `+` + url += urllib.parse.quote(query, safe="+") + + request = urllib.request.Request(url) + request.add_header("Authorization", f"Bearer: {self.token}") + request.add_header("Accept", "application/vnd.github+json") + return json.load( + urllib.request.urlopen(request) + )["total_count"] > 0 + + +def filter_intermittents( + unexpected_results: List[UnexpectedResult], + output_file: str +) -> bool: + print(80 * "=") + print(f"Filtering {len(unexpected_results)} unexpected " + "results for known intermittents") + if GITHUB_API_TOKEN_ENV_VAR in os.environ: + filter = GitHubQueryFilter(os.environ.get(GITHUB_API_TOKEN_ENV_VAR)) + else: + filter = TrackerFilter() + + intermittents = [] + actually_unexpected = [] + for i, result in enumerate(unexpected_results): + print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r") + if filter.is_failure_intermittent(result.test_name): + intermittents.append(result) + else: + actually_unexpected.append(result) + + output = "\n".join([ + f"{len(intermittents)} known-intermittent unexpected result", + *[result.output.strip() for result in intermittents], + "", + f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents", + *[result.output.strip() for result in actually_unexpected], + ]) + + if output_file: + with open(output_file, "w", encoding="utf-8") as file: + file.write(output) + + print(output) + print(80 * "=") + return not actually_unexpected + + def main(): from wptrunner import wptcommandline