Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-06 16:45:39 +00:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #29294 - servo:integrate-filter-intermittents, r=delan

Browse source 
Integrate filter-intermittents into test-wpt

This change integrates the `filter-intermittents` command into `test-wpt`. This is in preparation for future work on tracking intermittent failures. This change also:
- Removes `ServoJsonFormatter` and replaces it with a generic WPT log handler which tracks unexpected results.
- The intermittent filter is now controlled via environment variables and the GitHub version requires a token instead of credentials.
- Output is saved to a single file and is always text.

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes do not require tests because they are changes to build infrastructure.
This commit is contained in:
bors-servo 2023-01-27 15:43:47 +01:00 • committed by GitHub
commit 4d5a804d03
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 306 additions and 310 deletions
Download patch file Download diff file Expand all files Collapse all files

15
.github/workflows/main.yml vendored
View file

@ -147,12 +147,7 @@ jobs:
# --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
# --log-raw test-wpt.${{ matrix.chunk_id }}.log \
# --log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \
# --always-succeed
# python3 ./mach filter-intermittents wpt-jsonsummary.${{ matrix.chunk_id }}.log \
# --log-intermittents=intermittents.${{ matrix.chunk_id }}.log \
# --log-filteredsummary=filtered-wpt-summary.${{ matrix.chunk_id }}.log \
# --tracker-api=default --reporter-api=default
# --filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log
# - name: Archive logs
# uses: actions/upload-artifact@v3
# if: ${{ failure() }}
@ -162,7 +157,6 @@ jobs:
# test-wpt.${{ matrix.chunk_id }}.log
# wpt-jsonsummary.${{ matrix.chunk_id }}.log
# filtered-wpt-summary.${{ matrix.chunk_id }}.log
# intermittents.${{ matrix.chunk_id }}.log
build-linux:
name: Build (Linux)
@ -225,11 +219,7 @@ jobs:
--total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
--log-raw test-wpt.${{ matrix.chunk_id }}.log \
--log-servojson wpt-jsonsummary.${{ matrix.chunk_id }}.log \
--always-succeed
python3 ./mach filter-intermittents wpt-jsonsummary.${{ matrix.chunk_id }}.log \
--log-intermittents=intermittents.${{ matrix.chunk_id }}.log \
--log-filteredsummary=filtered-wpt-summary.${{ matrix.chunk_id }}.log \
--tracker-api=default --reporter-api=default
--filter-intermittents=filtered-wpt-summary.${{ matrix.chunk_id }}.log
- name: Archive logs
uses: actions/upload-artifact@v3
if: ${{ failure() }}
@ -239,7 +229,6 @@ jobs:
test-wpt.${{ matrix.chunk_id }}.log
wpt-jsonsummary.${{ matrix.chunk_id }}.log
filtered-wpt-summary.${{ matrix.chunk_id }}.log
intermittents.${{ matrix.chunk_id }}.log
build_result:
name: homu build finished

2
.github/workflows/wpt-nightly.yml vendored
View file

@ -80,8 +80,6 @@ jobs:
path: |
test-wpt.${{ matrix.chunk_id }}.log
wpt-jsonsummary.${{ matrix.chunk_id }}.log
filtered-wpt-summary.${{ matrix.chunk_id }}.log
intermittents.${{ matrix.chunk_id }}.log
sync:
name: Synchronize WPT Nightly

121
python/servo/testing_commands.py
View file

@ -17,9 +17,6 @@ import os.path as path
import copy
from collections import OrderedDict
import time
import json
import six.moves.urllib as urllib
import base64
import shutil
import subprocess
from xml.etree.ElementTree import XML
@ -48,6 +45,9 @@ PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
WEB_PLATFORM_TESTS_PATH = os.path.join("tests", "wpt", "web-platform-tests")
SERVO_TESTS_PATH = os.path.join("tests", "wpt", "mozilla", "tests")
sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt'))
import servowpt # noqa: E402
CLANGFMT_CPP_DIRS = ["support/hololens/"]
CLANGFMT_VERSION = "15"
@ -83,7 +83,10 @@ def create_parser_wpt():
parser.add_argument('--always-succeed', default=False, action="store_true",
help="Always yield exit code of zero")
parser.add_argument('--no-default-test-types', default=False, action="store_true",
help="Run all of the test types provided by wptrunner or specified explicitly by --test-types"),
help="Run all of the test types provided by wptrunner or specified explicitly by --test-types")
parser.add_argument('--filter-intermittents', default=None, action="store",
help="Filter intermittents against known intermittents "
"and save the filtered output to the given file.")
return parser
@ -427,9 +430,6 @@ class MachCommands(CommandBase):
def _test_wpt(self, android=False, **kwargs):
self.set_run_env(android)
sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt'))
import servowpt
return servowpt.run_tests(**kwargs)
# Helper to ensure all specified paths are handled, otherwise dispatch to appropriate test suite.
@ -477,115 +477,8 @@ class MachCommands(CommandBase):
if not patch and kwargs["sync"]:
print("Are you sure you don't want a patch?")
return 1
sys.path.insert(0, os.path.join(PROJECT_TOPLEVEL_PATH, 'tests', 'wpt'))
import servowpt
return servowpt.update_tests(**kwargs)
@Command('filter-intermittents',
description='Given a WPT error summary file, filter out intermittents and other cruft.',
category='testing')
@CommandArgument('summary',
help="Error summary log to take in")
@CommandArgument('--log-filteredsummary', default=None,
help='Print filtered log to file')
@CommandArgument('--log-intermittents', default=None,
help='Print intermittents to file')
@CommandArgument('--json', dest="json_mode", default=False, action="store_true",
help='Output filtered and intermittents as JSON')
@CommandArgument('--auth', default=None,
help='File containing basic authorization credentials for Github API (format `username:password`)')
@CommandArgument('--tracker-api', default=None, action='store',
help='The API endpoint for tracking known intermittent failures.')
@CommandArgument('--reporter-api', default=None, action='store',
help='The API endpoint for reporting tracked intermittent failures.')
def filter_intermittents(self,
summary,
log_filteredsummary,
log_intermittents,
json_mode,
auth,
tracker_api,
reporter_api):
encoded_auth = None
if auth:
with open(auth, "r") as file:
encoded_auth = base64.encodestring(file.read().strip()).replace('\n', '')
failures = []
with open(summary, "r") as file:
failures = [json.loads(line) for line in file]
actual_failures = []
intermittents = []
progress = 0
for failure in failures:
if tracker_api:
if tracker_api == 'default':
tracker_api = "https://build.servo.org/intermittent-tracker"
elif tracker_api.endswith('/'):
tracker_api = tracker_api[0:-1]
if 'test' not in failure:
continue
query = urllib.parse.quote(failure['test'], safe='')
request = urllib.request.Request("%s/query.py?name=%s" % (tracker_api, query))
search = urllib.request.urlopen(request)
data = json.load(search)
is_intermittent = len(data) > 0
else:
qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test']
# we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+`
query = urllib.parse.quote(qstr, safe='+')
request = urllib.request.Request("https://api.github.com/search/issues?q=%s" % query)
if encoded_auth:
request.add_header("Authorization", "Basic %s" % encoded_auth)
search = urllib.request.urlopen(request)
data = json.load(search)
is_intermittent = data['total_count'] > 0
progress += 1
print(f" [{progress}/{len(failures)}]", file=sys.stderr, end="\r")
if is_intermittent:
if json_mode:
intermittents.append(failure)
elif 'output' in failure:
intermittents.append(failure["output"])
else:
intermittents.append("%s [expected %s] %s \n"
% (failure["status"], failure["expected"], failure['test']))
else:
if json_mode:
actual_failures.append(failure)
elif 'output' in failure:
actual_failures.append(failure["output"])
else:
actual_failures.append("%s [expected %s] %s \n"
% (failure["status"], failure["expected"], failure['test']))
def format(outputs, description, file=sys.stdout):
if json_mode:
formatted = json.dumps(outputs)
else:
formatted = "%s %s:\n%s" % (len(outputs), description, "\n".join(outputs))
if file == sys.stdout:
file.write(formatted)
else:
file.write(formatted.encode("utf-8"))
if log_intermittents:
with open(log_intermittents, "wb") as file:
format(intermittents, "known-intermittent unexpected results", file)
description = "unexpected results that are NOT known-intermittents"
if log_filteredsummary:
with open(log_filteredsummary, "wb") as file:
format(actual_failures, description, file)
if actual_failures:
format(actual_failures, description)
return bool(actual_failures)
@Command('test-android-startup',
description='Extremely minimal testing of Servo for Android',
category='testing')

364
tests/wpt/grouping_formatter.py
View file

@ -2,42 +2,41 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
from mozlog.formatters import base
import collections
import json
import os
import sys
import subprocess
import platform
import mozlog
import mozlog.formatters.base
import mozlog.reader
from typing import Dict, List, NamedTuple
from six import itervalues, iteritems
DEFAULT_MOVE_UP_CODE = u"\x1b[A"
DEFAULT_CLEAR_EOL_CODE = u"\x1b[K"
class ServoFormatter(base.BaseFormatter):
"""Formatter designed to produce unexpected test results grouped
together in a readable format."""
class UnexpectedResult(NamedTuple):
test_name: str
test_status: str
output: str
class ServoHandler(mozlog.reader.LogHandler):
"""LogHandler designed to collect unexpected results for use by
script or by the ServoFormatter output formatter."""
def __init__(self):
self.reset_state()
def reset_state(self):
self.number_of_tests = 0
self.completed_tests = 0
self.need_to_erase_last_line = False
self.current_display = ""
self.running_tests = {}
self.running_tests: Dict[str, str] = {}
self.test_output = collections.defaultdict(str)
self.subtest_failures = collections.defaultdict(list)
self.test_failure_text = ""
self.tests_with_failing_subtests = []
self.interactive = os.isatty(sys.stdout.fileno())
# TODO(mrobinson, 8313): We need to add support for Windows terminals here.
if self.interactive:
self.move_up, self.clear_eol = self.get_move_up_and_clear_eol_codes()
if platform.system() != "Windows":
self.line_width = int(subprocess.check_output(['stty', 'size']).split()[1])
else:
# Until we figure out proper Windows support, this makes things work well enough to run.
self.line_width = 80
self.unexpected_results: List[UnexpectedResult] = []
self.expected = {
'OK': 0,
@ -60,19 +59,161 @@ class ServoFormatter(base.BaseFormatter):
'PRECONDITION_FAILED': [],
}
def get_move_up_and_clear_eol_codes(self):
try:
import blessings
except ImportError:
return DEFAULT_MOVE_UP_CODE, DEFAULT_CLEAR_EOL_CODE
def suite_start(self, data):
self.reset_state()
self.number_of_tests = sum(len(tests) for tests in itervalues(data["tests"]))
self.suite_start_time = data["time"]
try:
self.terminal = blessings.Terminal()
return self.terminal.move_up, self.terminal.clear_eol
except Exception as exception:
sys.stderr.write("GroupingFormatter: Could not get terminal "
"control characters: %s\n" % exception)
return DEFAULT_MOVE_UP_CODE, DEFAULT_CLEAR_EOL_CODE
def suite_end(self, _):
pass
def test_start(self, data):
self.running_tests[data['thread']] = data['test']
def wrap_and_indent_lines(self, lines, indent):
assert(len(lines) > 0)
output = indent + u"\u25B6 %s\n" % lines[0]
for line in lines[1:-1]:
output += indent + u"\u2502 %s\n" % line
if len(lines) > 1:
output += indent + u"\u2514 %s\n" % lines[-1]
return output
def get_lines_for_unexpected_result(self,
test_name,
status,
expected,
message,
stack):
# Test names sometimes contain control characters, which we want
# to be printed in their raw form, and not their interpreted form.
test_name = test_name.encode('unicode-escape')
if expected:
expected_text = f" [expected {expected}]"
else:
expected_text = u""
lines = [f"{status}{expected_text} {test_name}"]
if message:
for message_line in message.splitlines():
lines.append(f" \u2192 {message_line}")
if stack:
lines.append("")
lines.extend(stack.splitlines())
return lines
def get_output_for_unexpected_subtests(self, test_name, unexpected_subtests):
if not unexpected_subtests:
return ""
def add_subtest_failure(lines, subtest, stack=None):
lines += self.get_lines_for_unexpected_result(
subtest.get('subtest', None),
subtest.get('status', None),
subtest.get('expected', None),
subtest.get('message', None),
stack)
def make_subtests_failure(test_name, subtests, stack=None):
lines = [u"Unexpected subtest result in %s:" % test_name]
for subtest in subtests[:-1]:
add_subtest_failure(lines, subtest, None)
add_subtest_failure(lines, subtests[-1], stack)
return self.wrap_and_indent_lines(lines, " ")
# Organize the failures by stack trace so we don't print the same stack trace
# more than once. They are really tall and we don't want to flood the screen
# with duplicate information.
output = ""
failures_by_stack = collections.defaultdict(list)
for failure in unexpected_subtests:
# Print stackless results first. They are all separate.
if 'stack' not in failure:
output += make_subtests_failure(test_name, [failure], None)
else:
failures_by_stack[failure['stack']].append(failure)
for (stack, failures) in iteritems(failures_by_stack):
output += make_subtests_failure(test_name, failures, stack)
return output
def test_end(self, data):
self.completed_tests += 1
test_status = data["status"]
test_name = data["test"]
had_unexpected_test_result = "expected" in data
subtest_failures = self.subtest_failures.pop(test_name, [])
del self.running_tests[data['thread']]
if not had_unexpected_test_result and not subtest_failures:
self.expected[test_status] += 1
return None
# If the test crashed or timed out, we also include any process output,
# because there is a good chance that the test produced a stack trace
# or other error messages.
if test_status in ("CRASH", "TIMEOUT"):
stack = self.test_output[test_name] + data.get('stack', "")
else:
stack = data.get('stack', None)
output = ""
if had_unexpected_test_result:
self.unexpected_tests[test_status].append(data)
lines = self.get_lines_for_unexpected_result(
test_name,
test_status,
data.get('expected', None),
data.get('message', None),
stack)
output += self.wrap_and_indent_lines(lines, " ")
if subtest_failures:
self.tests_with_failing_subtests.append(test_name)
output += self.get_output_for_unexpected_subtests(test_name,
subtest_failures)
self.unexpected_results.append(
UnexpectedResult(test_name, test_status, output))
return output
def test_status(self, data):
if "expected" in data:
self.subtest_failures[data["test"]].append(data)
def process_output(self, data):
if data['thread'] not in self.running_tests:
return
test_name = self.running_tests[data['thread']]
self.test_output[test_name] += data['data'] + "\n"
def log(self, _):
pass
class ServoFormatter(mozlog.formatters.base.BaseFormatter, ServoHandler):
"""Formatter designed to produce unexpected test results grouped
together in a readable format."""
def __init__(self):
ServoHandler.__init__(self)
self.current_display = ""
self.interactive = os.isatty(sys.stdout.fileno())
if self.interactive:
self.line_width = os.get_terminal_size().columns
self.move_up = DEFAULT_MOVE_UP_CODE
self.clear_eol = DEFAULT_CLEAR_EOL_CODE
try:
import blessings
self.terminal = blessings.Terminal()
self.move_up = self.terminal.move_up
self.clear_eol = self.terminal.clear_eol
except Exception as exception:
sys.stderr.write("GroupingFormatter: Could not get terminal "
"control characters: %s\n" % exception)
def text_to_erase_display(self):
if not self.interactive or not self.current_display:
@ -80,7 +221,7 @@ class ServoFormatter(base.BaseFormatter):
return ((self.move_up + self.clear_eol)
* self.current_display.count('\n'))
def generate_output(self, text=None, new_display=None, unexpected_in_test=None):
def generate_output(self, text=None, new_display=None):
if not self.interactive:
return text
@ -112,148 +253,42 @@ class ServoFormatter(base.BaseFormatter):
return new_display + "No tests running.\n"
def suite_start(self, data):
self.number_of_tests = sum(len(tests) for tests in itervalues(data["tests"]))
self.start_time = data["time"]
ServoHandler.suite_start(self, data)
if self.number_of_tests == 0:
return "Running tests in %s\n\n" % data[u'source']
else:
return "Running %i tests in %s\n\n" % (self.number_of_tests, data[u'source'])
def test_start(self, data):
self.running_tests[data['thread']] = data['test']
ServoHandler.test_start(self, data)
if self.interactive:
return self.generate_output(new_display=self.build_status_line())
def wrap_and_indent_lines(self, lines, indent):
assert(len(lines) > 0)
output = indent + u"\u25B6 %s\n" % lines[0]
for line in lines[1:-1]:
output += indent + u"\u2502 %s\n" % line
if len(lines) > 1:
output += indent + u"\u2514 %s\n" % lines[-1]
return output
def get_lines_for_unexpected_result(self,
test_name,
status,
expected,
message,
stack):
# Test names sometimes contain control characters, which we want
# to be printed in their raw form, and not their interpreted form.
test_name = test_name.encode('unicode-escape')
if expected:
expected_text = u" [expected %s]" % expected
else:
expected_text = u""
lines = [u"%s%s %s" % (status, expected_text, test_name)]
if message:
for message_line in message.splitlines():
lines.append(u" \u2192 %s" % message_line)
if stack:
lines.append("")
lines.extend(stack.splitlines())
return lines
def get_output_for_unexpected_subtests(self, test_name, unexpected_subtests):
if not unexpected_subtests:
return ""
def add_subtest_failure(lines, subtest, stack=None):
lines += self.get_lines_for_unexpected_result(
subtest.get('subtest', None),
subtest.get('status', None),
subtest.get('expected', None),
subtest.get('message', None),
stack)
def make_subtests_failure(test_name, subtests, stack=None):
lines = [u"Unexpected subtest result in %s:" % test_name]
for subtest in subtests[:-1]:
add_subtest_failure(lines, subtest, None)
add_subtest_failure(lines, subtests[-1], stack)
return self.wrap_and_indent_lines(lines, " ") + "\n"
# Organize the failures by stack trace so we don't print the same stack trace
# more than once. They are really tall and we don't want to flood the screen
# with duplicate information.
output = ""
failures_by_stack = collections.defaultdict(list)
for failure in unexpected_subtests:
# Print stackless results first. They are all separate.
if 'stack' not in failure:
output += make_subtests_failure(test_name, [failure], None)
else:
failures_by_stack[failure['stack']].append(failure)
for (stack, failures) in iteritems(failures_by_stack):
output += make_subtests_failure(test_name, failures, stack)
return output
def test_end(self, data):
self.completed_tests += 1
test_status = data["status"]
test_name = data["test"]
had_unexpected_test_result = "expected" in data
subtest_failures = self.subtest_failures.pop(test_name, [])
del self.running_tests[data['thread']]
if not had_unexpected_test_result and not subtest_failures:
self.expected[test_status] += 1
output_for_unexpected_test = ServoHandler.test_end(self, data)
if not output_for_unexpected_test:
if self.interactive:
new_display = self.build_status_line()
return self.generate_output(new_display=new_display)
return self.generate_output(new_display=self.build_status_line())
else:
return self.generate_output(text="%s%s\n" % (self.test_counter(), test_name))
return self.generate_output(text="%s%s\n" % (self.test_counter(), data["test"]))
# If the test crashed or timed out, we also include any process output,
# because there is a good chance that the test produced a stack trace
# or other error messages.
if test_status in ("CRASH", "TIMEOUT"):
stack = self.test_output[test_name] + data.get('stack', "")
else:
stack = data.get('stack', None)
output = ""
if had_unexpected_test_result:
self.unexpected_tests[test_status].append(data)
lines = self.get_lines_for_unexpected_result(
test_name,
test_status,
data.get('expected', None),
data.get('message', None),
stack)
output += self.wrap_and_indent_lines(lines, " ") + "\n"
if subtest_failures:
self.tests_with_failing_subtests.append(test_name)
output += self.get_output_for_unexpected_subtests(test_name,
subtest_failures)
self.test_failure_text += output
new_display = self.build_status_line()
return self.generate_output(text=output, new_display=new_display,
unexpected_in_test=test_name)
# Surround test output by newlines so that it is easier to read.
output_for_unexpected_test = f"{output_for_unexpected_test}\n"
return self.generate_output(text=output_for_unexpected_test,
new_display=self.build_status_line())
def test_status(self, data):
if "expected" in data:
self.subtest_failures[data["test"]].append(data)
ServoHandler.test_status(self, data)
def suite_end(self, data):
self.end_time = data["time"]
ServoHandler.suite_end(self, data)
if not self.interactive:
output = u"\n"
else:
output = ""
output += u"Ran %i tests finished in %.1f seconds.\n" % (
self.completed_tests, (self.end_time - self.start_time) / 1000)
self.completed_tests, (data["time"] - self.suite_start_time) / 1000)
output += u" \u2022 %i ran as expected. %i tests skipped.\n" % (
sum(self.expected.values()), self.expected['SKIP'])
@ -279,18 +314,18 @@ class ServoFormatter(base.BaseFormatter):
# Repeat failing test output, so that it is easier to find, since the
# non-interactive version prints all the test names.
if not self.interactive and self.test_failure_text:
output += u"Tests with unexpected results:\n" + self.test_failure_text
if not self.interactive and self.unexpected_results:
output += u"Tests with unexpected results:\n"
output += "".join([result.output for result in self.unexpected_results])
return self.generate_output(text=output, new_display="")
def process_output(self, data):
if data['thread'] not in self.running_tests:
return
test_name = self.running_tests[data['thread']]
self.test_output[test_name] += data['data'] + "\n"
ServoHandler.process_output(self, data)
def log(self, data):
ServoHandler.log(self, data)
# We are logging messages that begin with STDERR, because that is how exceptions
# in this formatter are indicated.
if data['message'].startswith('STDERR'):
@ -298,16 +333,3 @@ class ServoFormatter(base.BaseFormatter):
if data['level'] in ('CRITICAL', 'ERROR'):
return self.generate_output(text=data['message'] + "\n")
class ServoJsonFormatter(ServoFormatter):
def suite_start(self, data):
ServoFormatter.suite_start(self, data)
# Don't forward the return value
def generate_output(self, text=None, new_display=None, unexpected_in_test=None):
if unexpected_in_test:
return "%s\n" % json.dumps({"test": unexpected_in_test, "output": text})
def log(self, _):
return

114
tests/wpt/servowpt.py
View file

@ -2,21 +2,32 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import grouping_formatter
import json
import os
import sys
import urllib.parse
import urllib.request
import grouping_formatter
import mozlog
import mozlog.formatters
import multiprocessing
from typing import List
from grouping_formatter import UnexpectedResult
SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
SERVO_ROOT = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
WPT_TOOLS_PATH = os.path.join(SCRIPT_PATH, "web-platform-tests", "tools")
CERTS_PATH = os.path.join(WPT_TOOLS_PATH, "certs")
sys.path.insert(0, WPT_TOOLS_PATH)
import update # noqa: F401,E402
import localpaths # noqa: F401,E402
import update # noqa: F401,E402
TRACKER_API = "https://build.servo.org/intermittent-tracker"
TRACKER_API_ENV_VAR = "INTERMITTENT_TRACKER_API"
GITHUB_API_TOKEN_ENV_VAR = "INTERMITTENT_TRACKER_GITHUB_API_TOKEN"
def determine_build_type(kwargs: dict, target_dir: str):
@ -109,6 +120,8 @@ def run_tests(**kwargs):
product = kwargs.get("product") or "servo"
kwargs["test_types"] = test_types[product]
filter_intermittents_output = kwargs.pop("filter_intermittents", None)
wptcommandline.check_args(kwargs)
update_args_for_layout_2020(kwargs)
@ -116,10 +129,6 @@ def run_tests(**kwargs):
grouping_formatter.ServoFormatter,
"Servo's grouping output formatter",
)
mozlog.commandline.log_formatters["servojson"] = (
grouping_formatter.ServoJsonFormatter,
"Servo's JSON logger of unexpected results",
)
use_mach_logging = False
if len(kwargs["test_list"]) == 1:
@ -128,12 +137,22 @@ def run_tests(**kwargs):
use_mach_logging = True
if use_mach_logging:
wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
logger = wptrunner.setup_logging(kwargs, {"mach": sys.stdout})
else:
wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
logger = wptrunner.setup_logging(kwargs, {"servo": sys.stdout})
success = wptrunner.run_tests(**kwargs)
return 0 if success else 1
handler = grouping_formatter.ServoHandler()
logger.add_handler(handler)
wptrunner.run_tests(**kwargs)
if handler.unexpected_results and filter_intermittents_output:
all_filtered = filter_intermittents(
handler.unexpected_results,
filter_intermittents_output,
)
return 0 if all_filtered else 1
else:
return 0 if not handler.unexpected_results else 1
def update_tests(**kwargs):
@ -150,6 +169,81 @@ def update_tests(**kwargs):
return 1 if return_value is update.exit_unclean else 0
class TrackerFilter():
def __init__(self):
self.url = os.environ.get(TRACKER_API_ENV_VAR, TRACKER_API)
if self.url.endswith("/"):
self.url = self.url[0:-1]
def is_failure_intermittent(self, test_name):
query = urllib.parse.quote(test_name, safe='')
request = urllib.request.Request("%s/query.py?name=%s" % (self.url, query))
search = urllib.request.urlopen(request)
return len(json.load(search)) > 0
class GitHubQueryFilter():
def __init__(self, token):
self.token = token
def is_failure_intermittent(self, test_name):
url = "https://api.github.com/search/issues?q="
query = "repo:servo/servo+" + \
"label:I-intermittent+" + \
"type:issue+" + \
"state:open+" + \
test_name
# we want `/` to get quoted, but not `+` (github's API doesn't like
# that), so we set `safe` to `+`
url += urllib.parse.quote(query, safe="+")
request = urllib.request.Request(url)
request.add_header("Authorization", f"Bearer: {self.token}")
request.add_header("Accept", "application/vnd.github+json")
return json.load(
urllib.request.urlopen(request)
)["total_count"] > 0
def filter_intermittents(
unexpected_results: List[UnexpectedResult],
output_file: str
) -> bool:
print(80 * "=")
print(f"Filtering {len(unexpected_results)} unexpected "
"results for known intermittents")
if GITHUB_API_TOKEN_ENV_VAR in os.environ:
filter = GitHubQueryFilter(os.environ.get(GITHUB_API_TOKEN_ENV_VAR))
else:
filter = TrackerFilter()
intermittents = []
actually_unexpected = []
for i, result in enumerate(unexpected_results):
print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
if filter.is_failure_intermittent(result.test_name):
intermittents.append(result)
else:
actually_unexpected.append(result)
output = "\n".join([
f"{len(intermittents)} known-intermittent unexpected result",
*[result.output.strip() for result in intermittents],
"",
f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents",
*[result.output.strip() for result in actually_unexpected],
])
if output_file:
with open(output_file, "w", encoding="utf-8") as file:
file.write(output)
print(output)
print(80 * "=")
return not actually_unexpected
def main():
from wptrunner import wptcommandline