mirror of
https://github.com/servo/servo.git
synced 2025-06-06 16:45:39 +00:00
Output test results as a GitHub comment
After filtering intermittents, output the results as JSON. Update the GitHub workflow to aggregate this JSON data into an artifact and use the aggregated data to generate a GitHub comment with details about the try run. The idea here is that this comment will make it easier to track intermittent tests and notice when a change affects a test marked as intermittent -- either causing it to permanently fail or fixing it.
This commit is contained in:
parent
3429e8fe3b
commit
d2a66fef0c
3 changed files with 192 additions and 21 deletions
|
@ -2,6 +2,7 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import dataclasses
|
||||
import grouping_formatter
|
||||
import json
|
||||
import os
|
||||
|
@ -218,30 +219,34 @@ def filter_intermittents(
|
|||
else:
|
||||
filter = TrackerFilter()
|
||||
|
||||
intermittents = []
|
||||
actually_unexpected = []
|
||||
known_intermittents: List[UnexpectedResult] = []
|
||||
unexpected: List[UnexpectedResult] = []
|
||||
for i, result in enumerate(unexpected_results):
|
||||
print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
|
||||
if filter.is_failure_intermittent(result.path):
|
||||
intermittents.append(result)
|
||||
known_intermittents.append(result)
|
||||
else:
|
||||
actually_unexpected.append(result)
|
||||
|
||||
output = "\n".join([
|
||||
f"{len(intermittents)} known-intermittent unexpected result",
|
||||
*[str(result) for result in intermittents],
|
||||
"",
|
||||
f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents",
|
||||
*[str(result) for result in actually_unexpected],
|
||||
])
|
||||
unexpected.append(result)
|
||||
|
||||
if output_file:
|
||||
with open(output_file, "w", encoding="utf-8") as file:
|
||||
file.write(output)
|
||||
file.write(json.dumps({
|
||||
"known_intermittents":
|
||||
[dataclasses.asdict(result) for result in known_intermittents],
|
||||
"unexpected":
|
||||
[dataclasses.asdict(result) for result in unexpected],
|
||||
}))
|
||||
|
||||
output = "\n".join([
|
||||
f"{len(known_intermittents)} known-intermittent unexpected result",
|
||||
*[str(result) for result in known_intermittents],
|
||||
"",
|
||||
f"{len(unexpected)} unexpected results that are NOT known-intermittents",
|
||||
*[str(result) for result in unexpected],
|
||||
])
|
||||
print(output)
|
||||
print(80 * "=")
|
||||
return not actually_unexpected
|
||||
return not unexpected
|
||||
|
||||
|
||||
def main():
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue