Output test results as a GitHub comment

After filtering intermittents, output the results as JSON. Update the
GitHub workflow to aggregate this JSON data into an artifact and use the
aggregated data to generate a GitHub comment with details about the try
run. The idea here is that this comment will make it easier to track
intermittent tests and notice when a change affects a test marked as
intermittent -- either causing it to permanently fail or fixing it.
This commit is contained in:
Martin Robinson 2023-01-31 19:17:18 +01:00
parent 3429e8fe3b
commit d2a66fef0c
3 changed files with 192 additions and 21 deletions

View file

@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import dataclasses
import grouping_formatter
import json
import os
@ -218,30 +219,34 @@ def filter_intermittents(
else:
filter = TrackerFilter()
intermittents = []
actually_unexpected = []
known_intermittents: List[UnexpectedResult] = []
unexpected: List[UnexpectedResult] = []
for i, result in enumerate(unexpected_results):
print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
if filter.is_failure_intermittent(result.path):
intermittents.append(result)
known_intermittents.append(result)
else:
actually_unexpected.append(result)
output = "\n".join([
f"{len(intermittents)} known-intermittent unexpected result",
*[str(result) for result in intermittents],
"",
f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents",
*[str(result) for result in actually_unexpected],
])
unexpected.append(result)
if output_file:
with open(output_file, "w", encoding="utf-8") as file:
file.write(output)
file.write(json.dumps({
"known_intermittents":
[dataclasses.asdict(result) for result in known_intermittents],
"unexpected":
[dataclasses.asdict(result) for result in unexpected],
}))
output = "\n".join([
f"{len(known_intermittents)} known-intermittent unexpected result",
*[str(result) for result in known_intermittents],
"",
f"{len(unexpected)} unexpected results that are NOT known-intermittents",
*[str(result) for result in unexpected],
])
print(output)
print(80 * "=")
return not actually_unexpected
return not unexpected
def main():