Auto merge of #29315 - mrobinson:isolate-intermittents-2, r=delan

Output test results as GitHub comments

After filtering intermittents, output the results as JSON. Update the
GitHub workflow to aggregate this JSON data into an artifact and use the
aggregated data to generate a GitHub comment with details about the try
run. The idea here is that this comment will make it easier to track
intermittent tests and notice when a change affects a test marked as
intermittent -- either causing it to permanently fail or fixing it.

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes do not require tests because they modify the CI infrastructure.

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->
This commit is contained in:
bors-servo 2023-02-07 09:31:55 +01:00 committed by GitHub
commit f04a466be7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 192 additions and 21 deletions

View file

@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import dataclasses
import grouping_formatter
import json
import os
@ -218,30 +219,34 @@ def filter_intermittents(
else:
filter = TrackerFilter()
intermittents = []
actually_unexpected = []
known_intermittents: List[UnexpectedResult] = []
unexpected: List[UnexpectedResult] = []
for i, result in enumerate(unexpected_results):
print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
if filter.is_failure_intermittent(result.path):
intermittents.append(result)
known_intermittents.append(result)
else:
actually_unexpected.append(result)
output = "\n".join([
f"{len(intermittents)} known-intermittent unexpected result",
*[str(result) for result in intermittents],
"",
f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents",
*[str(result) for result in actually_unexpected],
])
unexpected.append(result)
if output_file:
with open(output_file, "w", encoding="utf-8") as file:
file.write(output)
file.write(json.dumps({
"known_intermittents":
[dataclasses.asdict(result) for result in known_intermittents],
"unexpected":
[dataclasses.asdict(result) for result in unexpected],
}))
output = "\n".join([
f"{len(known_intermittents)} known-intermittent unexpected result",
*[str(result) for result in known_intermittents],
"",
f"{len(unexpected)} unexpected results that are NOT known-intermittents",
*[str(result) for result in unexpected],
])
print(output)
print(80 * "=")
return not actually_unexpected
return not unexpected
def main():