mirror of
https://github.com/servo/servo.git
synced 2025-08-12 00:45:33 +01:00
Auto merge of #29315 - mrobinson:isolate-intermittents-2, r=delan
Output test results as GitHub comments After filtering intermittents, output the results as JSON. Update the GitHub workflow to aggregate this JSON data into an artifact and use the aggregated data to generate a GitHub comment with details about the try run. The idea here is that this comment will make it easier to track intermittent tests and notice when a change affects a test marked as intermittent -- either causing it to permanently fail or fixing it. --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: --> - [x] `./mach build -d` does not report any errors - [x] `./mach test-tidy` does not report any errors - [x] These changes do not require tests because they modify the CI infrastructure. <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->
This commit is contained in:
commit
f04a466be7
3 changed files with 192 additions and 21 deletions
|
@ -2,6 +2,7 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
import dataclasses
|
||||
import grouping_formatter
|
||||
import json
|
||||
import os
|
||||
|
@ -218,30 +219,34 @@ def filter_intermittents(
|
|||
else:
|
||||
filter = TrackerFilter()
|
||||
|
||||
intermittents = []
|
||||
actually_unexpected = []
|
||||
known_intermittents: List[UnexpectedResult] = []
|
||||
unexpected: List[UnexpectedResult] = []
|
||||
for i, result in enumerate(unexpected_results):
|
||||
print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
|
||||
if filter.is_failure_intermittent(result.path):
|
||||
intermittents.append(result)
|
||||
known_intermittents.append(result)
|
||||
else:
|
||||
actually_unexpected.append(result)
|
||||
|
||||
output = "\n".join([
|
||||
f"{len(intermittents)} known-intermittent unexpected result",
|
||||
*[str(result) for result in intermittents],
|
||||
"",
|
||||
f"{len(actually_unexpected)} unexpected results that are NOT known-intermittents",
|
||||
*[str(result) for result in actually_unexpected],
|
||||
])
|
||||
unexpected.append(result)
|
||||
|
||||
if output_file:
|
||||
with open(output_file, "w", encoding="utf-8") as file:
|
||||
file.write(output)
|
||||
file.write(json.dumps({
|
||||
"known_intermittents":
|
||||
[dataclasses.asdict(result) for result in known_intermittents],
|
||||
"unexpected":
|
||||
[dataclasses.asdict(result) for result in unexpected],
|
||||
}))
|
||||
|
||||
output = "\n".join([
|
||||
f"{len(known_intermittents)} known-intermittent unexpected result",
|
||||
*[str(result) for result in known_intermittents],
|
||||
"",
|
||||
f"{len(unexpected)} unexpected results that are NOT known-intermittents",
|
||||
*[str(result) for result in unexpected],
|
||||
])
|
||||
print(output)
|
||||
print(80 * "=")
|
||||
return not actually_unexpected
|
||||
return not unexpected
|
||||
|
||||
|
||||
def main():
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue