Auto merge of #29346 - mrobinson:aggregate-unexpected-logs, r=delan

Aggregate unexpected results into logs

This makes it easier to run `update-wpt` based on results from the bots by writing a new version of the raw log that filters out all tests with expected results.  Then aggregate this output into the filtered results archive. A future version of this could aggregate all unexpected results that were not filtered as intermittents.

<!-- Please describe your changes on the following line: -->

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes do not require tests because they are infrastructure changes.

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

.github/workflows/main.yml

@@ -216,14 +216,16 @@ jobs:
             --release --processes $(nproc) --timeout-multiplier 2 \
             --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \
             --log-raw test-wpt.${{ matrix.chunk_id }}.log \
-            --filter-intermittents=filtered-wpt-results.${{ matrix.chunk_id }}.json
+            --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \
+            --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json
       - name: Archive filtered results
         uses: actions/upload-artifact@v3
         if: ${{ always() }}
         with:
           name: wpt-filtered-results-linux
           path: |
-            filtered-wpt-results.${{ matrix.chunk_id }}.json
+            filtered-test-wpt.${{ matrix.chunk_id }}.json
+            unexpected-test-wpt.${{ matrix.chunk_id }}.log
       - name: Archive logs
         uses: actions/upload-artifact@v3
         if: ${{ failure() }}
@@ -247,8 +249,16 @@ jobs:
         with:
           name: wpt-filtered-results-linux
           path: wpt-filtered-results-linux
+      - name: Create aggregated unexpected results
+        run: cat wpt-filtered-results-linux/*.log > unexpected-test-wpt.log
+      - name: Archive aggregate results
+        uses: actions/upload-artifact@v3
+        with:
+          name: wpt-filtered-results-linux
+          path: |
+            unexpected-test-wpt.log
       - name: Comment on PR with results
-        run: etc/ci/report_aggregated_expected_results.py wpt-filtered-results-linux/*
+        run: etc/ci/report_aggregated_expected_results.py wpt-filtered-results-linux/*.json
         env:
           GITHUB_CONTEXT: ${{ toJson(github) }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

python/servo/testing_commands.py

@@ -87,6 +87,9 @@ def create_parser_wpt():
     parser.add_argument('--filter-intermittents', default=None, action="store",
                         help="Filter intermittents against known intermittents "
                              "and save the filtered output to the given file.")
+    parser.add_argument('--log-raw-unexpected', default=None, action="store",
+                        help="Raw structured log messages for unexpected results."
+                             " '--log-raw' Must also be passed in order to use this.")
     return parser
 
 

tests/wpt/servowpt.py

@@ -122,6 +122,8 @@ def run_tests(**kwargs):
         kwargs["test_types"] = test_types[product]
 
     filter_intermittents_output = kwargs.pop("filter_intermittents", None)
+    unexpected_raw_log_output_file = kwargs.pop("log_raw_unexpected", None)
+    raw_log_outputs = kwargs.get("log_raw", [])
 
     wptcommandline.check_args(kwargs)
     update_args_for_layout_2020(kwargs)
@@ -146,14 +148,28 @@ def run_tests(**kwargs):
     logger.add_handler(handler)
 
     wptrunner.run_tests(**kwargs)
+    return_value = 0 if not handler.unexpected_results else 1
+
+    # Filter intermittents if that was specified on the command-line.
     if handler.unexpected_results and filter_intermittents_output:
         all_filtered = filter_intermittents(
             handler.unexpected_results,
-            filter_intermittents_output,
+            filter_intermittents_output
         )
-        return 0 if all_filtered else 1
+        return_value = 0 if all_filtered else 1
-    else:
+
-        return 0 if not handler.unexpected_results else 1
+    # Write the unexpected-only raw log if that was specified on the command-line.
+    if unexpected_raw_log_output_file:
+        if not raw_log_outputs:
+            print("'--log-raw-unexpected' not written without '--log-raw'.")
+        else:
+            write_unexpected_only_raw_log(
+                handler.unexpected_results,
+                raw_log_outputs[0].name,
+                unexpected_raw_log_output_file
+            )
+
+    return return_value
 
 
 def update_tests(**kwargs):
@@ -228,15 +244,6 @@ def filter_intermittents(
         else:
             unexpected.append(result)
 
-    if output_file:
-        with open(output_file, "w", encoding="utf-8") as file:
-            file.write(json.dumps({
-                "known_intermittents":
-                    [dataclasses.asdict(result) for result in known_intermittents],
-                "unexpected":
-                    [dataclasses.asdict(result) for result in unexpected],
-            }))
-
     output = "\n".join([
         f"{len(known_intermittents)} known-intermittent unexpected result",
         *[str(result) for result in known_intermittents],
@@ -246,9 +253,37 @@ def filter_intermittents(
     ])
     print(output)
     print(80 * "=")
+
+    if output_file:
+        print(f"Writing filtered results to {output_file}")
+        with open(output_file, "w", encoding="utf-8") as file:
+            file.write(json.dumps({
+                "known_intermittents":
+                    [dataclasses.asdict(result) for result in known_intermittents],
+                "unexpected":
+                    [dataclasses.asdict(result) for result in unexpected],
+            }))
+
     return not unexpected
 
 
+def write_unexpected_only_raw_log(
+    unexpected_results: List[UnexpectedResult],
+    raw_log_file: str,
+    filtered_raw_log_file: str
+):
+    tests = [result.path for result in unexpected_results]
+    print(f"Writing unexpected-only raw log to {filtered_raw_log_file}")
+
+    with open(filtered_raw_log_file, "w", encoding="utf-8") as output:
+        with open(raw_log_file) as input:
+            for line in input.readlines():
+                data = json.loads(line)
+                if data["action"] in ["suite_start", "suite_end"] or \
+                        ("test" in data and data["test"] in tests):
+                    output.write(line)
+
+
 def main():
     from wptrunner import wptcommandline