Add support for the intermittent dashboard

Use the new intermittent dashboard to report intermittents and get
information about open bugs. This is now used to filter out
known-intermittents from results. In addition, this also allows the
scripts to report bug information to the GitHub. Display that in all
output.

etc/ci/report_aggregated_expected_results.py

@@ -41,6 +41,13 @@ class Item:
             title = f"{actual} [expected {expected}] {title_prefix}{title}"
         else:
             title = f"{actual} {title_prefix}{title}"
+
+        issue_url = "http://github.com/servo/servo/issues/"
+        if "issues" in result and result["issues"]:
+            issues = ", ".join([f"[#{issue}]({issue_url}{issue})"
+                                for issue in result["issues"]])
+            title += f" ({issues})"
+
         stack = result["stack"] if result["stack"] and print_stack else ""
         body = f"{result['message']}\n{stack}".strip()
 

tests/wpt/grouping_formatter.py

@@ -41,24 +41,21 @@ class UnexpectedResult():
     stack: Optional[str]
     unexpected_subtest_results: list[UnexpectedSubtestResult] = field(
         default_factory=list)
+    issues: list[str] = field(default_factory=list)
 
     def __str__(self):
-        output = ""
+        output = UnexpectedResult.to_lines(self)
-        if self.expected != self.actual:
-            lines = UnexpectedResult.to_lines(self)
-            output += UnexpectedResult.wrap_and_indent_lines(lines, "  ")
 
         if self.unexpected_subtest_results:
-            def make_subtests_failure(result, subtest_results):
+            def make_subtests_failure(subtest_results):
                 # Test names sometimes contain control characters, which we want
                 # to be printed in their raw form, and not their interpreted form.
-                path = result.path.encode('unicode-escape')
+                lines = []
-                lines = [f"Unexpected subtest result in {path}:"]
                 for subtest in subtest_results[:-1]:
                     lines += UnexpectedResult.to_lines(
                         subtest, print_stack=False)
                 lines += UnexpectedResult.to_lines(subtest_results[-1])
-                return self.wrap_and_indent_lines(lines, "  ")
+                return self.wrap_and_indent_lines(lines, "  ").splitlines()
 
             # Organize the failures by stack trace so we don't print the same stack trace
             # more than once. They are really tall and we don't want to flood the screen
@@ -69,11 +66,11 @@ class UnexpectedResult():
 
             # Print stackless results first. They are all separate.
             if None in results_by_stack:
-                output = make_subtests_failure(
+                output += make_subtests_failure(results_by_stack.pop(None))
-                    self, results_by_stack.pop(None))
             for subtest_results in results_by_stack.values():
-                output += make_subtests_failure(self, subtest_results)
+                output += make_subtests_failure(subtest_results)
-        return output
+
+        return UnexpectedResult.wrap_and_indent_lines(output, "  ")
 
     @staticmethod
     def wrap_and_indent_lines(lines, indent):
@@ -89,15 +86,18 @@ class UnexpectedResult():
 
     @staticmethod
     def to_lines(result: Any[UnexpectedSubtestResult, UnexpectedResult], print_stack=True):
+        first_line = result.actual
         if result.expected != result.actual:
-            expected_text = f" [expected {result.expected}]"
+            first_line += f" [expected {result.expected}]"
-        else:
-            expected_text = u""
 
         # Test names sometimes contain control characters, which we want
         # to be printed in their raw form, and not their interpreted form.
-        path = result.path.encode('unicode-escape')
+        first_line += f" {result.path.encode('unicode-escape').decode('utf-8')}"
-        lines = [f"{result.actual}{expected_text} {path}"]
+
+        if isinstance(result, UnexpectedResult) and result.issues:
+            first_line += f" ({', '.join([f'#{bug}' for bug in result.issues])})"
+
+        lines = [first_line]
         if result.message:
             for message_line in result.message.splitlines():
                 lines.append(f"  \u2192 {message_line}")

tests/wpt/servowpt.py

@@ -6,7 +6,9 @@ import dataclasses
 import grouping_formatter
 import json
 import os
+import re
 import sys
+import urllib.error
 import urllib.parse
 import urllib.request
 
@@ -14,8 +16,8 @@ import mozlog
 import mozlog.formatters
 import multiprocessing
 
-from typing import List
+from typing import List, NamedTuple, Optional, Tuple, Union
-from grouping_formatter import UnexpectedResult
+from grouping_formatter import UnexpectedResult, UnexpectedSubtestResult
 
 SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
 SERVO_ROOT = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
@@ -28,7 +30,7 @@ import update  # noqa: F401,E402
 
 TRACKER_API = "https://build.servo.org/intermittent-tracker"
 TRACKER_API_ENV_VAR = "INTERMITTENT_TRACKER_API"
-GITHUB_API_TOKEN_ENV_VAR = "INTERMITTENT_TRACKER_GITHUB_API_TOKEN"
+TRACKER_DASHBOARD_SECRET_ENV_VAR = "INTERMITTENT_TRACKER_DASHBOARD_SECRET"
 
 
 def determine_build_type(kwargs: dict, target_dir: str):
@@ -186,41 +188,106 @@ def update_tests(**kwargs):
     return 1 if return_value is update.exit_unclean else 0
 
 
-class TrackerFilter():
+class GithubContextInformation(NamedTuple):
+    build_url: Optional[str]
+    pull_url: Optional[str]
+    branch_name: Optional[str]
+
+
+class TrackerDashboardFilter():
     def __init__(self):
-        self.url = os.environ.get(TRACKER_API_ENV_VAR, TRACKER_API)
+        base_url = os.environ.get(TRACKER_API_ENV_VAR, TRACKER_API)
-        if self.url.endswith("/"):
+        self.headers = {
-            self.url = self.url[0:-1]
+            "Content-Type": "application/json"
+        }
+        if TRACKER_DASHBOARD_SECRET_ENV_VAR in os.environ:
+            self.url = f"{base_url}/dashboard/attempts"
+            secret = os.environ[TRACKER_DASHBOARD_SECRET_ENV_VAR]
+            self.headers["Authorization"] = f"Bearer {secret}"
+        else:
+            self.url = f"{base_url}/dashboard/query"
 
-    def is_failure_intermittent(self, test_name):
+    @staticmethod
-        query = urllib.parse.quote(test_name, safe='')
+    def get_github_context_information() -> GithubContextInformation:
-        request = urllib.request.Request("%s/query.py?name=%s" % (self.url, query))
+        github_context = json.loads(os.environ.get("GITHUB_CONTEXT", "{}"))
-        search = urllib.request.urlopen(request)
+        if not github_context:
-        return len(json.load(search)) > 0
+            return GithubContextInformation(None, None, None)
 
+        repository = github_context['repository']
+        repo_url = f"https://github.com/{repository}"
 
-class GitHubQueryFilter():
+        run_id = github_context['run_id']
-    def __init__(self, token):
+        build_url = f"{repo_url}/actions/runs/{run_id})"
-        self.token = token
 
-    def is_failure_intermittent(self, test_name):
+        commit_title = github_context["event"]["head_commit"]["message"]
-        url = "https://api.github.com/search/issues?q="
+        match = re.match(r"^Auto merge of #(\d+)", commit_title)
-        query = "repo:servo/servo+" + \
+        pr_url = f"{repo_url}/pull/{match.group(1)}" if match else None
-            "label:I-intermittent+" + \
-            "type:issue+" + \
-            "state:open+" + \
-            test_name
 
-        # we want `/` to get quoted, but not `+` (github's API doesn't like
+        return GithubContextInformation(
-        # that), so we set `safe` to `+`
+            build_url,
-        url += urllib.parse.quote(query, safe="+")
+            pr_url,
+            github_context["ref_name"]
+        )
 
-        request = urllib.request.Request(url)
+    def make_data_from_result(
-        request.add_header("Authorization", f"Bearer: {self.token}")
+        self,
-        request.add_header("Accept", "application/vnd.github+json")
+        result: Union[UnexpectedResult, UnexpectedSubtestResult],
-        return json.load(
+    ) -> dict:
-            urllib.request.urlopen(request)
+        data = {
-        )["total_count"] > 0
+            'path': result.path,
+            'subtest': None,
+            'expected': result.expected,
+            'actual': result.actual,
+            'time': result.time // 1000,
+            'message': result.message,
+            'stack': result.stack,
+        }
+        if isinstance(result, UnexpectedSubtestResult):
+            data["subtest"] = result.subtest
+        return data
+
+    def filter_unexpected_results(
+        self,
+        unexpected_results: List[UnexpectedResult]
+    ) -> Tuple[List[UnexpectedResult], List[UnexpectedResult]]:
+        attempts = []
+        for result in unexpected_results:
+            attempts.append(self.make_data_from_result(result))
+            for subtest_result in result.unexpected_subtest_results:
+                attempts.append(self.make_data_from_result(subtest_result))
+
+        context = self.get_github_context_information()
+        try:
+            request = urllib.request.Request(
+                url=self.url,
+                method='POST',
+                data=json.dumps({
+                    'branch': context.branch_name,
+                    'build_url': context.build_url,
+                    'pull_url': context.pull_url,
+                    'attempts': attempts
+                }).encode('utf-8'),
+                headers=self.headers)
+
+            known_intermittents = dict()
+            with urllib.request.urlopen(request) as response:
+                for test in json.load(response)["known"]:
+                    known_intermittents[test["path"]] = \
+                        [issue["number"] for issue in test["issues"]]
+
+        except urllib.error.HTTPError as e:
+            print(e)
+            print(e.readlines())
+            raise(e)
+
+        known = [result for result in unexpected_results
+                 if result.path in known_intermittents]
+        unknown = [result for result in unexpected_results
+                   if result.path not in known_intermittents]
+        for result in known:
+            result.issues = known_intermittents[result.path]
+
+        return (known, unknown)
 
 
 def filter_intermittents(
@@ -230,19 +297,10 @@ def filter_intermittents(
     print(80 * "=")
     print(f"Filtering {len(unexpected_results)} unexpected "
           "results for known intermittents")
-    if GITHUB_API_TOKEN_ENV_VAR in os.environ:
-        filter = GitHubQueryFilter(os.environ.get(GITHUB_API_TOKEN_ENV_VAR))
-    else:
-        filter = TrackerFilter()
 
-    known_intermittents: List[UnexpectedResult] = []
+    filter = TrackerDashboardFilter()
-    unexpected: List[UnexpectedResult] = []
+    (known_intermittents, unexpected) = \
-    for i, result in enumerate(unexpected_results):
+        filter.filter_unexpected_results(unexpected_results)
-        print(f" [{i}/{len(unexpected_results)}]", file=sys.stderr, end="\r")
-        if filter.is_failure_intermittent(result.path):
-            known_intermittents.append(result)
-        else:
-            unexpected.append(result)
 
     output = "\n".join([
         f"{len(known_intermittents)} known-intermittent unexpected result",