Auto merge of #14331 - Manishearth:filter-intermittents, r=larsbergstrom,jdm

Add ./mach filter-intermittents for catching intermittents on CI

cc @metajack

The plan here is to run this on the error summary of the logs after each WPT/CSS run, as `./mach filter-intermittents wpt-errorsummary.log --output filtered-errorsummary.log --auth /path/to/authfile`

The `filtered-errorsummary.log` file will be a buildbot artifact for this run, and can be used for updating test results.

We change WPT/CSS runs to not cause test failures on Buildbot; instead; the test failure will be caused by this job.

We should at some point add a separate highfive task which ccs the appropriate bugs and tracks consistent failures. (We can change mach filter-intermittents to output a second file of intermittents matched to bug numbers to make this easy)

A small issue with this is that this simple task might mask failures of the WPT harness itself. We have a separate test that tests if the test harness works though. (This test will fail if the log files it requires don't exist, perhaps that's enough to solve the problem)

r? @larsbergstrom

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/14331)
<!-- Reviewable:end -->

etc/ci/buildbot_steps.yml

@@ -1,7 +1,8 @@
 mac-rel-wpt1:
   - ./mach build --release
   - ./mach test-wpt-failure
-  - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log
+  - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
+  - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
   - ./mach build-cef --release
   - bash ./etc/ci/lockfile_changed.sh
@@ -9,7 +10,8 @@ mac-rel-wpt1:
 
 mac-rel-wpt2:
   - ./mach build --release
-  - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log
+  - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
+  - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
 
 mac-dev-unit:
   - ./mach build --dev
@@ -21,7 +23,8 @@ mac-dev-unit:
 
 mac-rel-css:
   - ./mach build --release
-  - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log
+  - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
+  - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
   - ./mach build-geckolib --release
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
@@ -57,12 +60,14 @@ linux-dev:
 linux-rel-wpt:
   - ./mach build --release --with-debug-assertions
   - ./mach test-wpt-failure
-  - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log
+  - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
+  - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
 
 linux-rel-css:
   - ./mach build --release --with-debug-assertions
-  - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log
+  - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
+  - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
   - ./mach build-cef --release --with-debug-assertions
   - ./mach build-geckolib --release
   - ./mach test-stylo --release

python/servo/testing_commands.py

@@ -17,6 +17,9 @@ import os.path as path
 import copy
 from collections import OrderedDict
 from time import time
+import json
+import urllib2
+import base64
 
 from mach.registrar import Registrar
 from mach.decorators import (
@@ -65,6 +68,8 @@ def create_parser_wpt():
                         help="Run under chaos mode in rr until a failure is captured")
     parser.add_argument('--pref', default=[], action="append", dest="prefs",
                         help="Pass preferences to servo")
+    parser.add_argument('--always-succeed', default=False, action="store_true",
+                        help="Always yield exit code of zero")
     return parser
 
 
@@ -396,7 +401,11 @@ class MachCommands(CommandBase):
              parser=create_parser_wpt)
     def test_wpt(self, **kwargs):
         self.ensure_bootstrapped()
-        return self.run_test_list_or_dispatch(kwargs["test_list"], "wpt", self._test_wpt, **kwargs)
+        ret = self.run_test_list_or_dispatch(kwargs["test_list"], "wpt", self._test_wpt, **kwargs)
+        if kwargs["always_succeed"]:
+            return 0
+        else:
+            return ret
 
     def _test_wpt(self, **kwargs):
         hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts')
@@ -470,6 +479,61 @@ class MachCommands(CommandBase):
         execfile(run_file, run_globals)
         return run_globals["update_tests"](**kwargs)
 
+    @Command('filter-intermittents',
+             description='Given a WPT error summary file, filter out intermittents and other cruft.',
+             category='testing')
+    @CommandArgument('summary',
+                     help="Error summary log to take un")
+    @CommandArgument('--log-filteredsummary', default=None,
+                     help='Print filtered log to file')
+    @CommandArgument('--auth', default=None,
+                     help='File containing basic authorization credentials for Github API (format `username:password`)')
+    @CommandArgument('--use-tracker', default=False, action='store_true',
+                     help='Use https://www.joshmatthews.net/intermittent-tracker')
+    def filter_intermittents(self, summary, log_filteredsummary, auth, use_tracker):
+        encoded_auth = None
+        if auth:
+            with open(auth, "r") as file:
+                encoded_auth = base64.encodestring(file.read().strip()).replace('\n', '')
+        failures = []
+        with open(summary, "r") as file:
+            for line in file:
+                line_json = json.loads(line)
+                if 'status' in line_json:
+                    failures += [line_json]
+        actual_failures = []
+        for failure in failures:
+            if use_tracker:
+                query = urllib2.quote(failure['test'], safe='')
+                request = urllib2.Request("https://www.joshmatthews.net/intermittent-tracker/query.py?name=%s" % query)
+                search = urllib2.urlopen(request)
+                data = json.load(search)
+                if len(data) == 0:
+                    actual_failures += [failure]
+            else:
+                qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test']
+                # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+`
+                query = urllib2.quote(qstr, safe='+')
+                request = urllib2.Request("https://api.github.com/search/issues?q=%s" % query)
+                if encoded_auth:
+                    request.add_header("Authorization", "Basic %s" % encoded_auth)
+                search = urllib2.urlopen(request)
+                data = json.load(search)
+                if data['total_count'] == 0:
+                    actual_failures += [failure]
+
+        if len(actual_failures) == 0:
+            return 0
+
+        output = open(log_filteredsummary, "w") if log_filteredsummary else sys.stdout
+        for failure in actual_failures:
+            json.dump(failure, output)
+            print("\n", end='', file=output)
+
+        if output is not sys.stdout:
+            output.close()
+        return 1
+
     @Command('test-jquery',
              description='Run the jQuery test suite',
              category='testing')
@@ -508,7 +572,11 @@ class MachCommands(CommandBase):
              parser=create_parser_wpt)
     def test_css(self, **kwargs):
         self.ensure_bootstrapped()
-        return self.run_test_list_or_dispatch(kwargs["test_list"], "css", self._test_css, **kwargs)
+        ret = self.run_test_list_or_dispatch(kwargs["test_list"], "css", self._test_css, **kwargs)
+        if kwargs["always_succeed"]:
+            return 0
+        else:
+            return ret
 
     def _test_css(self, **kwargs):
         run_file = path.abspath(path.join("tests", "wpt", "run_css.py"))