From 4b4421c365d37d3a848860511498b893a06b89df Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Tue, 22 Nov 2016 17:14:11 -0800 Subject: [PATCH 1/4] Add ./mach filter-intermittents for catching intermittents on CI --- python/servo/testing_commands.py | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index f24750df245..dffc38c9646 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -17,6 +17,9 @@ import os.path as path import copy from collections import OrderedDict from time import time +import json +import urllib2 +import base64 from mach.registrar import Registrar from mach.decorators import ( @@ -470,6 +473,51 @@ class MachCommands(CommandBase): execfile(run_file, run_globals) return run_globals["update_tests"](**kwargs) + @Command('filter-intermittents', + description='Given a WPT error summary file, filter out intermittents and other cruft.', + category='testing') + @CommandArgument('summary', + help="Error summary log to take un") + @CommandArgument('--output', default=None, + help='Print filtered log to file') + @CommandArgument('--auth', default=None, + help='File containing basic authorization credentials for Github API (format `username:password`)') + def filter_intermittents(self, summary, output, auth): + encoded_auth = None + if auth: + with open(auth, "r") as file: + encoded_auth = base64.encodestring(file.read().strip()).replace('\n', '') + failures = [] + with open(summary, "r") as file: + for line in file: + line_json = json.loads(line) + if 'status' in line_json: + failures += [line_json] + actual_failures = [] + for failure in failures: + qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test'] + # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+` + query = urllib2.quote(qstr, safe='+') + request = urllib2.Request("https://api.github.com/search/issues?q=%s" % query) + if encoded_auth: + request.add_header("Authorization", "Basic %s" % encoded_auth) + search = urllib2.urlopen(request) + data = json.load(search) + if data['total_count'] == 0: + actual_failures += [failure] + + if len(actual_failures) == 0: + return 0 + + output = open(output, "w") if output else sys.stdout + for failure in actual_failures: + json.dump(failure, output) + print("\n", end='', file=output) + + if output is not sys.stdout: + output.close() + return 1 + @Command('test-jquery', description='Run the jQuery test suite', category='testing') From 4f021d3c532f1fd5c3cb3ae25cd6e5d58852c3c0 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 28 Nov 2016 10:15:19 -0800 Subject: [PATCH 2/4] Run filter-intermittents on buildbot --- etc/ci/buildbot_steps.yml | 12 ++++++++---- python/servo/testing_commands.py | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 19d67c02f25..4a944e5808d 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -1,7 +1,8 @@ mac-rel-wpt1: - ./mach build --release - ./mach test-wpt-failure - - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log + - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed + - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource - ./mach build-cef --release - bash ./etc/ci/lockfile_changed.sh @@ -21,7 +22,8 @@ mac-dev-unit: mac-rel-css: - ./mach build --release - - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log + - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed + - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log - ./mach build-geckolib --release - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -57,12 +59,14 @@ linux-dev: linux-rel-wpt: - ./mach build --release --with-debug-assertions - ./mach test-wpt-failure - - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log + - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed + - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource linux-rel-css: - ./mach build --release --with-debug-assertions - - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log + - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed + - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log - ./mach build-cef --release --with-debug-assertions - ./mach build-geckolib --release - ./mach test-stylo --release diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index dffc38c9646..1db3b4fb1bf 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -68,6 +68,8 @@ def create_parser_wpt(): help="Run under chaos mode in rr until a failure is captured") parser.add_argument('--pref', default=[], action="append", dest="prefs", help="Pass preferences to servo") + parser.add_argument('--always-succeed', default=False, action="store_true", + help="Always yield exit code of zero") return parser @@ -399,7 +401,11 @@ class MachCommands(CommandBase): parser=create_parser_wpt) def test_wpt(self, **kwargs): self.ensure_bootstrapped() - return self.run_test_list_or_dispatch(kwargs["test_list"], "wpt", self._test_wpt, **kwargs) + ret = self.run_test_list_or_dispatch(kwargs["test_list"], "wpt", self._test_wpt, **kwargs) + if kwargs["always_succeed"]: + return 0 + else: + return ret def _test_wpt(self, **kwargs): hosts_file_path = path.join(self.context.topdir, 'tests', 'wpt', 'hosts') @@ -556,7 +562,11 @@ class MachCommands(CommandBase): parser=create_parser_wpt) def test_css(self, **kwargs): self.ensure_bootstrapped() - return self.run_test_list_or_dispatch(kwargs["test_list"], "css", self._test_css, **kwargs) + ret = self.run_test_list_or_dispatch(kwargs["test_list"], "css", self._test_css, **kwargs) + if kwargs["always_succeed"]: + return 0 + else: + return ret def _test_css(self, **kwargs): run_file = path.abspath(path.join("tests", "wpt", "run_css.py")) From c0c65ea63797405f2fee48c58284933ae2c9b347 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 28 Nov 2016 13:29:10 -0800 Subject: [PATCH 3/4] Use jdm's tracker --- etc/ci/buildbot_steps.yml | 11 ++++++----- python/servo/testing_commands.py | 32 +++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 4a944e5808d..8d0a1117696 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -2,7 +2,7 @@ mac-rel-wpt1: - ./mach build --release - ./mach test-wpt-failure - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log + - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log --use-tracker - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource - ./mach build-cef --release - bash ./etc/ci/lockfile_changed.sh @@ -10,7 +10,8 @@ mac-rel-wpt1: mac-rel-wpt2: - ./mach build --release - - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log + - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed + - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log --use-tracker mac-dev-unit: - ./mach build --dev @@ -23,7 +24,7 @@ mac-dev-unit: mac-rel-css: - ./mach build --release - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log + - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log --use-tracker - ./mach build-geckolib --release - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -60,13 +61,13 @@ linux-rel-wpt: - ./mach build --release --with-debug-assertions - ./mach test-wpt-failure - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log + - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log --use-tracker - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource linux-rel-css: - ./mach build --release --with-debug-assertions - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log + - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log --use-tracker - ./mach build-cef --release --with-debug-assertions - ./mach build-geckolib --release - ./mach test-stylo --release diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 1db3b4fb1bf..6b48158137c 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -488,7 +488,9 @@ class MachCommands(CommandBase): help='Print filtered log to file') @CommandArgument('--auth', default=None, help='File containing basic authorization credentials for Github API (format `username:password`)') - def filter_intermittents(self, summary, output, auth): + @CommandArgument('--use-tracker', default=False, action='store_true', + help='Use https://www.joshmatthews.net/intermittent-tracker') + def filter_intermittents(self, summary, output, auth, use_tracker): encoded_auth = None if auth: with open(auth, "r") as file: @@ -501,16 +503,24 @@ class MachCommands(CommandBase): failures += [line_json] actual_failures = [] for failure in failures: - qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test'] - # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+` - query = urllib2.quote(qstr, safe='+') - request = urllib2.Request("https://api.github.com/search/issues?q=%s" % query) - if encoded_auth: - request.add_header("Authorization", "Basic %s" % encoded_auth) - search = urllib2.urlopen(request) - data = json.load(search) - if data['total_count'] == 0: - actual_failures += [failure] + if use_tracker: + query = urllib2.quote(failure['test'], safe='') + request = urllib2.Request("https://www.joshmatthews.net/intermittent-tracker/query.py?name=%s" % query) + search = urllib2.urlopen(request) + data = json.load(search) + if len(data) == 0: + actual_failures += [failure] + else: + qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test'] + # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+` + query = urllib2.quote(qstr, safe='+') + request = urllib2.Request("https://api.github.com/search/issues?q=%s" % query) + if encoded_auth: + request.add_header("Authorization", "Basic %s" % encoded_auth) + search = urllib2.urlopen(request) + data = json.load(search) + if data['total_count'] == 0: + actual_failures += [failure] if len(actual_failures) == 0: return 0 From f42ce6de92eb0b2764e18d586ce5810a47e0e515 Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 28 Nov 2016 14:28:30 -0800 Subject: [PATCH 4/4] Use log- prefix for filtered log arg so that buildbot picks it up and uploads as an artefact --- etc/ci/buildbot_steps.yml | 10 +++++----- python/servo/testing_commands.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 8d0a1117696..2921115c91b 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -2,7 +2,7 @@ mac-rel-wpt1: - ./mach build --release - ./mach test-wpt-failure - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource - ./mach build-cef --release - bash ./etc/ci/lockfile_changed.sh @@ -11,7 +11,7 @@ mac-rel-wpt1: mac-rel-wpt2: - ./mach build --release - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker mac-dev-unit: - ./mach build --dev @@ -24,7 +24,7 @@ mac-dev-unit: mac-rel-css: - ./mach build --release - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log --use-tracker + - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker - ./mach build-geckolib --release - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -61,13 +61,13 @@ linux-rel-wpt: - ./mach build --release --with-debug-assertions - ./mach test-wpt-failure - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --output filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource linux-rel-css: - ./mach build --release --with-debug-assertions - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - - ./mach filter-intermittents css-errorsummary.log --output filtered-css-errorsummary.log --use-tracker + - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker - ./mach build-cef --release --with-debug-assertions - ./mach build-geckolib --release - ./mach test-stylo --release diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 6b48158137c..f0486feb232 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -484,13 +484,13 @@ class MachCommands(CommandBase): category='testing') @CommandArgument('summary', help="Error summary log to take un") - @CommandArgument('--output', default=None, + @CommandArgument('--log-filteredsummary', default=None, help='Print filtered log to file') @CommandArgument('--auth', default=None, help='File containing basic authorization credentials for Github API (format `username:password`)') @CommandArgument('--use-tracker', default=False, action='store_true', help='Use https://www.joshmatthews.net/intermittent-tracker') - def filter_intermittents(self, summary, output, auth, use_tracker): + def filter_intermittents(self, summary, log_filteredsummary, auth, use_tracker): encoded_auth = None if auth: with open(auth, "r") as file: @@ -525,7 +525,7 @@ class MachCommands(CommandBase): if len(actual_failures) == 0: return 0 - output = open(output, "w") if output else sys.stdout + output = open(log_filteredsummary, "w") if log_filteredsummary else sys.stdout for failure in actual_failures: json.dump(failure, output) print("\n", end='', file=output)