From c23ea67af7b4684164ca44b2f26f4086b023645e Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Mon, 28 Nov 2016 16:46:33 -0800 Subject: [PATCH] Add a log of only intermittents --- etc/ci/buildbot_steps.yml | 10 +++++----- python/servo/testing_commands.py | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 2921115c91b..69353c304ea 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -2,7 +2,7 @@ mac-rel-wpt1: - ./mach build --release - ./mach test-wpt-failure - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource - ./mach build-cef --release - bash ./etc/ci/lockfile_changed.sh @@ -11,7 +11,7 @@ mac-rel-wpt1: mac-rel-wpt2: - ./mach build --release - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker mac-dev-unit: - ./mach build --dev @@ -24,7 +24,7 @@ mac-dev-unit: mac-rel-css: - ./mach build --release - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker + - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker - ./mach build-geckolib --release - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -61,13 +61,13 @@ linux-rel-wpt: - ./mach build --release --with-debug-assertions - ./mach test-wpt-failure - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource linux-rel-css: - ./mach build --release --with-debug-assertions - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed - - ./mach filter-intermittents css-errorsummary.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker + - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker - ./mach build-cef --release --with-debug-assertions - ./mach build-geckolib --release - ./mach test-stylo --release diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index f0486feb232..feeaed9b6ae 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -486,11 +486,13 @@ class MachCommands(CommandBase): help="Error summary log to take un") @CommandArgument('--log-filteredsummary', default=None, help='Print filtered log to file') + @CommandArgument('--log-intermittents', default=None, + help='Print intermittents to file') @CommandArgument('--auth', default=None, help='File containing basic authorization credentials for Github API (format `username:password`)') @CommandArgument('--use-tracker', default=False, action='store_true', help='Use https://www.joshmatthews.net/intermittent-tracker') - def filter_intermittents(self, summary, log_filteredsummary, auth, use_tracker): + def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, auth, use_tracker): encoded_auth = None if auth: with open(auth, "r") as file: @@ -502,6 +504,7 @@ class MachCommands(CommandBase): if 'status' in line_json: failures += [line_json] actual_failures = [] + intermittents = [] for failure in failures: if use_tracker: query = urllib2.quote(failure['test'], safe='') @@ -510,6 +513,8 @@ class MachCommands(CommandBase): data = json.load(search) if len(data) == 0: actual_failures += [failure] + else: + intermittents += [failure] else: qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test'] # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+` @@ -521,6 +526,14 @@ class MachCommands(CommandBase): data = json.load(search) if data['total_count'] == 0: actual_failures += [failure] + else: + intermittents += [failure] + + if log_intermittents: + with open(log_intermittents, "w") as intermittents_file: + for intermittent in intermittents: + json.dump(intermittent, intermittents_file) + print("\n", end='', file=intermittents_file) if len(actual_failures) == 0: return 0