From c0f85579c5fabeed46f149cceaf1713a7b992086 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Thu, 18 May 2017 17:19:40 -0400 Subject: [PATCH 1/3] Allow specifying alternative tracker endpoints. --- etc/ci/buildbot_steps.yml | 18 +++++++++--------- python/servo/testing_commands.py | 15 ++++++++++----- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 497ba972762..3036f2ed928 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -3,7 +3,7 @@ mac-rel-wpt1: - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt-failure - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -12,22 +12,22 @@ mac-rel-wpt2: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build-geckolib --release mac-rel-wpt3: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 3 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default mac-rel-wpt4: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default - ./mach test-wpt --release --pref dom.servoparser.async_html_tokenizer.enabled --processes=8 --log-raw test-async-parsing.log --log-errorsummary async-parsing-errorsummary.log --always-succeed domparsing html/syntax html/dom/documents html/dom/dynamic-markup-insertion - - ./mach filter-intermittents async-parsing-errorsummary.log --log-intermittents async-parsing-intermittents.log --log-filteredsummary filtered-async-parsing-errorsummary.log --use-tracker + - ./mach filter-intermittents async-parsing-errorsummary.log --log-intermittents async-parsing-intermittents.log --log-filteredsummary filtered-async-parsing-errorsummary.log --tracker-api default mac-dev-unit: - ./mach clean-nightlies --keep 3 --force @@ -43,7 +43,7 @@ mac-rel-css1: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 5 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -51,7 +51,7 @@ mac-rel-css2: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 6 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default mac-nightly: - ./mach clean-nightlies --keep 3 --force @@ -95,14 +95,14 @@ linux-rel-wpt: - ./mach build --release --with-debug-assertions - ./mach test-wpt-failure - ./mach test-wpt --release --processes 24 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource linux-rel-css: - ./mach clean-nightlies --keep 3 --force - ./mach build --release --with-debug-assertions - ./mach test-wpt --release --processes 24 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default - ./mach build-geckolib --release - ./mach test-stylo --release - bash ./etc/ci/lockfile_changed.sh diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index c21def01243..3e744afdfec 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -517,9 +517,9 @@ class MachCommands(CommandBase): help='Print intermittents to file') @CommandArgument('--auth', default=None, help='File containing basic authorization credentials for Github API (format `username:password`)') - @CommandArgument('--use-tracker', default=False, action='store_true', - help='Use https://www.joshmatthews.net/intermittent-tracker') - def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, auth, use_tracker): + @CommandArgument('--tracker-api', default=None, action='store', + help='The API endpoint for tracking known intermittent failures.') + def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, auth, tracker_api): encoded_auth = None if auth: with open(auth, "r") as file: @@ -533,9 +533,14 @@ class MachCommands(CommandBase): actual_failures = [] intermittents = [] for failure in failures: - if use_tracker: + if tracker_api: + if tracker_api == 'default': + tracker_api = "http://build.servo.org/intermittent-tracker" + else if tracker_api.endswith('/'): + tracker_api = tracker_api[0:-1] + query = urllib2.quote(failure['test'], safe='') - request = urllib2.Request("http://build.servo.org/intermittent-tracker/query.py?name=%s" % query) + request = urllib2.Request("%s/query.py?name=%s" % (tracker_api, query)) search = urllib2.urlopen(request) data = json.load(search) if len(data) == 0: From c469cbbe72ecd3bebb1604f8a8781ce38bd62c6d Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Thu, 18 May 2017 17:40:47 -0400 Subject: [PATCH 2/3] Report instances of intermittent failures to a tracker. --- python/servo/testing_commands.py | 40 ++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 3e744afdfec..92ad8793bb8 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -14,13 +14,16 @@ import re import sys import os import os.path as path +import platform import copy from collections import OrderedDict from time import time import json import urllib2 +import urllib import base64 import shutil +import subprocess from mach.registrar import Registrar from mach.decorators import ( @@ -519,7 +522,9 @@ class MachCommands(CommandBase): help='File containing basic authorization credentials for Github API (format `username:password`)') @CommandArgument('--tracker-api', default=None, action='store', help='The API endpoint for tracking known intermittent failures.') - def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, auth, tracker_api): + @CommandArgument('--reporter-api', default=None, action='store', + help='The API endpoint for reporting tracked intermittent failures.') + def filter_intermittents(self, summary, log_filteredsummary, log_intermittents, auth, tracker_api, reporter_api): encoded_auth = None if auth: with open(auth, "r") as file: @@ -536,7 +541,7 @@ class MachCommands(CommandBase): if tracker_api: if tracker_api == 'default': tracker_api = "http://build.servo.org/intermittent-tracker" - else if tracker_api.endswith('/'): + elif tracker_api.endswith('/'): tracker_api = tracker_api[0:-1] query = urllib2.quote(failure['test'], safe='') @@ -561,6 +566,37 @@ class MachCommands(CommandBase): else: intermittents += [failure] + if reporter_api: + if reporter_api.endswith('/'): + reporter_api = reporter_api[0:-1] + reported = set() + + proc = subprocess.Popen( + ["git", "log", "--merges", "--oneline", "-1"], + stdout=subprocess.PIPE) + (last_merge, _) = proc.communicate() + + # Extract the issue reference from "abcdef Auto merge of #NNN" + pull_request = int(last_merge.split(' ')[4][1:]) + + for intermittent in intermittents: + if intermittent['test'] in reported: + continue + reported.add(intermittent['test']) + + data = { + 'test_file': intermittent['test'], + 'platform': platform.system(), + 'builder': os.environ.get('BUILDER_NAME', 'BUILDER NAME MISSING'), + 'number': pull_request, + } + request = urllib2.Request("%s/record.py" % reporter_api, urllib.urlencode(data)) + request.add_header('Accept', 'application/json') + response = urllib2.urlopen(request) + data = json.load(response) + if data['status'] != "success": + print('Error reporting test failure: ' + data['error']) + if log_intermittents: with open(log_intermittents, "w") as intermittents_file: for intermittent in intermittents: From 86e3f0deefbe5131633ca446dcb5f8c58af9b601 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Wed, 18 Oct 2017 10:42:54 -0400 Subject: [PATCH 3/3] Use failure reporting API on CI. --- etc/ci/buildbot_steps.yml | 18 +++++++++--------- python/servo/testing_commands.py | 2 ++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml index 3036f2ed928..9b7117329bc 100644 --- a/etc/ci/buildbot_steps.yml +++ b/etc/ci/buildbot_steps.yml @@ -3,7 +3,7 @@ mac-rel-wpt1: - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt-failure - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default - ./mach test-wpt --release --binary-arg=--multiprocess --processes 8 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -12,22 +12,22 @@ mac-rel-wpt2: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build-geckolib --release mac-rel-wpt3: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 3 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default mac-rel-wpt4: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 4 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default - ./mach test-wpt --release --pref dom.servoparser.async_html_tokenizer.enabled --processes=8 --log-raw test-async-parsing.log --log-errorsummary async-parsing-errorsummary.log --always-succeed domparsing html/syntax html/dom/documents html/dom/dynamic-markup-insertion - - ./mach filter-intermittents async-parsing-errorsummary.log --log-intermittents async-parsing-intermittents.log --log-filteredsummary filtered-async-parsing-errorsummary.log --tracker-api default + - ./mach filter-intermittents async-parsing-errorsummary.log --log-intermittents async-parsing-intermittents.log --log-filteredsummary filtered-async-parsing-errorsummary.log --tracker-api default --reporter-api default mac-dev-unit: - ./mach clean-nightlies --keep 3 --force @@ -43,7 +43,7 @@ mac-rel-css1: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 5 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default - bash ./etc/ci/lockfile_changed.sh - bash ./etc/ci/manifest_changed.sh @@ -51,7 +51,7 @@ mac-rel-css2: - ./mach clean-nightlies --keep 3 --force - env PKG_CONFIG_PATH=/usr/local/opt/zlib/lib/pkgconfig ./mach build --release - ./mach test-wpt --release --processes 4 --total-chunks 6 --this-chunk 6 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default mac-nightly: - ./mach clean-nightlies --keep 3 --force @@ -95,14 +95,14 @@ linux-rel-wpt: - ./mach build --release --with-debug-assertions - ./mach test-wpt-failure - ./mach test-wpt --release --processes 24 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource linux-rel-css: - ./mach clean-nightlies --keep 3 --force - ./mach build --release --with-debug-assertions - ./mach test-wpt --release --processes 24 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed - - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default + - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --tracker-api default --reporter-api default - ./mach build-geckolib --release - ./mach test-stylo --release - bash ./etc/ci/lockfile_changed.sh diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 92ad8793bb8..d09c456e2f3 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -567,6 +567,8 @@ class MachCommands(CommandBase): intermittents += [failure] if reporter_api: + if reporter_api == 'default': + reporter_api = "http://build.servo.org/intermittent-failure-tracker" if reporter_api.endswith('/'): reporter_api = reporter_api[0:-1] reported = set()