From 4a77ef03c8a6576e55aee11e464b7cde6bc3c101 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:42:37 -0500 Subject: [PATCH 01/17] Added tp5n prefix to URLs in manifest. --- .../page_load_test/tp5n/20160509.manifest | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/etc/ci/performance/page_load_test/tp5n/20160509.manifest b/etc/ci/performance/page_load_test/tp5n/20160509.manifest index 5dcbdace620..f4cf7b30188 100644 --- a/etc/ci/performance/page_load_test/tp5n/20160509.manifest +++ b/etc/ci/performance/page_load_test/tp5n/20160509.manifest @@ -1,52 +1,52 @@ http://localhost:8000/tp6/news.ycombinator.com/index.html -# http://localhost:8000/page_load_test/163.com/www.163.com/index.html -http://localhost:8000/page_load_test/56.com/www.56.com/index.html -# http://localhost:8000/page_load_test/aljazeera.net/aljazeera.net/portal.html -http://localhost:8000/page_load_test/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html -http://localhost:8000/page_load_test/bbc.co.uk/www.bbc.co.uk/news/index.html -http://localhost:8000/page_load_test/beatonna.livejournal.com/beatonna.livejournal.com/index.html -# http://localhost:8000/page_load_test/bild.de/www.bild.de/index.html -http://localhost:8000/page_load_test/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html -http://localhost:8000/page_load_test/chemistry.about.com/chemistry.about.com/index.html -# http://localhost:8000/page_load_test/chinaz.com/chinaz.com/index.html -http://localhost:8000/page_load_test/cnn.com/www.cnn.com/index.html -http://localhost:8000/page_load_test/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html -http://localhost:8000/page_load_test/dailymotion.com/www.dailymotion.com/us.html -# http://localhost:8000/page_load_test/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html -http://localhost:8000/page_load_test/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html -http://localhost:8000/page_load_test/globo.com/www.globo.com/index.html -http://localhost:8000/page_load_test/google.com/www.google.com/search@q=mozilla.html -http://localhost:8000/page_load_test/goo.ne.jp/goo.ne.jp/index.html -# http://localhost:8000/page_load_test/guardian.co.uk/www.guardian.co.uk/index.html -# http://localhost:8000/page_load_test/homeway.com.cn/www.hexun.com/index.html -http://localhost:8000/page_load_test/huffingtonpost.com/www.huffingtonpost.com/index.html -# http://localhost:8000/page_load_test/ifeng.com/ifeng.com/index.html -# http://localhost:8000/page_load_test/imdb.com/www.imdb.com/title/tt1099212/index.html -http://localhost:8000/page_load_test/imgur.com/imgur.com/gallery/index.html -# http://localhost:8000/page_load_test/indiatimes.com/www.indiatimes.com/index.html -http://localhost:8000/page_load_test/mail.ru/mail.ru/index.html -# http://localhost:8000/page_load_test/mashable.com/mashable.com/index.html -http://localhost:8000/page_load_test/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html -http://localhost:8000/page_load_test/myspace.com/www.myspace.com/albumart.html -# http://localhost:8000/page_load_test/naver.com/www.naver.com/index.html -# http://localhost:8000/page_load_test/noimpactman.typepad.com/noimpactman.typepad.com/index.html -http://localhost:8000/page_load_test/page.renren.com/page.renren.com/index.html -# http://localhost:8000/page_load_test/people.com.cn/people.com.cn/index.html -http://localhost:8000/page_load_test/rakuten.co.jp/www.rakuten.co.jp/index.html -http://localhost:8000/page_load_test/reddit.com/www.reddit.com/index.html -# http://localhost:8000/page_load_test/reuters.com/www.reuters.com/index.html -# http://localhost:8000/page_load_test/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html -# http://localhost:8000/page_load_test/sohu.com/www.sohu.com/index.html -http://localhost:8000/page_load_test/spiegel.de/www.spiegel.de/index.html -http://localhost:8000/page_load_test/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html -# http://localhost:8000/page_load_test/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html -# http://localhost:8000/page_load_test/thepiratebay.org/thepiratebay.org/top/201.html -http://localhost:8000/page_load_test/tudou.com/www.tudou.com/index.html -# http://localhost:8000/page_load_test/uol.com.br/www.uol.com.br/index.html -http://localhost:8000/page_load_test/w3.org/www.w3.org/standards/webdesign/htmlcss.html -# http://localhost:8000/page_load_test/wsj.com/online.wsj.com/home-page.html -# http://localhost:8000/page_load_test/xinhuanet.com/xinhuanet.com/index.html -http://localhost:8000/page_load_test/xunlei.com/xunlei.com/index.html -http://localhost:8000/page_load_test/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html -http://localhost:8000/page_load_test/youku.com/www.youku.com/index.html -http://localhost:8000/page_load_test/youtube.com/www.youtube.com/music.html +# http://localhost:8000/page_load_test/tp5n/163.com/www.163.com/index.html +http://localhost:8000/page_load_test/tp5n/56.com/www.56.com/index.html +# http://localhost:8000/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html +http://localhost:8000/page_load_test/tp5n/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html +http://localhost:8000/page_load_test/tp5n/bbc.co.uk/www.bbc.co.uk/news/index.html +http://localhost:8000/page_load_test/tp5n/beatonna.livejournal.com/beatonna.livejournal.com/index.html +# http://localhost:8000/page_load_test/tp5n/bild.de/www.bild.de/index.html +http://localhost:8000/page_load_test/tp5n/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html +http://localhost:8000/page_load_test/tp5n/chemistry.about.com/chemistry.about.com/index.html +# http://localhost:8000/page_load_test/tp5n/chinaz.com/chinaz.com/index.html +http://localhost:8000/page_load_test/tp5n/cnn.com/www.cnn.com/index.html +http://localhost:8000/page_load_test/tp5n/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html +http://localhost:8000/page_load_test/tp5n/dailymotion.com/www.dailymotion.com/us.html +# http://localhost:8000/page_load_test/tp5n/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html +http://localhost:8000/page_load_test/tp5n/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html +http://localhost:8000/page_load_test/tp5n/globo.com/www.globo.com/index.html +http://localhost:8000/page_load_test/tp5n/google.com/www.google.com/search@q=mozilla.html +http://localhost:8000/page_load_test/tp5n/goo.ne.jp/goo.ne.jp/index.html +# http://localhost:8000/page_load_test/tp5n/guardian.co.uk/www.guardian.co.uk/index.html +# http://localhost:8000/page_load_test/tp5n/homeway.com.cn/www.hexun.com/index.html +http://localhost:8000/page_load_test/tp5n/huffingtonpost.com/www.huffingtonpost.com/index.html +# http://localhost:8000/page_load_test/tp5n/ifeng.com/ifeng.com/index.html +# http://localhost:8000/page_load_test/tp5n/imdb.com/www.imdb.com/title/tt1099212/index.html +http://localhost:8000/page_load_test/tp5n/imgur.com/imgur.com/gallery/index.html +# http://localhost:8000/page_load_test/tp5n/indiatimes.com/www.indiatimes.com/index.html +http://localhost:8000/page_load_test/tp5n/mail.ru/mail.ru/index.html +# http://localhost:8000/page_load_test/tp5n/mashable.com/mashable.com/index.html +http://localhost:8000/page_load_test/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html +http://localhost:8000/page_load_test/tp5n/myspace.com/www.myspace.com/albumart.html +# http://localhost:8000/page_load_test/tp5n/naver.com/www.naver.com/index.html +# http://localhost:8000/page_load_test/tp5n/noimpactman.typepad.com/noimpactman.typepad.com/index.html +http://localhost:8000/page_load_test/tp5n/page.renren.com/page.renren.com/index.html +# http://localhost:8000/page_load_test/tp5n/people.com.cn/people.com.cn/index.html +http://localhost:8000/page_load_test/tp5n/rakuten.co.jp/www.rakuten.co.jp/index.html +http://localhost:8000/page_load_test/tp5n/reddit.com/www.reddit.com/index.html +# http://localhost:8000/page_load_test/tp5n/reuters.com/www.reuters.com/index.html +# http://localhost:8000/page_load_test/tp5n/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html +# http://localhost:8000/page_load_test/tp5n/sohu.com/www.sohu.com/index.html +http://localhost:8000/page_load_test/tp5n/spiegel.de/www.spiegel.de/index.html +http://localhost:8000/page_load_test/tp5n/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html +# http://localhost:8000/page_load_test/tp5n/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html +# http://localhost:8000/page_load_test/tp5n/thepiratebay.org/thepiratebay.org/top/201.html +http://localhost:8000/page_load_test/tp5n/tudou.com/www.tudou.com/index.html +# http://localhost:8000/page_load_test/tp5n/uol.com.br/www.uol.com.br/index.html +http://localhost:8000/page_load_test/tp5n/w3.org/www.w3.org/standards/webdesign/htmlcss.html +# http://localhost:8000/page_load_test/tp5n/wsj.com/online.wsj.com/home-page.html +# http://localhost:8000/page_load_test/tp5n/xinhuanet.com/xinhuanet.com/index.html +http://localhost:8000/page_load_test/tp5n/xunlei.com/xunlei.com/index.html +http://localhost:8000/page_load_test/tp5n/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html +http://localhost:8000/page_load_test/tp5n/youku.com/www.youku.com/index.html +http://localhost:8000/page_load_test/tp5n/youtube.com/www.youtube.com/music.html From 53c0765bfbb1d5c94f0dadd86b1079ffe61154df Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:43:16 -0500 Subject: [PATCH 02/17] Diabled missing tp6 test from manifest. --- etc/ci/performance/page_load_test/tp5n/20160509.manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/performance/page_load_test/tp5n/20160509.manifest b/etc/ci/performance/page_load_test/tp5n/20160509.manifest index f4cf7b30188..52fe1c70ab3 100644 --- a/etc/ci/performance/page_load_test/tp5n/20160509.manifest +++ b/etc/ci/performance/page_load_test/tp5n/20160509.manifest @@ -1,4 +1,4 @@ -http://localhost:8000/tp6/news.ycombinator.com/index.html +# http://localhost:8000/tp6/news.ycombinator.com/index.html # http://localhost:8000/page_load_test/tp5n/163.com/www.163.com/index.html http://localhost:8000/page_load_test/tp5n/56.com/www.56.com/index.html # http://localhost:8000/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html From 1096351543a42047c009bf1aa50aa25905b4070c Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:43:49 -0500 Subject: [PATCH 03/17] Diabled never-terminating photobucket test from manifest. --- etc/ci/performance/page_load_test/tp5n/20160509.manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/performance/page_load_test/tp5n/20160509.manifest b/etc/ci/performance/page_load_test/tp5n/20160509.manifest index 52fe1c70ab3..bd3d485651f 100644 --- a/etc/ci/performance/page_load_test/tp5n/20160509.manifest +++ b/etc/ci/performance/page_load_test/tp5n/20160509.manifest @@ -26,7 +26,7 @@ http://localhost:8000/page_load_test/tp5n/imgur.com/imgur.com/gallery/index.html # http://localhost:8000/page_load_test/tp5n/indiatimes.com/www.indiatimes.com/index.html http://localhost:8000/page_load_test/tp5n/mail.ru/mail.ru/index.html # http://localhost:8000/page_load_test/tp5n/mashable.com/mashable.com/index.html -http://localhost:8000/page_load_test/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html +# http://localhost:8000/page_load_test/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html http://localhost:8000/page_load_test/tp5n/myspace.com/www.myspace.com/albumart.html # http://localhost:8000/page_load_test/tp5n/naver.com/www.naver.com/index.html # http://localhost:8000/page_load_test/tp5n/noimpactman.typepad.com/noimpactman.typepad.com/index.html From eb18665f5b8248e127398436208c28384b62fbcd Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:45:55 -0500 Subject: [PATCH 04/17] Removed shell option from subprocess. --- etc/ci/performance/runner.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index d34b9e49312..690a7445333 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -28,8 +28,7 @@ def execute_test(url, command, timeout): print(' '.join(command)) print("Timeout:{}".format(timeout)) try: - return subprocess.check_output(command, stderr=subprocess.STDOUT, - shell=True, timeout=timeout) + return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout) except subprocess.CalledProcessError as e: print("Unexpected Fail:") print(e) From a2a45a77fd2978ed4d8e5eb1994993c47a79a420 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:46:28 -0500 Subject: [PATCH 05/17] Removed extra space from --userscripts option. --- etc/ci/performance/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index 690a7445333..ccb6b146c9c 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -42,7 +42,7 @@ def execute_test(url, command, timeout): def get_servo_command(url): ua_script_path = "{}/user-agent-js".format(os.getcwd()) return ["./servo/servo", url, - " --userscripts", ua_script_path, + "--userscripts", ua_script_path, "-x", "-o", "output.png"] From 8e7837159eba806f9dcc545a771c4070b13e7298 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:47:00 -0500 Subject: [PATCH 06/17] Only include [PERF] lines in report. --- etc/ci/performance/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index ccb6b146c9c..c098a361612 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -66,7 +66,7 @@ def parse_log(log, testcase=None): copy = False blocks.append(block) block = [] - elif copy: + elif copy and line.strip().startswith("[PERF]"): block.append(line) # We need to still include the failed tests, otherwise Treeherder will From 900518ae55ab46862bb9b24daf979cc63a354602 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:49:05 -0500 Subject: [PATCH 07/17] Filter reports for ones with title Error report. --- etc/ci/performance/runner.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index c098a361612..45e5af7844a 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -112,21 +112,19 @@ def parse_log(log, testcase=None): print('-----') return placeholder - if key == "testcase": + if key == "testcase" or key == "title": timing[key] = value else: timing[key] = None if (value == "undefined") else int(value) - if testcase is not None and timing['testcase'] != testcase: - print('[DEBUG] log:') - print('-----') - print(log) - print('-----') - return placeholder - return timing - if len(blocks) == 0: + def valid_timing(timing): + return (timing.get('title') != 'Error response') and (testcase is None or timing.get('testcase') == testcase) + + timings = list(filter(valid_timing, map(parse_block, blocks))) + + if len(timings) == 0: print("Didn't find any perf data in the log, test timeout?") print("Fillng in a dummy perf data") print('[DEBUG] log:') @@ -136,7 +134,7 @@ def parse_log(log, testcase=None): return [placeholder] else: - return map(parse_block, blocks) + return timings def filter_result_by_manifest(result_json, manifest): From 08d15e94663f269b54cf30790258449107a1f528 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:50:21 -0500 Subject: [PATCH 08/17] Fixed formatting of timeout argument. --- etc/ci/performance/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index 45e5af7844a..7da6711db85 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -239,7 +239,7 @@ def main(): testcases = load_manifest(args.tp5_manifest) results = [] for testcase in testcases: - command = (["timeout", "{timeout}s".format(args.timeout)] + + command = (["timeout", "{}s".format(args.timeout)] + command_factory(testcase)) for run in range(args.runs): print("Running test {}/{} on {}".format(run + 1, From 2857dbabbea5b92ff175bc6b92a43da46125edfb Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 13:51:12 -0500 Subject: [PATCH 09/17] Include document title in report. --- etc/ci/performance/user-agent-js/01.perf-timing.js | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/ci/performance/user-agent-js/01.perf-timing.js b/etc/ci/performance/user-agent-js/01.perf-timing.js index df2e3efcd55..9aff2aa8a56 100644 --- a/etc/ci/performance/user-agent-js/01.perf-timing.js +++ b/etc/ci/performance/user-agent-js/01.perf-timing.js @@ -12,6 +12,7 @@ function formatLine(name, t) { function printPerfTiming() { print("[PERF] perf block start") formatLine("testcase", window.location); + formatLine("title", document.title.replace(/,/g, ",")); formatLine("navigationStart", performance.timing.navigationStart); formatLine("unloadEventStart", performance.timing.unloadEventStart); formatLine("unloadEventEnd", performance.timing.unloadEventEnd); From f81e32e3341c3c867a2392bd37a8cb51adaa618d Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 14:00:58 -0500 Subject: [PATCH 10/17] Added --local option to test_all.sh. --- etc/ci/performance/test_all.sh | 37 +++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/etc/ci/performance/test_all.sh b/etc/ci/performance/test_all.sh index 850a4dd1505..b82e06d71f9 100755 --- a/etc/ci/performance/test_all.sh +++ b/etc/ci/performance/test_all.sh @@ -8,19 +8,29 @@ set -o errexit set -o nounset set -o pipefail -case "${1}" in +while (( "$#" )) +do +case "$1" in --servo) engine="--engine servo" ;; --gecko) engine="--engine gecko" ;; + --local) + local=1 + ;; *) - # This branch should never be reached with set -o nounset - echo "You didn't specify the engine to run." + echo "Unknown option $1." exit ;; esac +shift +done + +if [ -z "${engine:-}" ]; +then echo "You didn't specify the engine to run: either --servo or --gecko."; exit; +fi echo "Starting the local server" python3 -m http.server > /dev/null 2>&1 & @@ -33,15 +43,18 @@ MANIFEST="page_load_test/tp5n/20160509.manifest" # A manifest that excludes PERF_FILE="output/perf-$(date --iso-8601=seconds).json" echo "Running tests" -python3 runner.py "${engine}" --runs 3 "${MANIFEST}" "${PERF_FILE}" +python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}" -echo "Submitting to Perfherder" -# Perfherder SSL check will fail if time is not accurate, -# sync time before you submit -# TODO: we are using Servo's revision hash for Gecko's result to make both -# results appear on the same date. Use the correct result when Perfherder -# allows us to change the date. -python3 submit_to_perfherder.py "${engine}" "${PERF_FILE}" servo/revision.json +if [ -z "${local:-}" ]; +then + echo "Submitting to Perfherder" + # Perfherder SSL check will fail if time is not accurate, + # sync time before you submit + # TODO: we are using Servo's revision hash for Gecko's result to make both + # results appear on the same date. Use the correct result when Perfherder + # allows us to change the date. + python3 submit_to_perfherder.py "${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json +fi -# Kill the http server +echo "Stopping the local server" trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT From 425b44f78e3b0f38cde2f3e85dfcf2cf188ef140 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 14:08:50 -0500 Subject: [PATCH 11/17] Made test-tidy happy. --- etc/ci/performance/test_all.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/etc/ci/performance/test_all.sh b/etc/ci/performance/test_all.sh index b82e06d71f9..e63fe6a529b 100755 --- a/etc/ci/performance/test_all.sh +++ b/etc/ci/performance/test_all.sh @@ -8,9 +8,9 @@ set -o errexit set -o nounset set -o pipefail -while (( "$#" )) +while (( "${#}" )) do -case "$1" in +case "${1}" in --servo) engine="--engine servo" ;; @@ -21,15 +21,15 @@ case "$1" in local=1 ;; *) - echo "Unknown option $1." + echo "Unknown option ${1}." exit ;; esac shift done -if [ -z "${engine:-}" ]; -then echo "You didn't specify the engine to run: either --servo or --gecko."; exit; +if [[ -z "${engine:-}" ]]; +then echo "You didn't specify the engine to run: --servo or --gecko."; exit; fi echo "Starting the local server" @@ -43,9 +43,9 @@ MANIFEST="page_load_test/tp5n/20160509.manifest" # A manifest that excludes PERF_FILE="output/perf-$(date --iso-8601=seconds).json" echo "Running tests" -python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}" +echo python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}" -if [ -z "${local:-}" ]; +if [[ -z "${local:-}" ]]; then echo "Submitting to Perfherder" # Perfherder SSL check will fail if time is not accurate, @@ -53,7 +53,8 @@ then # TODO: we are using Servo's revision hash for Gecko's result to make both # results appear on the same date. Use the correct result when Perfherder # allows us to change the date. - python3 submit_to_perfherder.py "${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json + echo python3 submit_to_perfherder.py \ + "${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json fi echo "Stopping the local server" From fb289812cf491faf5fd5bb9577efecfe10e7564f Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 16:17:13 -0500 Subject: [PATCH 12/17] Added test differ. --- etc/ci/performance/test_differ.py | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 etc/ci/performance/test_differ.py diff --git a/etc/ci/performance/test_differ.py b/etc/ci/performance/test_differ.py new file mode 100644 index 00000000000..98b35a82bbd --- /dev/null +++ b/etc/ci/performance/test_differ.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import json + + +parser = argparse.ArgumentParser(description="Diff between two runs of performance tests.") +parser.add_argument("file1", help="the first output json from runner") +parser.add_argument("file2", help="the second output json from runner") + +args = parser.parse_args() + +def load_data(filename): + with open(filename, 'r') as f: + results = {} + totals = {} + counts = {} + records = json.load(f) + for record in records: + key = record.get('testcase') + value = record.get('domComplete') - record.get('domLoading') + totals[key] = totals.get('key', 0) + value + counts[key] = counts.get('key', 0) + 1 + results[key] = round(totals[key] / counts[key]) + return results + +data1 = load_data(args.file1) +data2 = load_data(args.file2) +keys = set(data1.keys()).union(data2.keys()) + +BLUE = '\033[94m' +GREEN = '\033[92m' +WARNING = '\033[93m' +END = '\033[0m' + +for key in keys: + value1 = data1.get(key) + value2 = data2.get(key) + if value1 and not(value2): + print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file2, END)) + elif value2 and not(value1): + print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file1, END)) + elif value1 and value2: + diff = value2 - value1 + change = diff / value1 + color = BLUE if value1 <= value2 else GREEN + print("{}Test {}: first={}, second={}, diff={}, change={:.2%}.{}".format(color, key, value1, value2, diff, change, END)) From 0bd05e50c8219c70b40152916b50ae24be6a3258 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Fri, 19 Aug 2016 16:29:34 -0500 Subject: [PATCH 13/17] Ooops, checked in a test version. --- etc/ci/performance/prepare_manifest.sh | 0 etc/ci/performance/test_all.sh | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 etc/ci/performance/prepare_manifest.sh diff --git a/etc/ci/performance/prepare_manifest.sh b/etc/ci/performance/prepare_manifest.sh old mode 100644 new mode 100755 diff --git a/etc/ci/performance/test_all.sh b/etc/ci/performance/test_all.sh index e63fe6a529b..b0257b34d2b 100755 --- a/etc/ci/performance/test_all.sh +++ b/etc/ci/performance/test_all.sh @@ -43,7 +43,7 @@ MANIFEST="page_load_test/tp5n/20160509.manifest" # A manifest that excludes PERF_FILE="output/perf-$(date --iso-8601=seconds).json" echo "Running tests" -echo python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}" +python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}" if [[ -z "${local:-}" ]]; then @@ -53,7 +53,7 @@ then # TODO: we are using Servo's revision hash for Gecko's result to make both # results appear on the same date. Use the correct result when Perfherder # allows us to change the date. - echo python3 submit_to_perfherder.py \ + python3 submit_to_perfherder.py \ "${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json fi From 9ebd156889204325729a8a7379b64836fe4329ac Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Sat, 20 Aug 2016 12:00:20 -0500 Subject: [PATCH 14/17] Updated format string for test_differ.py. --- etc/ci/performance/test_differ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/performance/test_differ.py b/etc/ci/performance/test_differ.py index 98b35a82bbd..401830e3ffa 100644 --- a/etc/ci/performance/test_differ.py +++ b/etc/ci/performance/test_differ.py @@ -48,4 +48,4 @@ for key in keys: diff = value2 - value1 change = diff / value1 color = BLUE if value1 <= value2 else GREEN - print("{}Test {}: first={}, second={}, diff={}, change={:.2%}.{}".format(color, key, value1, value2, diff, change, END)) + print("{}{:6} {:6} {:+6} {:+8.2%} {}.{}".format(color, value1, value2, diff, change, key, END)) From 646d9c8988d2404ec7ba67517b0290da6cc5096f Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Sat, 20 Aug 2016 12:01:00 -0500 Subject: [PATCH 15/17] Disabled xunlei.com perf testcase. --- etc/ci/performance/page_load_test/tp5n/20160509.manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ci/performance/page_load_test/tp5n/20160509.manifest b/etc/ci/performance/page_load_test/tp5n/20160509.manifest index bd3d485651f..e9f5b01c184 100644 --- a/etc/ci/performance/page_load_test/tp5n/20160509.manifest +++ b/etc/ci/performance/page_load_test/tp5n/20160509.manifest @@ -46,7 +46,7 @@ http://localhost:8000/page_load_test/tp5n/tudou.com/www.tudou.com/index.html http://localhost:8000/page_load_test/tp5n/w3.org/www.w3.org/standards/webdesign/htmlcss.html # http://localhost:8000/page_load_test/tp5n/wsj.com/online.wsj.com/home-page.html # http://localhost:8000/page_load_test/tp5n/xinhuanet.com/xinhuanet.com/index.html -http://localhost:8000/page_load_test/tp5n/xunlei.com/xunlei.com/index.html +# http://localhost:8000/page_load_test/tp5n/xunlei.com/xunlei.com/index.html http://localhost:8000/page_load_test/tp5n/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html http://localhost:8000/page_load_test/tp5n/youku.com/www.youku.com/index.html http://localhost:8000/page_load_test/tp5n/youtube.com/www.youtube.com/music.html From bc07d9939104b984a61a1fefbadf82a6fd51a799 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Sat, 20 Aug 2016 12:06:51 -0500 Subject: [PATCH 16/17] Making test-tidy happy. --- etc/ci/performance/test_differ.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/ci/performance/test_differ.py b/etc/ci/performance/test_differ.py index 401830e3ffa..1f699633d5a 100644 --- a/etc/ci/performance/test_differ.py +++ b/etc/ci/performance/test_differ.py @@ -14,6 +14,7 @@ parser.add_argument("file2", help="the second output json from runner") args = parser.parse_args() + def load_data(filename): with open(filename, 'r') as f: results = {} @@ -36,7 +37,7 @@ BLUE = '\033[94m' GREEN = '\033[92m' WARNING = '\033[93m' END = '\033[0m' - + for key in keys: value1 = data1.get(key) value2 = data2.get(key) @@ -46,6 +47,6 @@ for key in keys: print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file1, END)) elif value1 and value2: diff = value2 - value1 - change = diff / value1 + change = diff / value1 color = BLUE if value1 <= value2 else GREEN print("{}{:6} {:6} {:+6} {:+8.2%} {}.{}".format(color, value1, value2, diff, change, key, END)) From 105b11ffcd9c4efe4fc9a1398712bb8235a6d385 Mon Sep 17 00:00:00 2001 From: Alan Jeffrey Date: Mon, 5 Sep 2016 08:43:03 -0500 Subject: [PATCH 17/17] Made local perf testing the default. --- etc/ci/performance/test_all.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/ci/performance/test_all.sh b/etc/ci/performance/test_all.sh index b0257b34d2b..b5be5a745c0 100755 --- a/etc/ci/performance/test_all.sh +++ b/etc/ci/performance/test_all.sh @@ -17,8 +17,8 @@ case "${1}" in --gecko) engine="--engine gecko" ;; - --local) - local=1 + --submit) + submit=1 ;; *) echo "Unknown option ${1}." @@ -45,7 +45,7 @@ PERF_FILE="output/perf-$(date --iso-8601=seconds).json" echo "Running tests" python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}" -if [[ -z "${local:-}" ]]; +if [[ "${submit:-}" ]]; then echo "Submitting to Perfherder" # Perfherder SSL check will fail if time is not accurate,