diff --git a/etc/ci/performance/page_load_test/tp5n/20160509.manifest b/etc/ci/performance/page_load_test/tp5n/20160509.manifest index 5dcbdace620..e9f5b01c184 100644 --- a/etc/ci/performance/page_load_test/tp5n/20160509.manifest +++ b/etc/ci/performance/page_load_test/tp5n/20160509.manifest @@ -1,52 +1,52 @@ -http://localhost:8000/tp6/news.ycombinator.com/index.html -# http://localhost:8000/page_load_test/163.com/www.163.com/index.html -http://localhost:8000/page_load_test/56.com/www.56.com/index.html -# http://localhost:8000/page_load_test/aljazeera.net/aljazeera.net/portal.html -http://localhost:8000/page_load_test/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html -http://localhost:8000/page_load_test/bbc.co.uk/www.bbc.co.uk/news/index.html -http://localhost:8000/page_load_test/beatonna.livejournal.com/beatonna.livejournal.com/index.html -# http://localhost:8000/page_load_test/bild.de/www.bild.de/index.html -http://localhost:8000/page_load_test/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html -http://localhost:8000/page_load_test/chemistry.about.com/chemistry.about.com/index.html -# http://localhost:8000/page_load_test/chinaz.com/chinaz.com/index.html -http://localhost:8000/page_load_test/cnn.com/www.cnn.com/index.html -http://localhost:8000/page_load_test/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html -http://localhost:8000/page_load_test/dailymotion.com/www.dailymotion.com/us.html -# http://localhost:8000/page_load_test/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html -http://localhost:8000/page_load_test/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html -http://localhost:8000/page_load_test/globo.com/www.globo.com/index.html -http://localhost:8000/page_load_test/google.com/www.google.com/search@q=mozilla.html -http://localhost:8000/page_load_test/goo.ne.jp/goo.ne.jp/index.html -# http://localhost:8000/page_load_test/guardian.co.uk/www.guardian.co.uk/index.html -# http://localhost:8000/page_load_test/homeway.com.cn/www.hexun.com/index.html -http://localhost:8000/page_load_test/huffingtonpost.com/www.huffingtonpost.com/index.html -# http://localhost:8000/page_load_test/ifeng.com/ifeng.com/index.html -# http://localhost:8000/page_load_test/imdb.com/www.imdb.com/title/tt1099212/index.html -http://localhost:8000/page_load_test/imgur.com/imgur.com/gallery/index.html -# http://localhost:8000/page_load_test/indiatimes.com/www.indiatimes.com/index.html -http://localhost:8000/page_load_test/mail.ru/mail.ru/index.html -# http://localhost:8000/page_load_test/mashable.com/mashable.com/index.html -http://localhost:8000/page_load_test/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html -http://localhost:8000/page_load_test/myspace.com/www.myspace.com/albumart.html -# http://localhost:8000/page_load_test/naver.com/www.naver.com/index.html -# http://localhost:8000/page_load_test/noimpactman.typepad.com/noimpactman.typepad.com/index.html -http://localhost:8000/page_load_test/page.renren.com/page.renren.com/index.html -# http://localhost:8000/page_load_test/people.com.cn/people.com.cn/index.html -http://localhost:8000/page_load_test/rakuten.co.jp/www.rakuten.co.jp/index.html -http://localhost:8000/page_load_test/reddit.com/www.reddit.com/index.html -# http://localhost:8000/page_load_test/reuters.com/www.reuters.com/index.html -# http://localhost:8000/page_load_test/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html -# http://localhost:8000/page_load_test/sohu.com/www.sohu.com/index.html -http://localhost:8000/page_load_test/spiegel.de/www.spiegel.de/index.html -http://localhost:8000/page_load_test/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html -# http://localhost:8000/page_load_test/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html -# http://localhost:8000/page_load_test/thepiratebay.org/thepiratebay.org/top/201.html -http://localhost:8000/page_load_test/tudou.com/www.tudou.com/index.html -# http://localhost:8000/page_load_test/uol.com.br/www.uol.com.br/index.html -http://localhost:8000/page_load_test/w3.org/www.w3.org/standards/webdesign/htmlcss.html -# http://localhost:8000/page_load_test/wsj.com/online.wsj.com/home-page.html -# http://localhost:8000/page_load_test/xinhuanet.com/xinhuanet.com/index.html -http://localhost:8000/page_load_test/xunlei.com/xunlei.com/index.html -http://localhost:8000/page_load_test/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html -http://localhost:8000/page_load_test/youku.com/www.youku.com/index.html -http://localhost:8000/page_load_test/youtube.com/www.youtube.com/music.html +# http://localhost:8000/tp6/news.ycombinator.com/index.html +# http://localhost:8000/page_load_test/tp5n/163.com/www.163.com/index.html +http://localhost:8000/page_load_test/tp5n/56.com/www.56.com/index.html +# http://localhost:8000/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html +http://localhost:8000/page_load_test/tp5n/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html +http://localhost:8000/page_load_test/tp5n/bbc.co.uk/www.bbc.co.uk/news/index.html +http://localhost:8000/page_load_test/tp5n/beatonna.livejournal.com/beatonna.livejournal.com/index.html +# http://localhost:8000/page_load_test/tp5n/bild.de/www.bild.de/index.html +http://localhost:8000/page_load_test/tp5n/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html +http://localhost:8000/page_load_test/tp5n/chemistry.about.com/chemistry.about.com/index.html +# http://localhost:8000/page_load_test/tp5n/chinaz.com/chinaz.com/index.html +http://localhost:8000/page_load_test/tp5n/cnn.com/www.cnn.com/index.html +http://localhost:8000/page_load_test/tp5n/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html +http://localhost:8000/page_load_test/tp5n/dailymotion.com/www.dailymotion.com/us.html +# http://localhost:8000/page_load_test/tp5n/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html +http://localhost:8000/page_load_test/tp5n/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html +http://localhost:8000/page_load_test/tp5n/globo.com/www.globo.com/index.html +http://localhost:8000/page_load_test/tp5n/google.com/www.google.com/search@q=mozilla.html +http://localhost:8000/page_load_test/tp5n/goo.ne.jp/goo.ne.jp/index.html +# http://localhost:8000/page_load_test/tp5n/guardian.co.uk/www.guardian.co.uk/index.html +# http://localhost:8000/page_load_test/tp5n/homeway.com.cn/www.hexun.com/index.html +http://localhost:8000/page_load_test/tp5n/huffingtonpost.com/www.huffingtonpost.com/index.html +# http://localhost:8000/page_load_test/tp5n/ifeng.com/ifeng.com/index.html +# http://localhost:8000/page_load_test/tp5n/imdb.com/www.imdb.com/title/tt1099212/index.html +http://localhost:8000/page_load_test/tp5n/imgur.com/imgur.com/gallery/index.html +# http://localhost:8000/page_load_test/tp5n/indiatimes.com/www.indiatimes.com/index.html +http://localhost:8000/page_load_test/tp5n/mail.ru/mail.ru/index.html +# http://localhost:8000/page_load_test/tp5n/mashable.com/mashable.com/index.html +# http://localhost:8000/page_load_test/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html +http://localhost:8000/page_load_test/tp5n/myspace.com/www.myspace.com/albumart.html +# http://localhost:8000/page_load_test/tp5n/naver.com/www.naver.com/index.html +# http://localhost:8000/page_load_test/tp5n/noimpactman.typepad.com/noimpactman.typepad.com/index.html +http://localhost:8000/page_load_test/tp5n/page.renren.com/page.renren.com/index.html +# http://localhost:8000/page_load_test/tp5n/people.com.cn/people.com.cn/index.html +http://localhost:8000/page_load_test/tp5n/rakuten.co.jp/www.rakuten.co.jp/index.html +http://localhost:8000/page_load_test/tp5n/reddit.com/www.reddit.com/index.html +# http://localhost:8000/page_load_test/tp5n/reuters.com/www.reuters.com/index.html +# http://localhost:8000/page_load_test/tp5n/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html +# http://localhost:8000/page_load_test/tp5n/sohu.com/www.sohu.com/index.html +http://localhost:8000/page_load_test/tp5n/spiegel.de/www.spiegel.de/index.html +http://localhost:8000/page_load_test/tp5n/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html +# http://localhost:8000/page_load_test/tp5n/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html +# http://localhost:8000/page_load_test/tp5n/thepiratebay.org/thepiratebay.org/top/201.html +http://localhost:8000/page_load_test/tp5n/tudou.com/www.tudou.com/index.html +# http://localhost:8000/page_load_test/tp5n/uol.com.br/www.uol.com.br/index.html +http://localhost:8000/page_load_test/tp5n/w3.org/www.w3.org/standards/webdesign/htmlcss.html +# http://localhost:8000/page_load_test/tp5n/wsj.com/online.wsj.com/home-page.html +# http://localhost:8000/page_load_test/tp5n/xinhuanet.com/xinhuanet.com/index.html +# http://localhost:8000/page_load_test/tp5n/xunlei.com/xunlei.com/index.html +http://localhost:8000/page_load_test/tp5n/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html +http://localhost:8000/page_load_test/tp5n/youku.com/www.youku.com/index.html +http://localhost:8000/page_load_test/tp5n/youtube.com/www.youtube.com/music.html diff --git a/etc/ci/performance/prepare_manifest.sh b/etc/ci/performance/prepare_manifest.sh old mode 100644 new mode 100755 diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index d34b9e49312..7da6711db85 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -28,8 +28,7 @@ def execute_test(url, command, timeout): print(' '.join(command)) print("Timeout:{}".format(timeout)) try: - return subprocess.check_output(command, stderr=subprocess.STDOUT, - shell=True, timeout=timeout) + return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout) except subprocess.CalledProcessError as e: print("Unexpected Fail:") print(e) @@ -43,7 +42,7 @@ def execute_test(url, command, timeout): def get_servo_command(url): ua_script_path = "{}/user-agent-js".format(os.getcwd()) return ["./servo/servo", url, - " --userscripts", ua_script_path, + "--userscripts", ua_script_path, "-x", "-o", "output.png"] @@ -67,7 +66,7 @@ def parse_log(log, testcase=None): copy = False blocks.append(block) block = [] - elif copy: + elif copy and line.strip().startswith("[PERF]"): block.append(line) # We need to still include the failed tests, otherwise Treeherder will @@ -113,21 +112,19 @@ def parse_log(log, testcase=None): print('-----') return placeholder - if key == "testcase": + if key == "testcase" or key == "title": timing[key] = value else: timing[key] = None if (value == "undefined") else int(value) - if testcase is not None and timing['testcase'] != testcase: - print('[DEBUG] log:') - print('-----') - print(log) - print('-----') - return placeholder - return timing - if len(blocks) == 0: + def valid_timing(timing): + return (timing.get('title') != 'Error response') and (testcase is None or timing.get('testcase') == testcase) + + timings = list(filter(valid_timing, map(parse_block, blocks))) + + if len(timings) == 0: print("Didn't find any perf data in the log, test timeout?") print("Fillng in a dummy perf data") print('[DEBUG] log:') @@ -137,7 +134,7 @@ def parse_log(log, testcase=None): return [placeholder] else: - return map(parse_block, blocks) + return timings def filter_result_by_manifest(result_json, manifest): @@ -242,7 +239,7 @@ def main(): testcases = load_manifest(args.tp5_manifest) results = [] for testcase in testcases: - command = (["timeout", "{timeout}s".format(args.timeout)] + + command = (["timeout", "{}s".format(args.timeout)] + command_factory(testcase)) for run in range(args.runs): print("Running test {}/{} on {}".format(run + 1, diff --git a/etc/ci/performance/test_all.sh b/etc/ci/performance/test_all.sh index 850a4dd1505..b5be5a745c0 100755 --- a/etc/ci/performance/test_all.sh +++ b/etc/ci/performance/test_all.sh @@ -8,6 +8,8 @@ set -o errexit set -o nounset set -o pipefail +while (( "${#}" )) +do case "${1}" in --servo) engine="--engine servo" @@ -15,12 +17,20 @@ case "${1}" in --gecko) engine="--engine gecko" ;; + --submit) + submit=1 + ;; *) - # This branch should never be reached with set -o nounset - echo "You didn't specify the engine to run." + echo "Unknown option ${1}." exit ;; esac +shift +done + +if [[ -z "${engine:-}" ]]; +then echo "You didn't specify the engine to run: --servo or --gecko."; exit; +fi echo "Starting the local server" python3 -m http.server > /dev/null 2>&1 & @@ -33,15 +43,19 @@ MANIFEST="page_load_test/tp5n/20160509.manifest" # A manifest that excludes PERF_FILE="output/perf-$(date --iso-8601=seconds).json" echo "Running tests" -python3 runner.py "${engine}" --runs 3 "${MANIFEST}" "${PERF_FILE}" +python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}" -echo "Submitting to Perfherder" -# Perfherder SSL check will fail if time is not accurate, -# sync time before you submit -# TODO: we are using Servo's revision hash for Gecko's result to make both -# results appear on the same date. Use the correct result when Perfherder -# allows us to change the date. -python3 submit_to_perfherder.py "${engine}" "${PERF_FILE}" servo/revision.json +if [[ "${submit:-}" ]]; +then + echo "Submitting to Perfherder" + # Perfherder SSL check will fail if time is not accurate, + # sync time before you submit + # TODO: we are using Servo's revision hash for Gecko's result to make both + # results appear on the same date. Use the correct result when Perfherder + # allows us to change the date. + python3 submit_to_perfherder.py \ + "${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json +fi -# Kill the http server +echo "Stopping the local server" trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT diff --git a/etc/ci/performance/test_differ.py b/etc/ci/performance/test_differ.py new file mode 100644 index 00000000000..1f699633d5a --- /dev/null +++ b/etc/ci/performance/test_differ.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import json + + +parser = argparse.ArgumentParser(description="Diff between two runs of performance tests.") +parser.add_argument("file1", help="the first output json from runner") +parser.add_argument("file2", help="the second output json from runner") + +args = parser.parse_args() + + +def load_data(filename): + with open(filename, 'r') as f: + results = {} + totals = {} + counts = {} + records = json.load(f) + for record in records: + key = record.get('testcase') + value = record.get('domComplete') - record.get('domLoading') + totals[key] = totals.get('key', 0) + value + counts[key] = counts.get('key', 0) + 1 + results[key] = round(totals[key] / counts[key]) + return results + +data1 = load_data(args.file1) +data2 = load_data(args.file2) +keys = set(data1.keys()).union(data2.keys()) + +BLUE = '\033[94m' +GREEN = '\033[92m' +WARNING = '\033[93m' +END = '\033[0m' + +for key in keys: + value1 = data1.get(key) + value2 = data2.get(key) + if value1 and not(value2): + print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file2, END)) + elif value2 and not(value1): + print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file1, END)) + elif value1 and value2: + diff = value2 - value1 + change = diff / value1 + color = BLUE if value1 <= value2 else GREEN + print("{}{:6} {:6} {:+6} {:+8.2%} {}.{}".format(color, value1, value2, diff, change, key, END)) diff --git a/etc/ci/performance/user-agent-js/01.perf-timing.js b/etc/ci/performance/user-agent-js/01.perf-timing.js index df2e3efcd55..9aff2aa8a56 100644 --- a/etc/ci/performance/user-agent-js/01.perf-timing.js +++ b/etc/ci/performance/user-agent-js/01.perf-timing.js @@ -12,6 +12,7 @@ function formatLine(name, t) { function printPerfTiming() { print("[PERF] perf block start") formatLine("testcase", window.location); + formatLine("title", document.title.replace(/,/g, ",")); formatLine("navigationStart", performance.timing.navigationStart); formatLine("unloadEventStart", performance.timing.unloadEventStart); formatLine("unloadEventEnd", performance.timing.unloadEventEnd);