Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-08-05 13:40:08 +01:00
Code Issues Projects Releases Packages Wiki Activity

Auto merge of #12946 - asajeffrey:etc-ci-performance-run-locally, r=shinglyu

Browse source 
Etc ci performance run locally

<!-- Please describe your changes on the following line: -->

Changes to `etc/ci/performance` to get it to run locally on my machine at least!

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [X] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes do not require tests because this is performance-testing code

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/12946)
<!-- Reviewable:end -->
This commit is contained in:
bors-servo 2016-09-06 04:07:05 -05:00 • committed by GitHub
commit 9e349544f0
6 changed files with 142 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

104
etc/ci/performance/page_load_test/tp5n/20160509.manifest
View file

@ -1,52 +1,52 @@
http://localhost:8000/tp6/news.ycombinator.com/index.html # http://localhost:8000/tp6/news.ycombinator.com/index.html
# http://localhost:8000/page_load_test/163.com/www.163.com/index.html # http://localhost:8000/page_load_test/tp5n/163.com/www.163.com/index.html
http://localhost:8000/page_load_test/56.com/www.56.com/index.html http://localhost:8000/page_load_test/tp5n/56.com/www.56.com/index.html
# http://localhost:8000/page_load_test/aljazeera.net/aljazeera.net/portal.html # http://localhost:8000/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
http://localhost:8000/page_load_test/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html http://localhost:8000/page_load_test/tp5n/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html
http://localhost:8000/page_load_test/bbc.co.uk/www.bbc.co.uk/news/index.html http://localhost:8000/page_load_test/tp5n/bbc.co.uk/www.bbc.co.uk/news/index.html
http://localhost:8000/page_load_test/beatonna.livejournal.com/beatonna.livejournal.com/index.html http://localhost:8000/page_load_test/tp5n/beatonna.livejournal.com/beatonna.livejournal.com/index.html
# http://localhost:8000/page_load_test/bild.de/www.bild.de/index.html # http://localhost:8000/page_load_test/tp5n/bild.de/www.bild.de/index.html
http://localhost:8000/page_load_test/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html http://localhost:8000/page_load_test/tp5n/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html
http://localhost:8000/page_load_test/chemistry.about.com/chemistry.about.com/index.html http://localhost:8000/page_load_test/tp5n/chemistry.about.com/chemistry.about.com/index.html
# http://localhost:8000/page_load_test/chinaz.com/chinaz.com/index.html # http://localhost:8000/page_load_test/tp5n/chinaz.com/chinaz.com/index.html
http://localhost:8000/page_load_test/cnn.com/www.cnn.com/index.html http://localhost:8000/page_load_test/tp5n/cnn.com/www.cnn.com/index.html
http://localhost:8000/page_load_test/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html http://localhost:8000/page_load_test/tp5n/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html
http://localhost:8000/page_load_test/dailymotion.com/www.dailymotion.com/us.html http://localhost:8000/page_load_test/tp5n/dailymotion.com/www.dailymotion.com/us.html
# http://localhost:8000/page_load_test/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html # http://localhost:8000/page_load_test/tp5n/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html
http://localhost:8000/page_load_test/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html http://localhost:8000/page_load_test/tp5n/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html
http://localhost:8000/page_load_test/globo.com/www.globo.com/index.html http://localhost:8000/page_load_test/tp5n/globo.com/www.globo.com/index.html
http://localhost:8000/page_load_test/google.com/www.google.com/search@q=mozilla.html http://localhost:8000/page_load_test/tp5n/google.com/www.google.com/search@q=mozilla.html
http://localhost:8000/page_load_test/goo.ne.jp/goo.ne.jp/index.html http://localhost:8000/page_load_test/tp5n/goo.ne.jp/goo.ne.jp/index.html
# http://localhost:8000/page_load_test/guardian.co.uk/www.guardian.co.uk/index.html # http://localhost:8000/page_load_test/tp5n/guardian.co.uk/www.guardian.co.uk/index.html
# http://localhost:8000/page_load_test/homeway.com.cn/www.hexun.com/index.html # http://localhost:8000/page_load_test/tp5n/homeway.com.cn/www.hexun.com/index.html
http://localhost:8000/page_load_test/huffingtonpost.com/www.huffingtonpost.com/index.html http://localhost:8000/page_load_test/tp5n/huffingtonpost.com/www.huffingtonpost.com/index.html
# http://localhost:8000/page_load_test/ifeng.com/ifeng.com/index.html # http://localhost:8000/page_load_test/tp5n/ifeng.com/ifeng.com/index.html
# http://localhost:8000/page_load_test/imdb.com/www.imdb.com/title/tt1099212/index.html # http://localhost:8000/page_load_test/tp5n/imdb.com/www.imdb.com/title/tt1099212/index.html
http://localhost:8000/page_load_test/imgur.com/imgur.com/gallery/index.html http://localhost:8000/page_load_test/tp5n/imgur.com/imgur.com/gallery/index.html
# http://localhost:8000/page_load_test/indiatimes.com/www.indiatimes.com/index.html # http://localhost:8000/page_load_test/tp5n/indiatimes.com/www.indiatimes.com/index.html
http://localhost:8000/page_load_test/mail.ru/mail.ru/index.html http://localhost:8000/page_load_test/tp5n/mail.ru/mail.ru/index.html
# http://localhost:8000/page_load_test/mashable.com/mashable.com/index.html # http://localhost:8000/page_load_test/tp5n/mashable.com/mashable.com/index.html
http://localhost:8000/page_load_test/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html # http://localhost:8000/page_load_test/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html
http://localhost:8000/page_load_test/myspace.com/www.myspace.com/albumart.html http://localhost:8000/page_load_test/tp5n/myspace.com/www.myspace.com/albumart.html
# http://localhost:8000/page_load_test/naver.com/www.naver.com/index.html # http://localhost:8000/page_load_test/tp5n/naver.com/www.naver.com/index.html
# http://localhost:8000/page_load_test/noimpactman.typepad.com/noimpactman.typepad.com/index.html # http://localhost:8000/page_load_test/tp5n/noimpactman.typepad.com/noimpactman.typepad.com/index.html
http://localhost:8000/page_load_test/page.renren.com/page.renren.com/index.html http://localhost:8000/page_load_test/tp5n/page.renren.com/page.renren.com/index.html
# http://localhost:8000/page_load_test/people.com.cn/people.com.cn/index.html # http://localhost:8000/page_load_test/tp5n/people.com.cn/people.com.cn/index.html
http://localhost:8000/page_load_test/rakuten.co.jp/www.rakuten.co.jp/index.html http://localhost:8000/page_load_test/tp5n/rakuten.co.jp/www.rakuten.co.jp/index.html
http://localhost:8000/page_load_test/reddit.com/www.reddit.com/index.html http://localhost:8000/page_load_test/tp5n/reddit.com/www.reddit.com/index.html
# http://localhost:8000/page_load_test/reuters.com/www.reuters.com/index.html # http://localhost:8000/page_load_test/tp5n/reuters.com/www.reuters.com/index.html
# http://localhost:8000/page_load_test/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html # http://localhost:8000/page_load_test/tp5n/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html
# http://localhost:8000/page_load_test/sohu.com/www.sohu.com/index.html # http://localhost:8000/page_load_test/tp5n/sohu.com/www.sohu.com/index.html
http://localhost:8000/page_load_test/spiegel.de/www.spiegel.de/index.html http://localhost:8000/page_load_test/tp5n/spiegel.de/www.spiegel.de/index.html
http://localhost:8000/page_load_test/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html http://localhost:8000/page_load_test/tp5n/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html
# http://localhost:8000/page_load_test/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html # http://localhost:8000/page_load_test/tp5n/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html
# http://localhost:8000/page_load_test/thepiratebay.org/thepiratebay.org/top/201.html # http://localhost:8000/page_load_test/tp5n/thepiratebay.org/thepiratebay.org/top/201.html
http://localhost:8000/page_load_test/tudou.com/www.tudou.com/index.html http://localhost:8000/page_load_test/tp5n/tudou.com/www.tudou.com/index.html
# http://localhost:8000/page_load_test/uol.com.br/www.uol.com.br/index.html # http://localhost:8000/page_load_test/tp5n/uol.com.br/www.uol.com.br/index.html
http://localhost:8000/page_load_test/w3.org/www.w3.org/standards/webdesign/htmlcss.html http://localhost:8000/page_load_test/tp5n/w3.org/www.w3.org/standards/webdesign/htmlcss.html
# http://localhost:8000/page_load_test/wsj.com/online.wsj.com/home-page.html # http://localhost:8000/page_load_test/tp5n/wsj.com/online.wsj.com/home-page.html
# http://localhost:8000/page_load_test/xinhuanet.com/xinhuanet.com/index.html # http://localhost:8000/page_load_test/tp5n/xinhuanet.com/xinhuanet.com/index.html
http://localhost:8000/page_load_test/xunlei.com/xunlei.com/index.html # http://localhost:8000/page_load_test/tp5n/xunlei.com/xunlei.com/index.html
http://localhost:8000/page_load_test/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html http://localhost:8000/page_load_test/tp5n/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html
http://localhost:8000/page_load_test/youku.com/www.youku.com/index.html http://localhost:8000/page_load_test/tp5n/youku.com/www.youku.com/index.html
http://localhost:8000/page_load_test/youtube.com/www.youtube.com/music.html http://localhost:8000/page_load_test/tp5n/youtube.com/www.youtube.com/music.html

0
etc/ci/performance/prepare_manifest.sh Normal file → Executable file
View file

25
etc/ci/performance/runner.py
View file

@ -28,8 +28,7 @@ def execute_test(url, command, timeout):
print(' '.join(command)) print(' '.join(command))
print("Timeout:{}".format(timeout)) print("Timeout:{}".format(timeout))
try: try:
return subprocess.check_output(command, stderr=subprocess.STDOUT, return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout)
shell=True, timeout=timeout)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print("Unexpected Fail:") print("Unexpected Fail:")
print(e) print(e)
@ -67,7 +66,7 @@ def parse_log(log, testcase=None):
copy = False copy = False
blocks.append(block) blocks.append(block)
block = [] block = []
elif copy: elif copy and line.strip().startswith("[PERF]"):
block.append(line) block.append(line)
# We need to still include the failed tests, otherwise Treeherder will # We need to still include the failed tests, otherwise Treeherder will
@ -113,21 +112,19 @@ def parse_log(log, testcase=None):
print('-----') print('-----')
return placeholder return placeholder
if key == "testcase": if key == "testcase" or key == "title":
timing[key] = value timing[key] = value
else: else:
timing[key] = None if (value == "undefined") else int(value) timing[key] = None if (value == "undefined") else int(value)
if testcase is not None and timing['testcase'] != testcase:
print('[DEBUG] log:')
print('-----')
print(log)
print('-----')
return placeholder
return timing return timing
if len(blocks) == 0: def valid_timing(timing):
return (timing.get('title') != 'Error response') and (testcase is None or timing.get('testcase') == testcase)
timings = list(filter(valid_timing, map(parse_block, blocks)))
if len(timings) == 0:
print("Didn't find any perf data in the log, test timeout?") print("Didn't find any perf data in the log, test timeout?")
print("Fillng in a dummy perf data") print("Fillng in a dummy perf data")
print('[DEBUG] log:') print('[DEBUG] log:')
@ -137,7 +134,7 @@ def parse_log(log, testcase=None):
return [placeholder] return [placeholder]
else: else:
return map(parse_block, blocks) return timings
def filter_result_by_manifest(result_json, manifest): def filter_result_by_manifest(result_json, manifest):
@ -242,7 +239,7 @@ def main():
testcases = load_manifest(args.tp5_manifest) testcases = load_manifest(args.tp5_manifest)
results = [] results = []
for testcase in testcases: for testcase in testcases:
command = (["timeout", "{timeout}s".format(args.timeout)] + command = (["timeout", "{}s".format(args.timeout)] +
command_factory(testcase)) command_factory(testcase))
for run in range(args.runs): for run in range(args.runs):
print("Running test {}/{} on {}".format(run + 1, print("Running test {}/{} on {}".format(run + 1,

24
etc/ci/performance/test_all.sh
View file

@ -8,6 +8,8 @@ set -o errexit
set -o nounset set -o nounset
set -o pipefail set -o pipefail
while (( "${#}" ))
do
case "${1}" in case "${1}" in
--servo) --servo)
engine="--engine servo" engine="--engine servo"
@ -15,12 +17,20 @@ case "${1}" in
--gecko) --gecko)
engine="--engine gecko" engine="--engine gecko"
;; ;;
--submit)
submit=1
;;
*) *)
# This branch should never be reached with set -o nounset echo "Unknown option ${1}."
echo "You didn't specify the engine to run."
exit exit
;; ;;
esac esac
shift
done
if [[ -z "${engine:-}" ]];
then echo "You didn't specify the engine to run: --servo or --gecko."; exit;
fi
echo "Starting the local server" echo "Starting the local server"
python3 -m http.server > /dev/null 2>&1 & python3 -m http.server > /dev/null 2>&1 &
@ -33,15 +43,19 @@ MANIFEST="page_load_test/tp5n/20160509.manifest" # A manifest that excludes
PERF_FILE="output/perf-$(date --iso-8601=seconds).json" PERF_FILE="output/perf-$(date --iso-8601=seconds).json"
echo "Running tests" echo "Running tests"
python3 runner.py "${engine}" --runs 3 "${MANIFEST}" "${PERF_FILE}" python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}"
if [[ "${submit:-}" ]];
then
echo "Submitting to Perfherder" echo "Submitting to Perfherder"
# Perfherder SSL check will fail if time is not accurate, # Perfherder SSL check will fail if time is not accurate,
# sync time before you submit # sync time before you submit
# TODO: we are using Servo's revision hash for Gecko's result to make both # TODO: we are using Servo's revision hash for Gecko's result to make both
# results appear on the same date. Use the correct result when Perfherder # results appear on the same date. Use the correct result when Perfherder
# allows us to change the date. # allows us to change the date.
python3 submit_to_perfherder.py "${engine}" "${PERF_FILE}" servo/revision.json python3 submit_to_perfherder.py \
"${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json
fi
# Kill the http server echo "Stopping the local server"
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT

52
etc/ci/performance/test_differ.py Normal file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import argparse
import json
parser = argparse.ArgumentParser(description="Diff between two runs of performance tests.")
parser.add_argument("file1", help="the first output json from runner")
parser.add_argument("file2", help="the second output json from runner")
args = parser.parse_args()
def load_data(filename):
with open(filename, 'r') as f:
results = {}
totals = {}
counts = {}
records = json.load(f)
for record in records:
key = record.get('testcase')
value = record.get('domComplete') - record.get('domLoading')
totals[key] = totals.get('key', 0) + value
counts[key] = counts.get('key', 0) + 1
results[key] = round(totals[key] / counts[key])
return results
data1 = load_data(args.file1)
data2 = load_data(args.file2)
keys = set(data1.keys()).union(data2.keys())
BLUE = '\033[94m'
GREEN = '\033[92m'
WARNING = '\033[93m'
END = '\033[0m'
for key in keys:
value1 = data1.get(key)
value2 = data2.get(key)
if value1 and not(value2):
print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file2, END))
elif value2 and not(value1):
print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file1, END))
elif value1 and value2:
diff = value2 - value1
change = diff / value1
color = BLUE if value1 <= value2 else GREEN
print("{}{:6} {:6} {:+6} {:+8.2%} {}.{}".format(color, value1, value2, diff, change, key, END))

1
etc/ci/performance/user-agent-js/01.perf-timing.js
View file

@ -12,6 +12,7 @@ function formatLine(name, t) {
function printPerfTiming() { function printPerfTiming() {
print("[PERF] perf block start") print("[PERF] perf block start")
formatLine("testcase", window.location); formatLine("testcase", window.location);
formatLine("title", document.title.replace(/,/g, "&#44;"));
formatLine("navigationStart", performance.timing.navigationStart); formatLine("navigationStart", performance.timing.navigationStart);
formatLine("unloadEventStart", performance.timing.unloadEventStart); formatLine("unloadEventStart", performance.timing.unloadEventStart);
formatLine("unloadEventEnd", performance.timing.unloadEventEnd); formatLine("unloadEventEnd", performance.timing.unloadEventEnd);