mirror of
https://github.com/servo/servo.git
synced 2025-06-06 16:45:39 +00:00
Auto merge of #12946 - asajeffrey:etc-ci-performance-run-locally, r=shinglyu
Etc ci performance run locally <!-- Please describe your changes on the following line: --> Changes to `etc/ci/performance` to get it to run locally on my machine at least! --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: --> - [X] `./mach build -d` does not report any errors - [X] `./mach test-tidy` does not report any errors - [X] These changes do not require tests because this is performance-testing code <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. --> <!-- Reviewable:start --> --- This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/12946) <!-- Reviewable:end -->
This commit is contained in:
bors-servo
GitHub
commit
9e349544f0
6 changed files with 142 additions and 78 deletions
|
|
@ -1,52 +1,52 @@
|
|||
http://localhost:8000/tp6/news.ycombinator.com/index.html
|
||||
# http://localhost:8000/page_load_test/163.com/www.163.com/index.html
|
||||
http://localhost:8000/page_load_test/56.com/www.56.com/index.html
|
||||
# http://localhost:8000/page_load_test/aljazeera.net/aljazeera.net/portal.html
|
||||
http://localhost:8000/page_load_test/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html
|
||||
http://localhost:8000/page_load_test/bbc.co.uk/www.bbc.co.uk/news/index.html
|
||||
http://localhost:8000/page_load_test/beatonna.livejournal.com/beatonna.livejournal.com/index.html
|
||||
# http://localhost:8000/page_load_test/bild.de/www.bild.de/index.html
|
||||
http://localhost:8000/page_load_test/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html
|
||||
http://localhost:8000/page_load_test/chemistry.about.com/chemistry.about.com/index.html
|
||||
# http://localhost:8000/page_load_test/chinaz.com/chinaz.com/index.html
|
||||
http://localhost:8000/page_load_test/cnn.com/www.cnn.com/index.html
|
||||
http://localhost:8000/page_load_test/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html
|
||||
http://localhost:8000/page_load_test/dailymotion.com/www.dailymotion.com/us.html
|
||||
# http://localhost:8000/page_load_test/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html
|
||||
http://localhost:8000/page_load_test/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html
|
||||
http://localhost:8000/page_load_test/globo.com/www.globo.com/index.html
|
||||
http://localhost:8000/page_load_test/google.com/www.google.com/search@q=mozilla.html
|
||||
http://localhost:8000/page_load_test/goo.ne.jp/goo.ne.jp/index.html
|
||||
# http://localhost:8000/page_load_test/guardian.co.uk/www.guardian.co.uk/index.html
|
||||
# http://localhost:8000/page_load_test/homeway.com.cn/www.hexun.com/index.html
|
||||
http://localhost:8000/page_load_test/huffingtonpost.com/www.huffingtonpost.com/index.html
|
||||
# http://localhost:8000/page_load_test/ifeng.com/ifeng.com/index.html
|
||||
# http://localhost:8000/page_load_test/imdb.com/www.imdb.com/title/tt1099212/index.html
|
||||
http://localhost:8000/page_load_test/imgur.com/imgur.com/gallery/index.html
|
||||
# http://localhost:8000/page_load_test/indiatimes.com/www.indiatimes.com/index.html
|
||||
http://localhost:8000/page_load_test/mail.ru/mail.ru/index.html
|
||||
# http://localhost:8000/page_load_test/mashable.com/mashable.com/index.html
|
||||
http://localhost:8000/page_load_test/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html
|
||||
http://localhost:8000/page_load_test/myspace.com/www.myspace.com/albumart.html
|
||||
# http://localhost:8000/page_load_test/naver.com/www.naver.com/index.html
|
||||
# http://localhost:8000/page_load_test/noimpactman.typepad.com/noimpactman.typepad.com/index.html
|
||||
http://localhost:8000/page_load_test/page.renren.com/page.renren.com/index.html
|
||||
# http://localhost:8000/page_load_test/people.com.cn/people.com.cn/index.html
|
||||
http://localhost:8000/page_load_test/rakuten.co.jp/www.rakuten.co.jp/index.html
|
||||
http://localhost:8000/page_load_test/reddit.com/www.reddit.com/index.html
|
||||
# http://localhost:8000/page_load_test/reuters.com/www.reuters.com/index.html
|
||||
# http://localhost:8000/page_load_test/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html
|
||||
# http://localhost:8000/page_load_test/sohu.com/www.sohu.com/index.html
|
||||
http://localhost:8000/page_load_test/spiegel.de/www.spiegel.de/index.html
|
||||
http://localhost:8000/page_load_test/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html
|
||||
# http://localhost:8000/page_load_test/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html
|
||||
# http://localhost:8000/page_load_test/thepiratebay.org/thepiratebay.org/top/201.html
|
||||
http://localhost:8000/page_load_test/tudou.com/www.tudou.com/index.html
|
||||
# http://localhost:8000/page_load_test/uol.com.br/www.uol.com.br/index.html
|
||||
http://localhost:8000/page_load_test/w3.org/www.w3.org/standards/webdesign/htmlcss.html
|
||||
# http://localhost:8000/page_load_test/wsj.com/online.wsj.com/home-page.html
|
||||
# http://localhost:8000/page_load_test/xinhuanet.com/xinhuanet.com/index.html
|
||||
http://localhost:8000/page_load_test/xunlei.com/xunlei.com/index.html
|
||||
http://localhost:8000/page_load_test/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html
|
||||
http://localhost:8000/page_load_test/youku.com/www.youku.com/index.html
|
||||
http://localhost:8000/page_load_test/youtube.com/www.youtube.com/music.html
|
||||
# http://localhost:8000/tp6/news.ycombinator.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/163.com/www.163.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/56.com/www.56.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
|
||||
http://localhost:8000/page_load_test/tp5n/amazon.com/www.amazon.com/Kindle-Wireless-Reader-Wifi-Graphite/dp/B002Y27P3M/507846.html
|
||||
http://localhost:8000/page_load_test/tp5n/bbc.co.uk/www.bbc.co.uk/news/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/beatonna.livejournal.com/beatonna.livejournal.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/bild.de/www.bild.de/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/cgi.ebay.com/cgi.ebay.com/ALL-NEW-KINDLE-3-eBOOK-WIRELESS-READING-DEVICE-W-WIFI-/130496077314@pt=LH_DefaultDomain_0&hash=item1e622c1e02.html
|
||||
http://localhost:8000/page_load_test/tp5n/chemistry.about.com/chemistry.about.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/chinaz.com/chinaz.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/cnn.com/www.cnn.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/dailymail.co.uk/www.dailymail.co.uk/ushome/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/dailymotion.com/www.dailymotion.com/us.html
|
||||
# http://localhost:8000/page_load_test/tp5n/digg.com/digg.com/news/story/New_logo_for_Mozilla_Firefox_browser.html
|
||||
http://localhost:8000/page_load_test/tp5n/ezinearticles.com/ezinearticles.com/index.html@Migraine-Ocular---The-Eye-Migraines&id=4684133.html
|
||||
http://localhost:8000/page_load_test/tp5n/globo.com/www.globo.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/google.com/www.google.com/search@q=mozilla.html
|
||||
http://localhost:8000/page_load_test/tp5n/goo.ne.jp/goo.ne.jp/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/guardian.co.uk/www.guardian.co.uk/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/homeway.com.cn/www.hexun.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/huffingtonpost.com/www.huffingtonpost.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/ifeng.com/ifeng.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/imdb.com/www.imdb.com/title/tt1099212/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/imgur.com/imgur.com/gallery/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/indiatimes.com/www.indiatimes.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/mail.ru/mail.ru/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/mashable.com/mashable.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/media.photobucket.com/media.photobucket.com/image/funny%20gif/findstuff22/Best%20Images/Funny/funny-gif1.jpg@o=1.html
|
||||
http://localhost:8000/page_load_test/tp5n/myspace.com/www.myspace.com/albumart.html
|
||||
# http://localhost:8000/page_load_test/tp5n/naver.com/www.naver.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/noimpactman.typepad.com/noimpactman.typepad.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/page.renren.com/page.renren.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/people.com.cn/people.com.cn/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/rakuten.co.jp/www.rakuten.co.jp/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/reddit.com/www.reddit.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/reuters.com/www.reuters.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/slideshare.net/www.slideshare.net/jameswillamor/lolcats-in-popular-culture-a-historical-perspective.html
|
||||
# http://localhost:8000/page_load_test/tp5n/sohu.com/www.sohu.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/spiegel.de/www.spiegel.de/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/stackoverflow.com/stackoverflow.com/questions/184618/what-is-the-best-comment-in-source-code-you-have-ever-encountered.html
|
||||
# http://localhost:8000/page_load_test/tp5n/store.apple.com/store.apple.com/us@mco=Nzc1MjMwNA.html
|
||||
# http://localhost:8000/page_load_test/tp5n/thepiratebay.org/thepiratebay.org/top/201.html
|
||||
http://localhost:8000/page_load_test/tp5n/tudou.com/www.tudou.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/uol.com.br/www.uol.com.br/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/w3.org/www.w3.org/standards/webdesign/htmlcss.html
|
||||
# http://localhost:8000/page_load_test/tp5n/wsj.com/online.wsj.com/home-page.html
|
||||
# http://localhost:8000/page_load_test/tp5n/xinhuanet.com/xinhuanet.com/index.html
|
||||
# http://localhost:8000/page_load_test/tp5n/xunlei.com/xunlei.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/yelp.com/www.yelp.com/biz/alexanders-steakhouse-cupertino.html
|
||||
http://localhost:8000/page_load_test/tp5n/youku.com/www.youku.com/index.html
|
||||
http://localhost:8000/page_load_test/tp5n/youtube.com/www.youtube.com/music.html
|
||||
|
|
|
|||
0
etc/ci/performance/prepare_manifest.sh
Normal file → Executable file
0
etc/ci/performance/prepare_manifest.sh
Normal file → Executable file
|
|
@ -28,8 +28,7 @@ def execute_test(url, command, timeout):
|
|||
print(' '.join(command))
|
||||
print("Timeout:{}".format(timeout))
|
||||
try:
|
||||
return subprocess.check_output(command, stderr=subprocess.STDOUT,
|
||||
shell=True, timeout=timeout)
|
||||
return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Unexpected Fail:")
|
||||
print(e)
|
||||
|
|
@ -43,7 +42,7 @@ def execute_test(url, command, timeout):
|
|||
def get_servo_command(url):
|
||||
ua_script_path = "{}/user-agent-js".format(os.getcwd())
|
||||
return ["./servo/servo", url,
|
||||
" --userscripts", ua_script_path,
|
||||
"--userscripts", ua_script_path,
|
||||
"-x", "-o", "output.png"]
|
||||
|
||||
|
||||
|
|
@ -67,7 +66,7 @@ def parse_log(log, testcase=None):
|
|||
copy = False
|
||||
blocks.append(block)
|
||||
block = []
|
||||
elif copy:
|
||||
elif copy and line.strip().startswith("[PERF]"):
|
||||
block.append(line)
|
||||
|
||||
# We need to still include the failed tests, otherwise Treeherder will
|
||||
|
|
@ -113,21 +112,19 @@ def parse_log(log, testcase=None):
|
|||
print('-----')
|
||||
return placeholder
|
||||
|
||||
if key == "testcase":
|
||||
if key == "testcase" or key == "title":
|
||||
timing[key] = value
|
||||
else:
|
||||
timing[key] = None if (value == "undefined") else int(value)
|
||||
|
||||
if testcase is not None and timing['testcase'] != testcase:
|
||||
print('[DEBUG] log:')
|
||||
print('-----')
|
||||
print(log)
|
||||
print('-----')
|
||||
return placeholder
|
||||
|
||||
return timing
|
||||
|
||||
if len(blocks) == 0:
|
||||
def valid_timing(timing):
|
||||
return (timing.get('title') != 'Error response') and (testcase is None or timing.get('testcase') == testcase)
|
||||
|
||||
timings = list(filter(valid_timing, map(parse_block, blocks)))
|
||||
|
||||
if len(timings) == 0:
|
||||
print("Didn't find any perf data in the log, test timeout?")
|
||||
print("Fillng in a dummy perf data")
|
||||
print('[DEBUG] log:')
|
||||
|
|
@ -137,7 +134,7 @@ def parse_log(log, testcase=None):
|
|||
|
||||
return [placeholder]
|
||||
else:
|
||||
return map(parse_block, blocks)
|
||||
return timings
|
||||
|
||||
|
||||
def filter_result_by_manifest(result_json, manifest):
|
||||
|
|
@ -242,7 +239,7 @@ def main():
|
|||
testcases = load_manifest(args.tp5_manifest)
|
||||
results = []
|
||||
for testcase in testcases:
|
||||
command = (["timeout", "{timeout}s".format(args.timeout)] +
|
||||
command = (["timeout", "{}s".format(args.timeout)] +
|
||||
command_factory(testcase))
|
||||
for run in range(args.runs):
|
||||
print("Running test {}/{} on {}".format(run + 1,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ set -o errexit
|
|||
set -o nounset
|
||||
set -o pipefail
|
||||
|
||||
while (( "${#}" ))
|
||||
do
|
||||
case "${1}" in
|
||||
--servo)
|
||||
engine="--engine servo"
|
||||
|
|
@ -15,12 +17,20 @@ case "${1}" in
|
|||
--gecko)
|
||||
engine="--engine gecko"
|
||||
;;
|
||||
--submit)
|
||||
submit=1
|
||||
;;
|
||||
*)
|
||||
# This branch should never be reached with set -o nounset
|
||||
echo "You didn't specify the engine to run."
|
||||
echo "Unknown option ${1}."
|
||||
exit
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [[ -z "${engine:-}" ]];
|
||||
then echo "You didn't specify the engine to run: --servo or --gecko."; exit;
|
||||
fi
|
||||
|
||||
echo "Starting the local server"
|
||||
python3 -m http.server > /dev/null 2>&1 &
|
||||
|
|
@ -33,15 +43,19 @@ MANIFEST="page_load_test/tp5n/20160509.manifest" # A manifest that excludes
|
|||
PERF_FILE="output/perf-$(date --iso-8601=seconds).json"
|
||||
|
||||
echo "Running tests"
|
||||
python3 runner.py "${engine}" --runs 3 "${MANIFEST}" "${PERF_FILE}"
|
||||
python3 runner.py ${engine} --runs 3 "${MANIFEST}" "${PERF_FILE}"
|
||||
|
||||
echo "Submitting to Perfherder"
|
||||
# Perfherder SSL check will fail if time is not accurate,
|
||||
# sync time before you submit
|
||||
# TODO: we are using Servo's revision hash for Gecko's result to make both
|
||||
# results appear on the same date. Use the correct result when Perfherder
|
||||
# allows us to change the date.
|
||||
python3 submit_to_perfherder.py "${engine}" "${PERF_FILE}" servo/revision.json
|
||||
if [[ "${submit:-}" ]];
|
||||
then
|
||||
echo "Submitting to Perfherder"
|
||||
# Perfherder SSL check will fail if time is not accurate,
|
||||
# sync time before you submit
|
||||
# TODO: we are using Servo's revision hash for Gecko's result to make both
|
||||
# results appear on the same date. Use the correct result when Perfherder
|
||||
# allows us to change the date.
|
||||
python3 submit_to_perfherder.py \
|
||||
"${output:-}" "${engine}" "${PERF_FILE}" servo/revision.json
|
||||
fi
|
||||
|
||||
# Kill the http server
|
||||
echo "Stopping the local server"
|
||||
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
|
||||
|
|
|
|||
52
etc/ci/performance/test_differ.py
Normal file
52
etc/ci/performance/test_differ.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description="Diff between two runs of performance tests.")
|
||||
parser.add_argument("file1", help="the first output json from runner")
|
||||
parser.add_argument("file2", help="the second output json from runner")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def load_data(filename):
|
||||
with open(filename, 'r') as f:
|
||||
results = {}
|
||||
totals = {}
|
||||
counts = {}
|
||||
records = json.load(f)
|
||||
for record in records:
|
||||
key = record.get('testcase')
|
||||
value = record.get('domComplete') - record.get('domLoading')
|
||||
totals[key] = totals.get('key', 0) + value
|
||||
counts[key] = counts.get('key', 0) + 1
|
||||
results[key] = round(totals[key] / counts[key])
|
||||
return results
|
||||
|
||||
data1 = load_data(args.file1)
|
||||
data2 = load_data(args.file2)
|
||||
keys = set(data1.keys()).union(data2.keys())
|
||||
|
||||
BLUE = '\033[94m'
|
||||
GREEN = '\033[92m'
|
||||
WARNING = '\033[93m'
|
||||
END = '\033[0m'
|
||||
|
||||
for key in keys:
|
||||
value1 = data1.get(key)
|
||||
value2 = data2.get(key)
|
||||
if value1 and not(value2):
|
||||
print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file2, END))
|
||||
elif value2 and not(value1):
|
||||
print ("{}Test {}: missing from {}.{}".format(WARNING, key, args.file1, END))
|
||||
elif value1 and value2:
|
||||
diff = value2 - value1
|
||||
change = diff / value1
|
||||
color = BLUE if value1 <= value2 else GREEN
|
||||
print("{}{:6} {:6} {:+6} {:+8.2%} {}.{}".format(color, value1, value2, diff, change, key, END))
|
||||
|
|
@ -12,6 +12,7 @@ function formatLine(name, t) {
|
|||
function printPerfTiming() {
|
||||
print("[PERF] perf block start")
|
||||
formatLine("testcase", window.location);
|
||||
formatLine("title", document.title.replace(/,/g, ","));
|
||||
formatLine("navigationStart", performance.timing.navigationStart);
|
||||
formatLine("unloadEventStart", performance.timing.unloadEventStart);
|
||||
formatLine("unloadEventEnd", performance.timing.unloadEventEnd);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue