mirror of
https://github.com/servo/servo.git
synced 2025-06-06 16:45:39 +00:00
Bug 10452 - Page Load Time Test Runner
This commit is contained in:
Shing Lyu
parent
a83fed2144
commit
13b8f2531c
15 changed files with 1485 additions and 0 deletions
445
etc/ci/performance/test_runner.py
Normal file
445
etc/ci/performance/test_runner.py
Normal file
|
|
@ -0,0 +1,445 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import runner
|
||||
import pytest
|
||||
|
||||
|
||||
def test_log_parser():
|
||||
mock_log = b'''
|
||||
[PERF] perf block start
|
||||
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
|
||||
[PERF],navigationStart,1460358376
|
||||
[PERF],unloadEventStart,undefined
|
||||
[PERF],unloadEventEnd,undefined
|
||||
[PERF],redirectStart,undefined
|
||||
[PERF],redirectEnd,undefined
|
||||
[PERF],fetchStart,undefined
|
||||
[PERF],domainLookupStart,undefined
|
||||
[PERF],domainLookupEnd,undefined
|
||||
[PERF],connectStart,undefined
|
||||
[PERF],connectEnd,undefined
|
||||
[PERF],secureConnectionStart,undefined
|
||||
[PERF],requestStart,undefined
|
||||
[PERF],responseStart,undefined
|
||||
[PERF],responseEnd,undefined
|
||||
[PERF],domLoading,1460358376000
|
||||
[PERF],domInteractive,1460358388000
|
||||
[PERF],domContentLoadedEventStart,1460358388000
|
||||
[PERF],domContentLoadedEventEnd,1460358388000
|
||||
[PERF],domComplete,1460358389000
|
||||
[PERF],loadEventStart,undefined
|
||||
[PERF],loadEventEnd,undefined
|
||||
[PERF] perf block end
|
||||
Shutting down the Constellation after generating an output file or exit flag specified
|
||||
'''
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"navigationStart": 1460358376,
|
||||
"unloadEventStart": None,
|
||||
"unloadEventEnd": None,
|
||||
"redirectStart": None,
|
||||
"redirectEnd": None,
|
||||
"fetchStart": None,
|
||||
"domainLookupStart": None,
|
||||
"domainLookupEnd": None,
|
||||
"connectStart": None,
|
||||
"connectEnd": None,
|
||||
"secureConnectionStart": None,
|
||||
"requestStart": None,
|
||||
"responseStart": None,
|
||||
"responseEnd": None,
|
||||
"domLoading": 1460358376000,
|
||||
"domInteractive": 1460358388000,
|
||||
"domContentLoadedEventStart": 1460358388000,
|
||||
"domContentLoadedEventEnd": 1460358388000,
|
||||
"domComplete": 1460358389000,
|
||||
"loadEventStart": None,
|
||||
"loadEventEnd": None
|
||||
}]
|
||||
result = runner.parse_log(mock_log)
|
||||
assert(expected == list(result))
|
||||
|
||||
|
||||
def test_log_parser_complex():
|
||||
mock_log = b'''
|
||||
[PERF] perf block start
|
||||
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/content.html
|
||||
[PERF],navigationStart,1460358300
|
||||
[PERF],unloadEventStart,undefined
|
||||
[PERF],unloadEventEnd,undefined
|
||||
[PERF],redirectStart,undefined
|
||||
[PERF],redirectEnd,undefined
|
||||
[PERF],fetchStart,undefined
|
||||
[PERF],domainLookupStart,undefined
|
||||
[PERF],domainLookupEnd,undefined
|
||||
[PERF],connectStart,undefined
|
||||
[PERF],connectEnd,undefined
|
||||
[PERF],secureConnectionStart,undefined
|
||||
[PERF],requestStart,undefined
|
||||
[PERF],responseStart,undefined
|
||||
[PERF],responseEnd,undefined
|
||||
[PERF],domLoading,1460358376000
|
||||
[PERF],domInteractive,1460358388000
|
||||
[PERF],domContentLoadedEventStart,1460358388000
|
||||
[PERF],domContentLoadedEventEnd,1460358388000
|
||||
[PERF],domComplete,1460358389000
|
||||
[PERF],loadEventStart,undefined
|
||||
[PERF],loadEventEnd,undefined
|
||||
[PERF] perf block end
|
||||
Some other js error logs here
|
||||
|
||||
[PERF] perf block start
|
||||
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
|
||||
[PERF],navigationStart,1460358376
|
||||
[PERF],unloadEventStart,undefined
|
||||
[PERF],unloadEventEnd,undefined
|
||||
[PERF],redirectStart,undefined
|
||||
[PERF],redirectEnd,undefined
|
||||
[PERF],fetchStart,undefined
|
||||
[PERF],domainLookupStart,undefined
|
||||
[PERF],domainLookupEnd,undefined
|
||||
[PERF],connectStart,undefined
|
||||
[PERF],connectEnd,undefined
|
||||
[PERF],secureConnectionStart,undefined
|
||||
[PERF],requestStart,undefined
|
||||
[PERF],responseStart,undefined
|
||||
[PERF],responseEnd,undefined
|
||||
[PERF],domLoading,1460358376000
|
||||
[PERF],domInteractive,1460358388000
|
||||
[PERF],domContentLoadedEventStart,1460358388000
|
||||
[PERF],domContentLoadedEventEnd,1460358388000
|
||||
[PERF],domComplete,1460358389000
|
||||
[PERF],loadEventStart,undefined
|
||||
[PERF],loadEventEnd,undefined
|
||||
[PERF] perf block end
|
||||
Shutting down the Constellation after generating an output file or exit flag specified
|
||||
'''
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/content.html",
|
||||
"navigationStart": 1460358300,
|
||||
"unloadEventStart": None,
|
||||
"unloadEventEnd": None,
|
||||
"redirectStart": None,
|
||||
"redirectEnd": None,
|
||||
"fetchStart": None,
|
||||
"domainLookupStart": None,
|
||||
"domainLookupEnd": None,
|
||||
"connectStart": None,
|
||||
"connectEnd": None,
|
||||
"secureConnectionStart": None,
|
||||
"requestStart": None,
|
||||
"responseStart": None,
|
||||
"responseEnd": None,
|
||||
"domLoading": 1460358376000,
|
||||
"domInteractive": 1460358388000,
|
||||
"domContentLoadedEventStart": 1460358388000,
|
||||
"domContentLoadedEventEnd": 1460358388000,
|
||||
"domComplete": 1460358389000,
|
||||
"loadEventStart": None,
|
||||
"loadEventEnd": None
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"navigationStart": 1460358376,
|
||||
"unloadEventStart": None,
|
||||
"unloadEventEnd": None,
|
||||
"redirectStart": None,
|
||||
"redirectEnd": None,
|
||||
"fetchStart": None,
|
||||
"domainLookupStart": None,
|
||||
"domainLookupEnd": None,
|
||||
"connectStart": None,
|
||||
"connectEnd": None,
|
||||
"secureConnectionStart": None,
|
||||
"requestStart": None,
|
||||
"responseStart": None,
|
||||
"responseEnd": None,
|
||||
"domLoading": 1460358376000,
|
||||
"domInteractive": 1460358388000,
|
||||
"domContentLoadedEventStart": 1460358388000,
|
||||
"domContentLoadedEventEnd": 1460358388000,
|
||||
"domComplete": 1460358389000,
|
||||
"loadEventStart": None,
|
||||
"loadEventEnd": None
|
||||
}]
|
||||
result = runner.parse_log(mock_log)
|
||||
assert(expected == list(result))
|
||||
|
||||
|
||||
def test_log_parser_empty():
|
||||
mock_log = b'''
|
||||
[PERF] perf block start
|
||||
[PERF]BROKEN!!!!!!!!!1
|
||||
[PERF]BROKEN!!!!!!!!!1
|
||||
[PERF]BROKEN!!!!!!!!!1
|
||||
[PERF]BROKEN!!!!!!!!!1
|
||||
[PERF]BROKEN!!!!!!!!!1
|
||||
[PERF] perf block end
|
||||
'''
|
||||
mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"navigationStart": 0,
|
||||
"unloadEventStart": -1,
|
||||
"unloadEventEnd": -1,
|
||||
"redirectStart": -1,
|
||||
"redirectEnd": -1,
|
||||
"fetchStart": -1,
|
||||
"domainLookupStart": -1,
|
||||
"domainLookupEnd": -1,
|
||||
"connectStart": -1,
|
||||
"connectEnd": -1,
|
||||
"secureConnectionStart": -1,
|
||||
"requestStart": -1,
|
||||
"responseStart": -1,
|
||||
"responseEnd": -1,
|
||||
"domLoading": -1,
|
||||
"domInteractive": -1,
|
||||
"domContentLoadedEventStart": -1,
|
||||
"domContentLoadedEventEnd": -1,
|
||||
"domComplete": -1,
|
||||
"loadEventStart": -1,
|
||||
"loadEventEnd": -1
|
||||
}]
|
||||
result = runner.parse_log(mock_log, mock_testcase)
|
||||
assert(expected == list(result))
|
||||
|
||||
|
||||
def test_log_parser_error():
|
||||
mock_log = b'Nothing here! Test failed!'
|
||||
mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"navigationStart": 0,
|
||||
"unloadEventStart": -1,
|
||||
"unloadEventEnd": -1,
|
||||
"redirectStart": -1,
|
||||
"redirectEnd": -1,
|
||||
"fetchStart": -1,
|
||||
"domainLookupStart": -1,
|
||||
"domainLookupEnd": -1,
|
||||
"connectStart": -1,
|
||||
"connectEnd": -1,
|
||||
"secureConnectionStart": -1,
|
||||
"requestStart": -1,
|
||||
"responseStart": -1,
|
||||
"responseEnd": -1,
|
||||
"domLoading": -1,
|
||||
"domInteractive": -1,
|
||||
"domContentLoadedEventStart": -1,
|
||||
"domContentLoadedEventEnd": -1,
|
||||
"domComplete": -1,
|
||||
"loadEventStart": -1,
|
||||
"loadEventEnd": -1
|
||||
}]
|
||||
result = runner.parse_log(mock_log, mock_testcase)
|
||||
assert(expected == list(result))
|
||||
|
||||
|
||||
def test_log_parser_bad_testcase_name():
|
||||
mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
|
||||
# Notice the testcase is about:blank, servo crashed
|
||||
mock_log = b'''
|
||||
[PERF] perf block start
|
||||
[PERF],testcase,about:blank
|
||||
[PERF],navigationStart,1460358376
|
||||
[PERF],unloadEventStart,undefined
|
||||
[PERF],unloadEventEnd,undefined
|
||||
[PERF],redirectStart,undefined
|
||||
[PERF],redirectEnd,undefined
|
||||
[PERF],fetchStart,undefined
|
||||
[PERF],domainLookupStart,undefined
|
||||
[PERF],domainLookupEnd,undefined
|
||||
[PERF],connectStart,undefined
|
||||
[PERF],connectEnd,undefined
|
||||
[PERF],secureConnectionStart,undefined
|
||||
[PERF],requestStart,undefined
|
||||
[PERF],responseStart,undefined
|
||||
[PERF],responseEnd,undefined
|
||||
[PERF],domLoading,1460358376000
|
||||
[PERF],domInteractive,1460358388000
|
||||
[PERF],domContentLoadedEventStart,1460358388000
|
||||
[PERF],domContentLoadedEventEnd,1460358388000
|
||||
[PERF],domComplete,1460358389000
|
||||
[PERF],loadEventStart,undefined
|
||||
[PERF],loadEventEnd,undefined
|
||||
[PERF] perf block end
|
||||
Shutting down the Constellation after generating an output file or exit flag specified
|
||||
'''
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"navigationStart": 0,
|
||||
"unloadEventStart": -1,
|
||||
"unloadEventEnd": -1,
|
||||
"redirectStart": -1,
|
||||
"redirectEnd": -1,
|
||||
"fetchStart": -1,
|
||||
"domainLookupStart": -1,
|
||||
"domainLookupEnd": -1,
|
||||
"connectStart": -1,
|
||||
"connectEnd": -1,
|
||||
"secureConnectionStart": -1,
|
||||
"requestStart": -1,
|
||||
"responseStart": -1,
|
||||
"responseEnd": -1,
|
||||
"domLoading": -1,
|
||||
"domInteractive": -1,
|
||||
"domContentLoadedEventStart": -1,
|
||||
"domContentLoadedEventEnd": -1,
|
||||
"domComplete": -1,
|
||||
"loadEventStart": -1,
|
||||
"loadEventEnd": -1
|
||||
}]
|
||||
result = runner.parse_log(mock_log, mock_testcase)
|
||||
assert(expected == list(result))
|
||||
|
||||
|
||||
def test_manifest_loader():
|
||||
|
||||
text = '''
|
||||
http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html
|
||||
http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html
|
||||
|
||||
http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
|
||||
# Disabled! http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
|
||||
'''
|
||||
expected = [
|
||||
"http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html",
|
||||
"http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html",
|
||||
"http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html"
|
||||
]
|
||||
assert(expected == list(runner.parse_manifest(text)))
|
||||
|
||||
|
||||
def test_filter_result_by_manifest():
|
||||
input_json = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/content.html",
|
||||
"domComplete": 1460358389000,
|
||||
}, {
|
||||
"testcase": "non-existing-html",
|
||||
"domComplete": 1460358389000,
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389000,
|
||||
}]
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389000,
|
||||
}]
|
||||
|
||||
manifest = [
|
||||
"http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
]
|
||||
|
||||
assert(expected == runner.filter_result_by_manifest(input_json, manifest))
|
||||
|
||||
|
||||
def test_filter_result_by_manifest_error():
|
||||
input_json = [{
|
||||
"testcase": "1.html",
|
||||
"domComplete": 1460358389000,
|
||||
}]
|
||||
|
||||
manifest = [
|
||||
"1.html",
|
||||
"2.html"
|
||||
]
|
||||
|
||||
with pytest.raises(Exception) as execinfo:
|
||||
runner.filter_result_by_manifest(input_json, manifest)
|
||||
assert "Missing test result" in str(execinfo.value)
|
||||
|
||||
|
||||
def test_take_result_median_odd():
|
||||
input_json = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389001,
|
||||
"domLoading": 1460358380002
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389002,
|
||||
"domLoading": 1460358380001
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389003,
|
||||
"domLoading": 1460358380003
|
||||
}]
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389002,
|
||||
"domLoading": 1460358380002
|
||||
}]
|
||||
|
||||
assert(expected == runner.take_result_median(input_json, len(input_json)))
|
||||
|
||||
|
||||
def test_take_result_median_even():
|
||||
input_json = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389001,
|
||||
"domLoading": 1460358380002
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389002,
|
||||
"domLoading": 1460358380001
|
||||
}]
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389001.5,
|
||||
"domLoading": 1460358380001.5
|
||||
}]
|
||||
|
||||
assert(expected == runner.take_result_median(input_json, len(input_json)))
|
||||
|
||||
|
||||
def test_take_result_median_error():
|
||||
input_json = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": None,
|
||||
"domLoading": 1460358380002
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389002,
|
||||
"domLoading": 1460358380001
|
||||
}]
|
||||
|
||||
expected = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": 1460358389002,
|
||||
"domLoading": 1460358380001.5
|
||||
}]
|
||||
|
||||
assert(expected == runner.take_result_median(input_json, len(input_json)))
|
||||
|
||||
|
||||
def test_log_result():
|
||||
results = [{
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": -1
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
|
||||
"domComplete": -1
|
||||
}, {
|
||||
"testcase": "http://localhost:8000/page_load_test/104.com/www.104.com/index.html",
|
||||
"domComplete": 123456789
|
||||
}]
|
||||
|
||||
expected = """
|
||||
========================================
|
||||
Total 3 tests; 1 succeeded, 2 failed.
|
||||
|
||||
Failure summary:
|
||||
- http://localhost:8000/page_load_test/56.com/www.56.com/index.html
|
||||
========================================
|
||||
"""
|
||||
assert(expected == runner.format_result_summary(results))
|
||||
Loading…
Add table
Add a link
Reference in a new issue