Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-04 07:35:36 +00:00
Code Issues Projects Releases Packages Wiki Activity
servo/etc/ci/performance/test_runner.py
zefr0x c96de69e80
Use ruff to enforce python code formatting (#37117) 
Requires servo/servo#37045 for deps and config.

Testing: No need for tests to test tests.
Fixes: servo/servo#37041

---------

Signed-off-by: zefr0x <zer0-x.7ty50@aleeas.com>
2025-05-26 11:54:43 +00:00

462 lines
15 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import runner
import pytest
def test_log_parser():
mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
mock_log = b"""
[PERF] perf block start
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
[PERF],navigationStart,1460358376
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Shutting down the Constellation after generating an output file or exit flag specified
"""
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"navigationStart": 1460358376,
"unloadEventStart": None,
"unloadEventEnd": None,
"redirectStart": None,
"redirectEnd": None,
"fetchStart": None,
"domainLookupStart": None,
"domainLookupEnd": None,
"connectStart": None,
"connectEnd": None,
"secureConnectionStart": None,
"requestStart": None,
"responseStart": None,
"responseEnd": None,
"domLoading": 1460358376000,
"domInteractive": 1460358388000,
"domContentLoadedEventStart": 1460358388000,
"domContentLoadedEventEnd": 1460358388000,
"domComplete": 1460358389000,
"loadEventStart": None,
"loadEventEnd": None,
}
]
result = runner.parse_log(mock_log, mock_url)
assert expected == list(result)
def test_log_parser_complex():
mock_log = b"""
[PERF] perf block start
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/content.html
[PERF],navigationStart,1460358300
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Some other js error logs here
[PERF] perf block start
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
[PERF],navigationStart,1460358376
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Shutting down the Constellation after generating an output file or exit flag specified
"""
mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"navigationStart": 1460358376,
"unloadEventStart": None,
"unloadEventEnd": None,
"redirectStart": None,
"redirectEnd": None,
"fetchStart": None,
"domainLookupStart": None,
"domainLookupEnd": None,
"connectStart": None,
"connectEnd": None,
"secureConnectionStart": None,
"requestStart": None,
"responseStart": None,
"responseEnd": None,
"domLoading": 1460358376000,
"domInteractive": 1460358388000,
"domContentLoadedEventStart": 1460358388000,
"domContentLoadedEventEnd": 1460358388000,
"domComplete": 1460358389000,
"loadEventStart": None,
"loadEventEnd": None,
}
]
result = runner.parse_log(mock_log, mock_url)
assert expected == list(result)
def test_log_parser_empty():
mock_log = b"""
[PERF] perf block start
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF] perf block end
"""
mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"title": "",
"navigationStart": 0,
"unloadEventStart": -1,
"unloadEventEnd": -1,
"redirectStart": -1,
"redirectEnd": -1,
"fetchStart": -1,
"domainLookupStart": -1,
"domainLookupEnd": -1,
"connectStart": -1,
"connectEnd": -1,
"secureConnectionStart": -1,
"requestStart": -1,
"responseStart": -1,
"responseEnd": -1,
"domLoading": -1,
"domInteractive": -1,
"domContentLoadedEventStart": -1,
"domContentLoadedEventEnd": -1,
"domComplete": -1,
"loadEventStart": -1,
"loadEventEnd": -1,
}
]
result = runner.parse_log(mock_log, mock_testcase)
assert expected == list(result)
def test_log_parser_error():
mock_log = b"Nothing here! Test failed!"
mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"title": "",
"navigationStart": 0,
"unloadEventStart": -1,
"unloadEventEnd": -1,
"redirectStart": -1,
"redirectEnd": -1,
"fetchStart": -1,
"domainLookupStart": -1,
"domainLookupEnd": -1,
"connectStart": -1,
"connectEnd": -1,
"secureConnectionStart": -1,
"requestStart": -1,
"responseStart": -1,
"responseEnd": -1,
"domLoading": -1,
"domInteractive": -1,
"domContentLoadedEventStart": -1,
"domContentLoadedEventEnd": -1,
"domComplete": -1,
"loadEventStart": -1,
"loadEventEnd": -1,
}
]
result = runner.parse_log(mock_log, mock_testcase)
assert expected == list(result)
def test_log_parser_bad_testcase_name():
mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
# Notice the testcase is about:blank, servo crashed
mock_log = b"""
[PERF] perf block start
[PERF],testcase,about:blank
[PERF],navigationStart,1460358376
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Shutting down the Constellation after generating an output file or exit flag specified
"""
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"title": "",
"navigationStart": 0,
"unloadEventStart": -1,
"unloadEventEnd": -1,
"redirectStart": -1,
"redirectEnd": -1,
"fetchStart": -1,
"domainLookupStart": -1,
"domainLookupEnd": -1,
"connectStart": -1,
"connectEnd": -1,
"secureConnectionStart": -1,
"requestStart": -1,
"responseStart": -1,
"responseEnd": -1,
"domLoading": -1,
"domInteractive": -1,
"domContentLoadedEventStart": -1,
"domContentLoadedEventEnd": -1,
"domComplete": -1,
"loadEventStart": -1,
"loadEventEnd": -1,
}
]
result = runner.parse_log(mock_log, mock_testcase)
assert expected == list(result)
def test_manifest_loader():
text = """
http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html
http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html
http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
# Disabled! http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
"""
expected = [
("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False),
("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", False),
("http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html", False),
]
assert expected == list(runner.parse_manifest(text))
def test_manifest_loader_async():
text = """
http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html
async http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html
"""
expected = [
("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False),
("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", True),
]
assert expected == list(runner.parse_manifest(text))
def test_filter_result_by_manifest():
input_json = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/content.html",
"domComplete": 1460358389000,
},
{
"testcase": "non-existing-html",
"domComplete": 1460358389000,
},
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389000,
},
]
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389000,
}
]
manifest = [("http://localhost:8000/page_load_test/56.com/www.56.com/index.html", False)]
assert expected == runner.filter_result_by_manifest(input_json, manifest)
def test_filter_result_by_manifest_error():
input_json = [
{
"testcase": "1.html",
"domComplete": 1460358389000,
}
]
manifest = [("1.html", False), ("2.html", False)]
with pytest.raises(Exception) as execinfo:
runner.filter_result_by_manifest(input_json, manifest)
assert "Missing test result" in str(execinfo.value)
def test_take_result_median_odd():
input_json = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389001,
"domLoading": 1460358380002,
},
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389002,
"domLoading": 1460358380001,
},
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389003,
"domLoading": 1460358380003,
},
]
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389002,
"domLoading": 1460358380002,
}
]
assert expected == runner.take_result_median(input_json, len(input_json))
def test_take_result_median_even():
input_json = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389001,
"domLoading": 1460358380002,
},
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389002,
"domLoading": 1460358380001,
},
]
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389001.5,
"domLoading": 1460358380001.5,
}
]
assert expected == runner.take_result_median(input_json, len(input_json))
def test_take_result_median_error():
input_json = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": None,
"domLoading": 1460358380002,
},
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389002,
"domLoading": 1460358380001,
},
]
expected = [
{
"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
"domComplete": 1460358389002,
"domLoading": 1460358380001.5,
}
]
assert expected == runner.take_result_median(input_json, len(input_json))
def test_log_result():
results = [
{"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", "domComplete": -1},
{"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", "domComplete": -1},
{"testcase": "http://localhost:8000/page_load_test/104.com/www.104.com/index.html", "domComplete": 123456789},
]
expected = """
========================================
Total 3 tests; 1 succeeded, 2 failed.
Failure summary:
- http://localhost:8000/page_load_test/56.com/www.56.com/index.html
========================================
"""
assert expected == runner.format_result_summary(results)
Reference in a new issue View git blame Copy permalink