From d941c5b91673d006304d90f792930d6c107dbef6 Mon Sep 17 00:00:00 2001 From: Shing Lyu Date: Fri, 28 Oct 2016 15:03:32 +0800 Subject: [PATCH 1/3] Cached tp5 zip downloading and unzipping --- etc/ci/performance/test_perf.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/etc/ci/performance/test_perf.sh b/etc/ci/performance/test_perf.sh index 87897ee3ef9..4b239b8831f 100755 --- a/etc/ci/performance/test_perf.sh +++ b/etc/ci/performance/test_perf.sh @@ -10,10 +10,21 @@ set -o pipefail TP5N_SOURCE="https://people.mozilla.org/~jmaher/taloszips/zips/tp5n.zip" TP5N_PATH="page_load_test/tp5n.zip" -if [[ ! -f "${TP5N_PATH}" ]]; then - wget "${TP5N_SOURCE}" -O "${TP5N_PATH}" + +if [[ ! -f "$(dirname "${TP5N_PATH}")/tp5n/tp5n.manifest" ]]; then + if [[ ! -f "${TP5N_PATH}" ]]; then + echo "Downloading the test cases..." + wget "${TP5N_SOURCE}" -O "${TP5N_PATH}" + echo "done" + else + echo "Found existing test cases, skipping download." + fi + echo -n "Unzipping the test cases..." + unzip -q -o "${TP5N_PATH}" -d "$(dirname "${TP5N_PATH}")" + echo "done" +else + echo "Found existing test cases, skipping download and unzip." fi -unzip -o "${TP5N_PATH}" -d "$(dirname "${TP5N_PATH}")" virtualenv venv --python="$(which python3)" PS1="" source venv/bin/activate From 9aaa0886240aa4863ae2f8620932ecf6084f4f1f Mon Sep 17 00:00:00 2001 From: Shing Lyu Date: Fri, 28 Oct 2016 15:05:07 +0800 Subject: [PATCH 2/3] Use headless rendering for performance test --- etc/ci/performance/runner.py | 1 + python/servo/testing_commands.py | 55 +++++++++++++++++--------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index ab5a31c4a6d..a7014e24f28 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -43,6 +43,7 @@ def get_servo_command(url): ua_script_path = "{}/user-agent-js".format(os.getcwd()) return ["../../../target/release/servo", url, "--userscripts", ua_script_path, + "--headless", "-x", "-o", "output.png"] diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 88a6d7b24ca..fdb471a437d 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -162,6 +162,8 @@ class MachCommands(CommandBase): description='Run the page load performance test', category='testing') def test_perf(self): + self.set_software_rendering_env(True) + self.ensure_bootstrapped() env = self.build_env() return call(["bash", "test_perf.sh"], @@ -422,31 +424,7 @@ class MachCommands(CommandBase): # Helper for test_css and test_wpt: def wptrunner(self, run_file, **kwargs): - # On Linux and mac, find the OSMesa software rendering library and - # add it to the dynamic linker search path. - if sys.platform.startswith('linux'): - try: - args = [self.get_binary_path(kwargs["release"], not kwargs["release"])] - osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), "out", "lib", "gallium") - os.environ["LD_LIBRARY_PATH"] = osmesa_path - os.environ["GALLIUM_DRIVER"] = "softpipe" - except BuildNotFound: - # This can occur when cross compiling (e.g. arm64), in which case - # we won't run the tests anyway so can safely ignore this step. - pass - if sys.platform.startswith('darwin'): - try: - args = [self.get_binary_path(kwargs["release"], not kwargs["release"])] - osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), - "out", "src", "gallium", "targets", "osmesa", ".libs") - glapi_path = path.join(find_dep_path_newest('osmesa-src', args[0]), - "out", "src", "mapi", "shared-glapi", ".libs") - os.environ["DYLD_LIBRARY_PATH"] = osmesa_path + ":" + glapi_path - os.environ["GALLIUM_DRIVER"] = "softpipe" - except BuildNotFound: - # This can occur when cross compiling (e.g. arm64), in which case - # we won't run the tests anyway so can safely ignore this step. - pass + self.set_software_rendering_env(kwargs['release']) os.environ["RUST_BACKTRACE"] = "1" kwargs["debug"] = not kwargs["release"] @@ -658,6 +636,33 @@ class MachCommands(CommandBase): return check_call( [run_file, "|".join(tests), bin_path, base_dir]) + def set_software_rendering_env(self, use_release): + # On Linux and mac, find the OSMesa software rendering library and + # add it to the dynamic linker search path. + if sys.platform.startswith('linux'): + try: + args = [self.get_binary_path(use_release, not use_release)] + osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), "out", "lib", "gallium") + os.environ["LD_LIBRARY_PATH"] = osmesa_path + os.environ["GALLIUM_DRIVER"] = "softpipe" + except BuildNotFound: + # This can occur when cross compiling (e.g. arm64), in which case + # we won't run the tests anyway so can safely ignore this step. + pass + elif sys.platform.startswith('darwin'): + try: + args = [self.get_binary_path(use_release, not use_release)] + osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), + "out", "src", "gallium", "targets", "osmesa", ".libs") + glapi_path = path.join(find_dep_path_newest('osmesa-src', args[0]), + "out", "src", "mapi", "shared-glapi", ".libs") + os.environ["DYLD_LIBRARY_PATH"] = osmesa_path + ":" + glapi_path + os.environ["GALLIUM_DRIVER"] = "softpipe" + except BuildNotFound: + # This can occur when cross compiling (e.g. arm64), in which case + # we won't run the tests anyway so can safely ignore this step. + pass + def create_parser_create(): import argparse From 2407fe614be6eb7ca8fddb2c666d3bb3b672dcf8 Mon Sep 17 00:00:00 2001 From: Shing Lyu Date: Fri, 28 Oct 2016 15:05:32 +0800 Subject: [PATCH 3/3] Log and readme improvements --- etc/ci/performance/README.md | 9 +++++++++ etc/ci/performance/runner.py | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/etc/ci/performance/README.md b/etc/ci/performance/README.md index bd48ab6d72b..789236d3c08 100644 --- a/etc/ci/performance/README.md +++ b/etc/ci/performance/README.md @@ -68,3 +68,12 @@ If you want to test the data submission code in `submit_to_perfherder.py` withou * Run `jpm xpi` in the `firefox/addon` folder * Install the generated `xpi` file to your Firefox Nightly +# Troubleshooting + + If you saw this error message: + +``` +venv/bin/activate: line 8: _OLD_VIRTUAL_PATH: unbound variable +``` + +That means your `virtualenv` is too old, try run `pip install -U virtualenv` to upgrade (If you installed ubuntu's `python-virtualenv` package, uninstall it first then install it through `pip`) diff --git a/etc/ci/performance/runner.py b/etc/ci/performance/runner.py index a7014e24f28..81c4ac6c33b 100644 --- a/etc/ci/performance/runner.py +++ b/etc/ci/performance/runner.py @@ -24,9 +24,6 @@ def parse_manifest(text): def execute_test(url, command, timeout): - print("Running test:") - print(' '.join(command)) - print("Timeout:{}".format(timeout)) try: return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout) except subprocess.CalledProcessError as e: @@ -35,7 +32,7 @@ def execute_test(url, command, timeout): print("You may want to re-run the test manually:\n{}" .format(' '.join(command))) except subprocess.TimeoutExpired: - print("Test timeout: {}".format(url)) + print("Test FAILED due to timeout: {}".format(url)) return "" @@ -247,9 +244,12 @@ def main(): args.runs, testcase)) log = execute_test(testcase, command, args.timeout) + print("Finished") result = parse_log(log, testcase) # TODO: Record and analyze other performance.timing properties results += result + print("To reproduce the above test, run the following command:") + print(" {0}\n".format(' '.join(command))) print(format_result_summary(results)) save_result_json(results, args.output_file, testcases, args.runs)