Auto merge of #13930 - shinglyu:perf-llvmpipe, r=aneeshusa

Use headless rendering for performance test

<!-- Please describe your changes on the following line: -->

Properly set the software rendering environment variables and use `-z` to run the performance test in headless mode. Also changed some logging format to improve the readability and reduce log size.
---

<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [x] These changes fix #13903 (github issue number if applicable).

<!-- Either: -->
- [ ] There are tests for these changes OR
- [x] These changes do not require tests because need manual test

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---

This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/13930)

<!-- Reviewable:end -->

etc/ci/performance/README.md

@@ -68,3 +68,12 @@ If you want to test the data submission code in `submit_to_perfherder.py` withou
 * Run `jpm xpi` in the `firefox/addon` folder
 * Install the generated `xpi` file to your Firefox Nightly
 
+# Troubleshooting
+
+ If you saw this error message:
+
+```
+venv/bin/activate: line 8: _OLD_VIRTUAL_PATH: unbound variable
+```
+
+That means your `virtualenv` is too old, try run `pip install -U virtualenv` to upgrade (If you installed ubuntu's `python-virtualenv` package, uninstall it first then install it through `pip`)

etc/ci/performance/runner.py

@@ -24,9 +24,6 @@ def parse_manifest(text):
 
 
 def execute_test(url, command, timeout):
-    print("Running test:")
-    print(' '.join(command))
-    print("Timeout:{}".format(timeout))
     try:
         return subprocess.check_output(command, stderr=subprocess.STDOUT, timeout=timeout)
     except subprocess.CalledProcessError as e:
@@ -35,7 +32,7 @@ def execute_test(url, command, timeout):
         print("You may want to re-run the test manually:\n{}"
               .format(' '.join(command)))
     except subprocess.TimeoutExpired:
-        print("Test timeout: {}".format(url))
+        print("Test FAILED due to timeout: {}".format(url))
     return ""
 
 
@@ -43,6 +40,7 @@ def get_servo_command(url):
     ua_script_path = "{}/user-agent-js".format(os.getcwd())
     return ["../../../target/release/servo", url,
             "--userscripts", ua_script_path,
+            "--headless",
             "-x", "-o", "output.png"]
 
 
@@ -246,9 +244,12 @@ def main():
                                                         args.runs,
                                                         testcase))
                 log = execute_test(testcase, command, args.timeout)
+                print("Finished")
                 result = parse_log(log, testcase)
                 # TODO: Record and analyze other performance.timing properties
                 results += result
+            print("To reproduce the above test, run the following command:")
+            print("    {0}\n".format(' '.join(command)))
 
         print(format_result_summary(results))
         save_result_json(results, args.output_file, testcases, args.runs)

etc/ci/performance/test_perf.sh

@@ -10,10 +10,21 @@ set -o pipefail
 
 TP5N_SOURCE="https://people.mozilla.org/~jmaher/taloszips/zips/tp5n.zip"
 TP5N_PATH="page_load_test/tp5n.zip"
-if [[ ! -f "${TP5N_PATH}" ]]; then
-    wget "${TP5N_SOURCE}" -O "${TP5N_PATH}"
+
+if [[ ! -f "$(dirname "${TP5N_PATH}")/tp5n/tp5n.manifest" ]]; then
+    if [[ ! -f "${TP5N_PATH}" ]]; then
+        echo "Downloading the test cases..."
+        wget "${TP5N_SOURCE}" -O "${TP5N_PATH}"
+        echo "done"
+    else
+        echo "Found existing test cases, skipping download."
+    fi
+    echo -n "Unzipping the test cases..."
+    unzip -q -o "${TP5N_PATH}" -d "$(dirname "${TP5N_PATH}")"
+    echo "done"
+else
+    echo "Found existing test cases, skipping download and unzip."
 fi
-unzip -o "${TP5N_PATH}" -d "$(dirname "${TP5N_PATH}")"
 
 virtualenv venv --python="$(which python3)"
 PS1="" source venv/bin/activate

python/servo/testing_commands.py

@@ -162,6 +162,8 @@ class MachCommands(CommandBase):
              description='Run the page load performance test',
              category='testing')
     def test_perf(self):
+        self.set_software_rendering_env(True)
+
         self.ensure_bootstrapped()
         env = self.build_env()
         return call(["bash", "test_perf.sh"],
@@ -422,31 +424,7 @@ class MachCommands(CommandBase):
 
     # Helper for test_css and test_wpt:
     def wptrunner(self, run_file, **kwargs):
-        # On Linux and mac, find the OSMesa software rendering library and
-        # add it to the dynamic linker search path.
-        if sys.platform.startswith('linux'):
-            try:
-                args = [self.get_binary_path(kwargs["release"], not kwargs["release"])]
-                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), "out", "lib", "gallium")
-                os.environ["LD_LIBRARY_PATH"] = osmesa_path
-                os.environ["GALLIUM_DRIVER"] = "softpipe"
-            except BuildNotFound:
-                # This can occur when cross compiling (e.g. arm64), in which case
-                # we won't run the tests anyway so can safely ignore this step.
-                pass
-        if sys.platform.startswith('darwin'):
-            try:
-                args = [self.get_binary_path(kwargs["release"], not kwargs["release"])]
-                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
-                                        "out", "src", "gallium", "targets", "osmesa", ".libs")
-                glapi_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
-                                       "out", "src", "mapi", "shared-glapi", ".libs")
-                os.environ["DYLD_LIBRARY_PATH"] = osmesa_path + ":" + glapi_path
-                os.environ["GALLIUM_DRIVER"] = "softpipe"
-            except BuildNotFound:
-                # This can occur when cross compiling (e.g. arm64), in which case
-                # we won't run the tests anyway so can safely ignore this step.
-                pass
+        self.set_software_rendering_env(kwargs['release'])
 
         os.environ["RUST_BACKTRACE"] = "1"
         kwargs["debug"] = not kwargs["release"]
@@ -658,6 +636,33 @@ class MachCommands(CommandBase):
         return check_call(
             [run_file, "|".join(tests), bin_path, base_dir])
 
+    def set_software_rendering_env(self, use_release):
+        # On Linux and mac, find the OSMesa software rendering library and
+        # add it to the dynamic linker search path.
+        if sys.platform.startswith('linux'):
+            try:
+                args = [self.get_binary_path(use_release, not use_release)]
+                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]), "out", "lib", "gallium")
+                os.environ["LD_LIBRARY_PATH"] = osmesa_path
+                os.environ["GALLIUM_DRIVER"] = "softpipe"
+            except BuildNotFound:
+                # This can occur when cross compiling (e.g. arm64), in which case
+                # we won't run the tests anyway so can safely ignore this step.
+                pass
+        elif sys.platform.startswith('darwin'):
+            try:
+                args = [self.get_binary_path(use_release, not use_release)]
+                osmesa_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
+                                        "out", "src", "gallium", "targets", "osmesa", ".libs")
+                glapi_path = path.join(find_dep_path_newest('osmesa-src', args[0]),
+                                       "out", "src", "mapi", "shared-glapi", ".libs")
+                os.environ["DYLD_LIBRARY_PATH"] = osmesa_path + ":" + glapi_path
+                os.environ["GALLIUM_DRIVER"] = "softpipe"
+            except BuildNotFound:
+                # This can occur when cross compiling (e.g. arm64), in which case
+                # we won't run the tests anyway so can safely ignore this step.
+                pass
+
 
 def create_parser_create():
     import argparse