From ff217106d33a248b6048c8fa460bfce5b1b4805c Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Tue, 28 Jul 2015 16:52:02 -0700 Subject: [PATCH 1/6] First try at Dromaeo test runner. --- tests/dromaeo/run_dromaeo.py | 118 +++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100755 tests/dromaeo/run_dromaeo.py diff --git a/tests/dromaeo/run_dromaeo.py b/tests/dromaeo/run_dromaeo.py new file mode 100755 index 00000000000..1765cb6d18f --- /dev/null +++ b/tests/dromaeo/run_dromaeo.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess +import sys +import BaseHTTPServer +import SimpleHTTPServer +import SocketServer +import threading +import urlparse + +# Port to run the HTTP server on for Dromaeo. +TEST_SERVER_PORT = 8192 + +# Run servo and print / parse the results for a specific jQuery test module. +def run_servo(servo_exe): + url = "http://localhost:{0}/?recommended&automated&post_json".format(TEST_SERVER_PORT) + #args = [servo_exe, url, "-z", "-f"] + args = [servo_exe, url, "-z"] + return subprocess.Popen(args) + +# Print usage if command line args are incorrect +def print_usage(): + print("USAGE: {0} test servo_binary dromaeo_base_dir".format(sys.argv[0])) + + +# A simple HTTP server to serve up the test suite +class Server(SocketServer.ThreadingMixIn, + BaseHTTPServer.HTTPServer): + allow_reuse_address = True + got_post = False + +class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + # TODO(gw): HACK copy the fixed version from python + # main repo - due to https://bugs.python.org/issue23112 + def send_head(self): + path = self.translate_path(self.path) + f = None + if os.path.isdir(path): + parts = urlparse.urlsplit(self.path) + if not parts.path.endswith('/'): + # redirect browser - doing basically what apache does + self.send_response(301) + new_parts = (parts[0], parts[1], parts[2] + '/', + parts[3], parts[4]) + new_url = urlparse.urlunsplit(new_parts) + self.send_header("Location", new_url) + self.end_headers() + return None + for index in "index.html", "index.htm": + index = os.path.join(path, index) + if os.path.exists(index): + path = index + break + else: + return self.list_directory(path) + ctype = self.guess_type(path) + try: + # Always read in binary mode. Opening files in text mode may cause + # newline translations, making the actual size of the content + # transmitted *less* than the content-length! + f = open(path, 'rb') + except IOError: + self.send_error(404, "File not found") + return None + try: + self.send_response(200) + self.send_header("Content-type", ctype) + fs = os.fstat(f.fileno()) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + return f + except: + f.close() + raise + def do_POST(self): + self.send_response(200) + self.end_headers() + self.wfile.write("POST OK.

"); + length = int(self.headers.getheader('content-length')) + parameters = urlparse.parse_qs(self.rfile.read(length)) + self.server.got_post = True + self.server.post_data = parameters['data'] + def log_message(self, format, *args): + return + +if __name__ == '__main__': + if len(sys.argv) == 4: + cmd = sys.argv[1] + servo_exe = sys.argv[2] + base_dir = sys.argv[3] + os.chdir(base_dir) + + # Ensure servo binary can be found + if not os.path.isfile(servo_exe): + print("Unable to find {0}. This script expects an existing build of Servo.".format(servo_exe)) + sys.exit(1) + + # Start the test server + server = Server(('', TEST_SERVER_PORT), RequestHandler) + + if cmd == "test": + print("Testing Dromaeo on Servo!") + proc = run_servo(servo_exe) + while not server.got_post: + server.handle_request() + print("dromaeo: %s" % server.post_data) + proc.kill() + else: + print_usage() + else: + print_usage() From c602af20505b9b97cc8148be475ca7f93688b317 Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Wed, 29 Jul 2015 11:04:42 -0700 Subject: [PATCH 2/6] Integrate dromaeo with mach. --- python/servo/testing_commands.py | 36 ++++++++++++++++++++++++++++++++ tests/dromaeo/run_dromaeo.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 4f39c702edd..0f17274bac0 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -271,6 +271,16 @@ class MachCommands(CommandBase): def test_jquery(self, release, dev): return self.jquery_test_runner("test", release, dev) + @Command('test-dromaeo', + description='Run the Dromaeo test suite', + category='testing') + @CommandArgument('--release', '-r', action='store_true', + help='Run the release build') + @CommandArgument('--dev', '-d', action='store_true', + help='Run the dev build') + def test_dromaeo(self, release, dev): + return self.dromaeo_test_runner("test", release, dev) + @Command('update-jquery', description='Update the jQuery test suite expected results', category='testing') @@ -367,3 +377,29 @@ class MachCommands(CommandBase): return subprocess.check_call( [run_file, cmd, bin_path, base_dir]) + + def dromaeo_test_runner(self, cmd, release, dev): + self.ensure_bootstrapped() + base_dir = path.abspath(path.join("tests", "dromaeo")) + dromaeo_dir = path.join(base_dir, "dromaeo") + run_file = path.join(base_dir, "run_dromaeo.py") + + # Clone the Dromaeo repository if it doesn't exist + if not os.path.isdir(dromaeo_dir): + subprocess.check_call( + ["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/notriddle/dromaeo", dromaeo_dir]) + + # Run pull in case the Dromaeo repo was updated since last test run + subprocess.check_call( + ["git", "-C", dromaeo_dir, "pull"]) + + # Compile test suite + subprocess.check_call( + ["make", "-C", dromaeo_dir, "web"]) + + # Check that a release servo build exists + bin_path = path.abspath(self.get_binary_path(release, dev)) + + return subprocess.check_call( + [run_file, cmd, bin_path, base_dir]) + diff --git a/tests/dromaeo/run_dromaeo.py b/tests/dromaeo/run_dromaeo.py index 1765cb6d18f..3e82014a8f4 100755 --- a/tests/dromaeo/run_dromaeo.py +++ b/tests/dromaeo/run_dromaeo.py @@ -19,7 +19,7 @@ TEST_SERVER_PORT = 8192 # Run servo and print / parse the results for a specific jQuery test module. def run_servo(servo_exe): - url = "http://localhost:{0}/?recommended&automated&post_json".format(TEST_SERVER_PORT) + url = "http://localhost:{0}/dromaeo/web/?recommended&automated&post_json".format(TEST_SERVER_PORT) #args = [servo_exe, url, "-z", "-f"] args = [servo_exe, url, "-z"] return subprocess.Popen(args) From 6749bed1f56880e09aec574df49a8ae31fc8917c Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Fri, 31 Jul 2015 20:41:43 -0700 Subject: [PATCH 3/6] Fix it so it actually exits correctly. --- tests/dromaeo/run_dromaeo.py | 57 ++++-------------------------------- 1 file changed, 5 insertions(+), 52 deletions(-) diff --git a/tests/dromaeo/run_dromaeo.py b/tests/dromaeo/run_dromaeo.py index 3e82014a8f4..c00f6c2c195 100755 --- a/tests/dromaeo/run_dromaeo.py +++ b/tests/dromaeo/run_dromaeo.py @@ -20,8 +20,8 @@ TEST_SERVER_PORT = 8192 # Run servo and print / parse the results for a specific jQuery test module. def run_servo(servo_exe): url = "http://localhost:{0}/dromaeo/web/?recommended&automated&post_json".format(TEST_SERVER_PORT) - #args = [servo_exe, url, "-z", "-f"] - args = [servo_exe, url, "-z"] + #url = "http://localhost:{0}/dromaeo/web/?sunspider-string-validate-input&automated&post_json".format(TEST_SERVER_PORT) + args = [servo_exe, url, "-z", "-f"] return subprocess.Popen(args) # Print usage if command line args are incorrect @@ -29,56 +29,8 @@ def print_usage(): print("USAGE: {0} test servo_binary dromaeo_base_dir".format(sys.argv[0])) -# A simple HTTP server to serve up the test suite -class Server(SocketServer.ThreadingMixIn, - BaseHTTPServer.HTTPServer): - allow_reuse_address = True - got_post = False - +# Handle the POST at the end class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - # TODO(gw): HACK copy the fixed version from python - # main repo - due to https://bugs.python.org/issue23112 - def send_head(self): - path = self.translate_path(self.path) - f = None - if os.path.isdir(path): - parts = urlparse.urlsplit(self.path) - if not parts.path.endswith('/'): - # redirect browser - doing basically what apache does - self.send_response(301) - new_parts = (parts[0], parts[1], parts[2] + '/', - parts[3], parts[4]) - new_url = urlparse.urlunsplit(new_parts) - self.send_header("Location", new_url) - self.end_headers() - return None - for index in "index.html", "index.htm": - index = os.path.join(path, index) - if os.path.exists(index): - path = index - break - else: - return self.list_directory(path) - ctype = self.guess_type(path) - try: - # Always read in binary mode. Opening files in text mode may cause - # newline translations, making the actual size of the content - # transmitted *less* than the content-length! - f = open(path, 'rb') - except IOError: - self.send_error(404, "File not found") - return None - try: - self.send_response(200) - self.send_header("Content-type", ctype) - fs = os.fstat(f.fileno()) - self.send_header("Content-Length", str(fs[6])) - self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) - self.end_headers() - return f - except: - f.close() - raise def do_POST(self): self.send_response(200) self.end_headers() @@ -103,11 +55,12 @@ if __name__ == '__main__': sys.exit(1) # Start the test server - server = Server(('', TEST_SERVER_PORT), RequestHandler) + server = BaseHTTPServer.HTTPServer(('', TEST_SERVER_PORT), RequestHandler) if cmd == "test": print("Testing Dromaeo on Servo!") proc = run_servo(servo_exe) + server.got_post = False while not server.got_post: server.handle_request() print("dromaeo: %s" % server.post_data) From f82f46680eae635bb45eb4ba5526d1d5adb9c2d3 Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Fri, 31 Jul 2015 21:22:58 -0700 Subject: [PATCH 4/6] UI cheeze. --- python/servo/testing_commands.py | 10 +++++---- tests/dromaeo/run_dromaeo.py | 38 ++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 0f17274bac0..2b53f8f46e9 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -274,12 +274,14 @@ class MachCommands(CommandBase): @Command('test-dromaeo', description='Run the Dromaeo test suite', category='testing') + @CommandArgument('tests', default=["recommended"], nargs="...", + help="Specific tests to run") @CommandArgument('--release', '-r', action='store_true', help='Run the release build') @CommandArgument('--dev', '-d', action='store_true', help='Run the dev build') - def test_dromaeo(self, release, dev): - return self.dromaeo_test_runner("test", release, dev) + def test_dromaeo(self, tests, release, dev): + return self.dromaeo_test_runner(tests, release, dev) @Command('update-jquery', description='Update the jQuery test suite expected results', @@ -378,7 +380,7 @@ class MachCommands(CommandBase): return subprocess.check_call( [run_file, cmd, bin_path, base_dir]) - def dromaeo_test_runner(self, cmd, release, dev): + def dromaeo_test_runner(self, tests, release, dev): self.ensure_bootstrapped() base_dir = path.abspath(path.join("tests", "dromaeo")) dromaeo_dir = path.join(base_dir, "dromaeo") @@ -401,5 +403,5 @@ class MachCommands(CommandBase): bin_path = path.abspath(self.get_binary_path(release, dev)) return subprocess.check_call( - [run_file, cmd, bin_path, base_dir]) + [run_file, "|".join(tests), bin_path, base_dir]) diff --git a/tests/dromaeo/run_dromaeo.py b/tests/dromaeo/run_dromaeo.py index c00f6c2c195..22c459ffd37 100755 --- a/tests/dromaeo/run_dromaeo.py +++ b/tests/dromaeo/run_dromaeo.py @@ -13,20 +13,20 @@ import SimpleHTTPServer import SocketServer import threading import urlparse +import json # Port to run the HTTP server on for Dromaeo. TEST_SERVER_PORT = 8192 -# Run servo and print / parse the results for a specific jQuery test module. -def run_servo(servo_exe): - url = "http://localhost:{0}/dromaeo/web/?recommended&automated&post_json".format(TEST_SERVER_PORT) - #url = "http://localhost:{0}/dromaeo/web/?sunspider-string-validate-input&automated&post_json".format(TEST_SERVER_PORT) +# Run servo and print / parse the results for a specific Dromaeo module. +def run_servo(servo_exe, tests): + url = "http://localhost:{0}/dromaeo/web/?{1}&automated&post_json".format(TEST_SERVER_PORT, tests) args = [servo_exe, url, "-z", "-f"] return subprocess.Popen(args) # Print usage if command line args are incorrect def print_usage(): - print("USAGE: {0} test servo_binary dromaeo_base_dir".format(sys.argv[0])) + print("USAGE: {0} tests servo_binary dromaeo_base_dir".format(sys.argv[0])) # Handle the POST at the end @@ -44,7 +44,7 @@ class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): if __name__ == '__main__': if len(sys.argv) == 4: - cmd = sys.argv[1] + tests = sys.argv[1] servo_exe = sys.argv[2] base_dir = sys.argv[3] os.chdir(base_dir) @@ -57,15 +57,21 @@ if __name__ == '__main__': # Start the test server server = BaseHTTPServer.HTTPServer(('', TEST_SERVER_PORT), RequestHandler) - if cmd == "test": - print("Testing Dromaeo on Servo!") - proc = run_servo(servo_exe) - server.got_post = False - while not server.got_post: - server.handle_request() - print("dromaeo: %s" % server.post_data) - proc.kill() - else: - print_usage() + print("Testing Dromaeo on Servo!") + proc = run_servo(servo_exe, tests) + server.got_post = False + while not server.got_post: + server.handle_request() + data = json.loads(server.post_data[0]) + n = 0 + l = 0 + for test in data: + n = max(n, len(test) + len(data[test]) + 3) + l = max(l, len(test)) + print("\n Test{0} | Time".format(" " * (l - len("Test")))) + print("-" * (n+2)) + for test in data: + print(" {0}{1} | {2}".format(test, " " * (l - len(test)), data[test])) + proc.kill() else: print_usage() From a93b4bba6d1dd046ade178cf6c24ce5cda33bb2b Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Fri, 31 Jul 2015 21:49:01 -0700 Subject: [PATCH 5/6] test-tidy fixes. --- python/servo/testing_commands.py | 1 - tests/dromaeo/run_dromaeo.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index 2b53f8f46e9..76e5bf9cb79 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -404,4 +404,3 @@ class MachCommands(CommandBase): return subprocess.check_call( [run_file, "|".join(tests), bin_path, base_dir]) - diff --git a/tests/dromaeo/run_dromaeo.py b/tests/dromaeo/run_dromaeo.py index 22c459ffd37..de2d0f4f689 100755 --- a/tests/dromaeo/run_dromaeo.py +++ b/tests/dromaeo/run_dromaeo.py @@ -15,15 +15,18 @@ import threading import urlparse import json + # Port to run the HTTP server on for Dromaeo. TEST_SERVER_PORT = 8192 + # Run servo and print / parse the results for a specific Dromaeo module. def run_servo(servo_exe, tests): url = "http://localhost:{0}/dromaeo/web/?{1}&automated&post_json".format(TEST_SERVER_PORT, tests) - args = [servo_exe, url, "-z", "-f"] + args = [servo_exe, url, "-z", "-f"] return subprocess.Popen(args) + # Print usage if command line args are incorrect def print_usage(): print("USAGE: {0} tests servo_binary dromaeo_base_dir".format(sys.argv[0])) @@ -34,14 +37,16 @@ class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): def do_POST(self): self.send_response(200) self.end_headers() - self.wfile.write("POST OK.

"); + self.wfile.write("POST OK.

") length = int(self.headers.getheader('content-length')) parameters = urlparse.parse_qs(self.rfile.read(length)) self.server.got_post = True self.server.post_data = parameters['data'] + def log_message(self, format, *args): return + if __name__ == '__main__': if len(sys.argv) == 4: tests = sys.argv[1] @@ -69,7 +74,7 @@ if __name__ == '__main__': n = max(n, len(test) + len(data[test]) + 3) l = max(l, len(test)) print("\n Test{0} | Time".format(" " * (l - len("Test")))) - print("-" * (n+2)) + print("-" * (n + 2)) for test in data: print(" {0}{1} | {2}".format(test, " " * (l - len(test)), data[test])) proc.kill() From ac65817d2eb45017fcf44a733a3b82cecec5e2cf Mon Sep 17 00:00:00 2001 From: Michael Howell Date: Fri, 31 Jul 2015 21:57:26 -0700 Subject: [PATCH 6/6] Make output results valid Markdown tables. --- tests/dromaeo/run_dromaeo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dromaeo/run_dromaeo.py b/tests/dromaeo/run_dromaeo.py index de2d0f4f689..6a2cea41dee 100755 --- a/tests/dromaeo/run_dromaeo.py +++ b/tests/dromaeo/run_dromaeo.py @@ -71,10 +71,10 @@ if __name__ == '__main__': n = 0 l = 0 for test in data: - n = max(n, len(test) + len(data[test]) + 3) + n = max(n, len(data[test])) l = max(l, len(test)) print("\n Test{0} | Time".format(" " * (l - len("Test")))) - print("-" * (n + 2)) + print("-{0}-|-{1}-".format("-" * l, "-" * n)) for test in data: print(" {0}{1} | {2}".format(test, " " * (l - len(test)), data[test])) proc.kill()