From 4c8216129ee655026e4c0e8c3781645017969614 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 12:42:50 -0500 Subject: [PATCH 01/10] Add script to summarize WPT test directories with failing tests. --- etc/wpt_result_analyzer.py | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 etc/wpt_result_analyzer.py diff --git a/etc/wpt_result_analyzer.py b/etc/wpt_result_analyzer.py new file mode 100644 index 00000000000..eb2277db48a --- /dev/null +++ b/etc/wpt_result_analyzer.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright 2019 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import configparser +import os + +test_root = os.path.join('tests', 'wpt', 'web-platform-tests') +meta_root = os.path.join('tests', 'wpt', 'metadata') + +test_counts = {} +meta_counts = {} + +for base_dir, dir_names, files in os.walk(test_root): + if base_dir == test_root: + continue + + rel_base = os.path.relpath(base_dir, test_root) + if not os.path.exists(os.path.join(meta_root, rel_base)): + continue + + test_files = [] + exts = ['.html', '.htm', '.xht', '.xhtml', '.window.js', '.worker.js', '.any.js'] + for f in files: + for ext in exts: + if f.endswith(ext): + test_files += [f] + test_counts[rel_base] = len(test_files) + +for base_dir, dir_names, files in os.walk(meta_root): + if base_dir == meta_root: + continue + + rel_base = os.path.relpath(base_dir, meta_root) + num_files = len(files) + if '__dir__.ini' in files: + num_files -= 1 + meta_counts[rel_base] = num_files + +final_counts = [] +for (test_dir, test_count) in test_counts.items(): + if not test_count: + continue + meta_count = meta_counts.get(test_dir, 0) + final_counts += [(test_dir, test_count, meta_count)] + +print('Test counts') +print('dir: %% failed (num tests / num failures)') +s = sorted(final_counts, key=lambda x: x[2] / x[1]) +for (test_dir, test_count, meta_count) in reversed(sorted(s, key=lambda x: x[2])): + if not meta_count: + continue + print('%s: %.2f%% (%d / %d)' % (test_dir, meta_count / test_count * 100, test_count, meta_count)) From a634d90e4ac80027466782975007de33afcb4c28 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 12:46:36 -0500 Subject: [PATCH 02/10] Add a script to extra logs for particular test filenames from full WPT logs. --- etc/wpt-summarize.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 etc/wpt-summarize.py diff --git a/etc/wpt-summarize.py b/etc/wpt-summarize.py new file mode 100644 index 00000000000..48883a74126 --- /dev/null +++ b/etc/wpt-summarize.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Copyright 2019 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import sys +import json + +full_search = len(sys.argv) > 3 and sys.argv[3] == '--full' + +with open(sys.argv[1]) as f: + data = f.readlines() + thread = None + for entry in data: + entry = json.loads(entry) + if thread and "thread" in entry: + if entry["thread"] == thread: + print(json.dumps(entry)) + if "action" in entry and entry["action"] == "test_end": + thread = None + else: + if "action" in entry and \ + entry["action"] == "test_start" and \ + entry["test"] == sys.argv[2]: + thread = entry["thread"] + print(json.dumps(entry)) + elif full_search and \ + "command" in entry and \ + sys.argv[2] in entry["command"]: + thread = entry["thread"] + print(json.dumps(entry)) From ed715fc16f3a3d77b0cd50fb93b590dd3b58d40c Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 12:47:19 -0500 Subject: [PATCH 03/10] Add a script to report timing data for all tests in a WPT log. --- etc/wpt-timing.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 etc/wpt-timing.py diff --git a/etc/wpt-timing.py b/etc/wpt-timing.py new file mode 100644 index 00000000000..61455c1a604 --- /dev/null +++ b/etc/wpt-timing.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python + +# Copyright 2019 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import sys +import json +import itertools +import collections +import csv + +def process_log(data): + tests = {} + test_results = collections.defaultdict(list) + + for entry in data: + entry = json.loads(entry) + if "action" in entry: + if entry["action"] == "test_start": + tests[entry["test"]] = { + "start": int(entry["time"]), + "end": 0, + } + elif entry["action"] == "test_end": + test = tests[entry["test"]] + test["end"] = int(entry["time"]) + test_results[entry["status"]] += [ + (entry["test"], test["end"] - test["start"]) + ] + + return test_results + +test_results = { + "SKIP": [], + "OK": [], + "PASS": [], + "ERROR": [], + "FAIL": [], + "CRASH": [], + "TIMEOUT": [], +} +for log_path in sys.argv[1:]: + with open(log_path) as f: + data = f.readlines() + for k, v in process_log(data).items(): + test_results[k] += v + +print("Skipped %d tests." % len(test_results["SKIP"])) +print("%d tests timed out." % len(test_results["TIMEOUT"])) + +longest_crash = sorted(test_results["CRASH"], key=lambda x: x[1], reverse=True) +print("Longest CRASH test took %dms (%s)" % (longest_crash[0][1], longest_crash[0][0])) + +longest_ok = sorted( + test_results["PASS"] + test_results["OK"], + key=lambda x: x[1], reverse=True +) +csv_data = [['Test path', 'Milliseconds']] +with open('longest_ok.csv', 'w') as csv_file: + writer = csv.writer(csv_file) + writer.writerows(csv_data + longest_ok) + +longest_fail = sorted( + test_results["ERROR"] + test_results["FAIL"], + key=lambda x: x[1], reverse=True +) +with open('longest_err.csv', 'w') as csv_file: + writer = csv.writer(csv_file) + writer.writerows(csv_data + longest_fail) + +longest_timeout = sorted(test_results["TIMEOUT"], key=lambda x: x[1], reverse=True) +with open('timeouts.csv', 'w') as csv_file: + writer = csv.writer(csv_file) + writer.writerows(csv_data + longest_timeout) From 5867e115c26d55a2f9e442b5d26cf40a6357d37a Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 12:58:42 -0500 Subject: [PATCH 04/10] Fix tidy issues in wpt-timing.py. --- etc/wpt-timing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/wpt-timing.py b/etc/wpt-timing.py index 61455c1a604..85773e8bab5 100644 --- a/etc/wpt-timing.py +++ b/etc/wpt-timing.py @@ -11,10 +11,10 @@ import sys import json -import itertools import collections import csv + def process_log(data): tests = {} test_results = collections.defaultdict(list) @@ -73,7 +73,7 @@ longest_fail = sorted( with open('longest_err.csv', 'w') as csv_file: writer = csv.writer(csv_file) writer.writerows(csv_data + longest_fail) - + longest_timeout = sorted(test_results["TIMEOUT"], key=lambda x: x[1], reverse=True) with open('timeouts.csv', 'w') as csv_file: writer = csv.writer(csv_file) From 46af28a0e159ca8cb749cff7123175a8e9f95309 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 13:03:18 -0500 Subject: [PATCH 05/10] Fix tidy issues in wpt-summarize.py. --- etc/wpt-summarize.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/etc/wpt-summarize.py b/etc/wpt-summarize.py index 48883a74126..06cf9dd3f5a 100644 --- a/etc/wpt-summarize.py +++ b/etc/wpt-summarize.py @@ -13,7 +13,7 @@ import sys import json full_search = len(sys.argv) > 3 and sys.argv[3] == '--full' - + with open(sys.argv[1]) as f: data = f.readlines() thread = None @@ -25,13 +25,13 @@ with open(sys.argv[1]) as f: if "action" in entry and entry["action"] == "test_end": thread = None else: - if "action" in entry and \ - entry["action"] == "test_start" and \ - entry["test"] == sys.argv[2]: + if ("action" in entry and + entry["action"] == "test_start" and + entry["test"] == sys.argv[2]): thread = entry["thread"] print(json.dumps(entry)) - elif full_search and \ - "command" in entry and \ - sys.argv[2] in entry["command"]: + elif (full_search and + "command" in entry and + sys.argv[2] in entry["command"]): thread = entry["thread"] print(json.dumps(entry)) From 6cad3dbe3f06c288e177a60f4e2b2a321f7247c6 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 13:03:41 -0500 Subject: [PATCH 06/10] Remove unused import. --- etc/wpt_result_analyzer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/wpt_result_analyzer.py b/etc/wpt_result_analyzer.py index eb2277db48a..5b7171f44a7 100644 --- a/etc/wpt_result_analyzer.py +++ b/etc/wpt_result_analyzer.py @@ -9,7 +9,6 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. -import configparser import os test_root = os.path.join('tests', 'wpt', 'web-platform-tests') From cc1aadfd20fee6d75962902bcd85d440743dcbce Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 13:14:30 -0500 Subject: [PATCH 07/10] Add explanatory comment for wpt-summarize.py. --- etc/wpt-summarize.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/etc/wpt-summarize.py b/etc/wpt-summarize.py index 06cf9dd3f5a..96f55632740 100644 --- a/etc/wpt-summarize.py +++ b/etc/wpt-summarize.py @@ -9,6 +9,15 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +# Usage: python wpt-summarize.py /wpt/test/url.html [--full] +# +# Extract all log lines for a particular test file from a WPT +# logs, outputting invidual JSON objects that can be manipulated +# with tools like jq. If a particular URL results in no output, +# the URL is likely used as a reference test's reference file, +# so passing `--full` will find any output from Servo process +# command lines that include the URL. + import sys import json From 0c294de4a793d4635360adade60ab336326ddecc Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 13:19:28 -0500 Subject: [PATCH 08/10] Add explanatory comment to wpt-timing.py. --- etc/wpt-timing.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/etc/wpt-timing.py b/etc/wpt-timing.py index 85773e8bab5..cc5d81aad9a 100644 --- a/etc/wpt-timing.py +++ b/etc/wpt-timing.py @@ -9,6 +9,21 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +# Usage: python wpt-timing.py [path/to/wpt.log] ... +# +# Given a series of WPT log files as arguments, this script +# extracts the status of each test file (ok; error; timeout; etc.) +# and how long it took to ran, then creates three CSV files, each +# sorted by runtime: +# +# - longest_ok.csv: all tests that passed +# - longest_err.csv: all tests that failed or had an error +# - timeouts.csv: all tests that timed out +# +# This information can be used to quickly determine the longest-running +# tests in the WPT testsuite in order to improve the overall testsuite +# runtime on CI. + import sys import json import collections From 804780f93316b861dfc1f23f94e711307eb39b8f Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 13:23:43 -0500 Subject: [PATCH 09/10] Add explanatory comment for wpt_result_analyzer.py. --- etc/wpt_result_analyzer.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/etc/wpt_result_analyzer.py b/etc/wpt_result_analyzer.py index 5b7171f44a7..22db58bbc9b 100644 --- a/etc/wpt_result_analyzer.py +++ b/etc/wpt_result_analyzer.py @@ -9,6 +9,15 @@ # option. This file may not be copied, modified, or distributed # except according to those terms. +# Usage: python etc/wpt_result_analyzer.py +# +# Analyze the state of WPT tests in Servo by walking all of the +# test directories, counting the number of tests present, and +# counting the number of ini files present in the corresponding +# test result directory. Prints out a list of directories that +# have non-zero failure counts, ordered by overall number of tests +# and percentage of tests that fail. + import os test_root = os.path.join('tests', 'wpt', 'web-platform-tests') From 41d1ecac64c4c36e6bd232a016143692843dfa92 Mon Sep 17 00:00:00 2001 From: Josh Matthews Date: Fri, 22 Nov 2019 14:34:26 -0500 Subject: [PATCH 10/10] Fix visual indent error. --- etc/wpt-summarize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/wpt-summarize.py b/etc/wpt-summarize.py index 96f55632740..14cf6c4067c 100644 --- a/etc/wpt-summarize.py +++ b/etc/wpt-summarize.py @@ -35,8 +35,8 @@ with open(sys.argv[1]) as f: thread = None else: if ("action" in entry and - entry["action"] == "test_start" and - entry["test"] == sys.argv[2]): + entry["action"] == "test_start" and + entry["test"] == sys.argv[2]): thread = entry["thread"] print(json.dumps(entry)) elif (full_search and