Auto merge of #19929 - terracotaPie:master, r=jdm

Verify that all values in include.ini represent real directories

<!-- Please describe your changes on the following line: -->
wpt manifest(include.ini) contains directories as headers. With this test in **test-tidy** we check if they are present in respective wtp test folders.

---
- [X] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors
- [X] These changes fix #19703

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/19929)
<!-- Reviewable:end -->

python/tidy/servo_tidy/tidy.py

@@ -22,10 +22,19 @@ import colorama
 import toml
 import voluptuous
 import yaml
-
 from licenseck import MPL, APACHE, COPYRIGHT, licenses_toml, licenses_dep_toml
+topdir = os.path.abspath(os.path.dirname(sys.argv[0]))
+wpt = os.path.join(topdir, "tests", "wpt")
+
+
+def wpt_path(*args):
+    return os.path.join(wpt, *args)
+
+sys.path.append(wpt_path("web-platform-tests", "tools", "wptrunner", "wptrunner"))
+from wptmanifest import parser, node
 
 CONFIG_FILE_PATH = os.path.join(".", "servo-tidy.toml")
+WPT_MANIFEST_PATH = wpt_path("include.ini")
 
 # Default configs
 config = {
@@ -440,6 +449,38 @@ def check_shell(file_name, lines):
                     yield(idx + 1, "variable substitutions should use the full \"${VAR}\" form")
 
 
+def rec_parse(current_path, root_node):
+    dirs = []
+    for item in root_node.children:
+        if isinstance(item, node.DataNode):
+            next_depth = os.path.join(current_path, item.data)
+            dirs.append(next_depth)
+            dirs += rec_parse(next_depth, item)
+    return dirs
+
+
+def check_manifest_dirs(config_file, print_text=True):
+    if not os.path.exists(config_file):
+        yield(config_file, 0, "%s manifest file is required but was not found" % config_file)
+        return
+
+    # Load configs from include.ini
+    with open(config_file) as content:
+        conf_file = content.read()
+        lines = conf_file.splitlines(True)
+
+    if print_text:
+        print '\rChecking the wpt manifest file...'
+
+    p = parser.parse(lines)
+    paths = rec_parse(wpt_path("web-platform-tests"), p)
+    for idx, path in enumerate(paths):
+        if path.endswith("_mozilla"):
+            continue
+        if not os.path.isdir(path):
+            yield(config_file, idx + 1, "Path in manifest was not found: {}".format(path))
+
+
 def check_rust(file_name, lines):
     if not file_name.endswith(".rs") or \
        file_name.endswith(".mako.rs") or \
@@ -1122,6 +1163,8 @@ def run_lint_scripts(only_changed_files=False, progress=True, stylo=False):
 def scan(only_changed_files=False, progress=True, stylo=False):
     # check config file for errors
     config_errors = check_config_file(CONFIG_FILE_PATH)
+    # check ini directories exist
+    manifest_errors = check_manifest_dirs(WPT_MANIFEST_PATH)
     # check directories contain expected files
     directory_errors = check_directory_files(config['check_ext'])
     # standard checks
@@ -1135,7 +1178,7 @@ def scan(only_changed_files=False, progress=True, stylo=False):
     # other lint checks
     lint_errors = run_lint_scripts(only_changed_files, progress, stylo=stylo)
     # chain all the iterators
-    errors = itertools.chain(config_errors, directory_errors, lint_errors,
+    errors = itertools.chain(config_errors, manifest_errors, directory_errors, lint_errors,
                              file_errors, dep_license_errors)
 
     error = None

python/tidy/servo_tidy_tests/manifest-include.ini

@@ -0,0 +1,5 @@
+skip: true
+[html]
+  skip: false
+[never_going_to_exist]
+  skip: false

python/tidy/servo_tidy_tests/test_tidy.py

@@ -33,6 +33,15 @@ class CheckTidiness(unittest.TestCase):
         self.assertEqual("ignored directory './fake/dir' doesn't exist", errors.next()[2])
         self.assertNoMoreErrors(errors)
 
+    def test_non_existing_wpt_manifest_checks(self):
+        wrong_path = "/wrong/path.ini"
+        errors = tidy.check_manifest_dirs(wrong_path, print_text=False)
+        self.assertEqual("%s manifest file is required but was not found" % wrong_path, errors.next()[2])
+        self.assertNoMoreErrors(errors)
+        errors = tidy.check_manifest_dirs(os.path.join(base_path, 'manifest-include.ini'), print_text=False)
+        self.assertTrue(errors.next()[2].endswith("never_going_to_exist"))
+        self.assertNoMoreErrors(errors)
+
     def test_directory_checks(self):
         dirs = {
             os.path.join(base_path, "dir_check/webidl_plus"): ['webidl', 'test'],

tests/wpt/include.ini

@@ -3,8 +3,6 @@ skip: true
   skip: false
 [2dcontext]
   skip: false
-[animation-timing]
-  skip: false
 [cors]
   skip: false
 [css]
@@ -55,12 +53,6 @@ skip: true
     skip: false
   [selectors]
     skip: false
-[cssom]
-  skip: false
-[cssom-view]
-  skip: false
-[css-values]
-  skip: false
 [custom-elements]
   skip: false
 [dom]
@@ -89,24 +81,16 @@ skip: true
   skip: true
   [submission]
     skip: true
-    [Opera]
-      skip: true
-      [script_scheduling]
-        skip: false
 [performance-timeline]
   skip: false
 [quirks]
   skip: false
 [referrer-policy]
   skip: false
-[selectors]
-  skip: false
 [subresource-integrity]
   skip: false
 [touch-events]
   skip: false
-[typedarrays]
-  skip: false
 [uievents]
   skip: false
 [url]
@@ -123,5 +107,5 @@ skip: true
   skip: false
 [workers]
   skip: false
-[XMLHttpRequest]
+[xhr]
   skip: false

tests/wpt/metadata/xhr/event-upload-progress-crossorigin.htm.ini

@@ -6,3 +6,9 @@
   [Upload events registered too late (resources/redirect.py?code=307&location=http://www1.web-platform.test:8000/XMLHttpRequest/resources/corsenabled.py)]
     expected: FAIL
 
+  [Upload events registered too late (http://www1.web-platform.test:8000/xhr/resources/corsenabled.py)]
+    expected: FAIL
+
+  [Upload events registered too late (resources/redirect.py?code=307&location=http://www1.web-platform.test:8000/xhr/resources/corsenabled.py)]
+    expected: FAIL
+

tests/wpt/metadata/xhr/responsexml-document-properties.htm.ini

@@ -6,9 +6,6 @@
   [readyState]
     expected: FAIL
 
-  [body]
-    expected: FAIL
-
   [images]
     expected: FAIL