tidy: checking directories for unexpected file extensions

update config, check_dir func to use config

pass only_changed_files and exclude_dirs as args

simplify config

better conditional

remove unused default, amend error message

undo doc change

python/tidy/servo_tidy/tidy.py

@@ -35,7 +35,8 @@ config = {
             "./.",   # ignore hidden directories
         ],
         "packages": [],
-    }
+    },
+    "check_ext": {}
 }
 
 COMMENTS = ["// ", "# ", " *", "/* "]
@@ -686,7 +687,7 @@ def check_config_file(config_file, print_text=True):
         # Check for invalid tables
         if re.match("\[(.*?)\]", line.strip()):
             table_name = re.findall(r"\[(.*?)\]", line)[0].strip()
-            if table_name not in ("configs", "ignore"):
+            if table_name not in ("configs", "ignore", "check_ext"):
                 yield config_file, idx + 1, "invalid config table [%s]" % table_name
             current_table = table_name
             continue
@@ -723,6 +724,13 @@ def parse_config(content):
     config['ignore']['directories'] = map(lambda path: os.path.join(*path.split('/')),
                                           config['ignore']['directories'])
 
+    # Add dict of dir, list of expected ext to config
+    dirs_to_check = config_file.get("check_ext", {})
+    # Fix the paths (OS-dependent)
+    for path, exts in dirs_to_check.items():
+        fixed_path = os.path.join(*path.split('/'))
+        config['check_ext'][fixed_path] = exts
+
     # Override default configs
     user_configs = config_file.get("configs", [])
     for pref in user_configs:
@@ -730,6 +738,23 @@ def parse_config(content):
             config[pref] = user_configs[pref]
 
 
+def check_directory_files(directories, print_text=True):
+    if print_text:
+        print '\rChecking directories for correct file extensions...'
+    for directory, file_extensions in directories.items():
+        files = sorted(os.listdir(directory))
+        for filename in files:
+            if not any(filename.endswith(ext) for ext in file_extensions):
+                details = {
+                    "name": os.path.basename(filename),
+                    "ext": ", ".join(file_extensions),
+                    "dir_name": directory
+                }
+                message = '''Unexpected extension found for {name}. \
+We only expect files with {ext} extensions in {dir_name}'''.format(**details)
+                yield (filename, 1, message)
+
+
 def collect_errors_for_files(files_to_check, checking_functions, line_checking_functions, print_text=True):
     (has_element, files_to_check) = is_iter_empty(files_to_check)
     if not has_element:
@@ -830,6 +855,8 @@ def get_file_list(directory, only_changed_files=False, exclude_dirs=[]):
 def scan(only_changed_files=False, progress=True):
     # check config file for errors
     config_errors = check_config_file(CONFIG_FILE_PATH)
+    # check directories contain expected files
+    directory_errors = check_directory_files(config['check_ext'])
     # standard checks
     files_to_check = filter_files('.', only_changed_files, progress)
     checking_functions = (check_flake8, check_lock, check_webidl_spec, check_json)
@@ -841,7 +868,7 @@ def scan(only_changed_files=False, progress=True):
     # wpt lint checks
     wpt_lint_errors = check_wpt_lint_errors(get_wpt_files(only_changed_files, progress))
     # chain all the iterators
-    errors = itertools.chain(config_errors, file_errors, dep_license_errors, wpt_lint_errors)
+    errors = itertools.chain(config_errors, directory_errors, file_errors, dep_license_errors, wpt_lint_errors)
 
     error = None
     for error in errors:

python/tidy/servo_tidy_tests/test_tidy.py

@@ -9,6 +9,7 @@
 
 import os
 import unittest
+
 from servo_tidy import tidy
 
 base_path = 'servo_tidy_tests/' if os.path.exists('servo_tidy_tests/') else 'python/tidy/servo_tidy_tests/'
@@ -30,6 +31,17 @@ class CheckTidiness(unittest.TestCase):
         self.assertEqual('invalid config table [wrong]', errors.next()[2])
         self.assertNoMoreErrors(errors)
 
+    def test_directory_checks(self):
+        dirs = {
+            os.path.join(base_path, "dir_check/webidl_plus"): ['webidl', 'test'],
+            os.path.join(base_path, "dir_check/only_webidl"): ['webidl']
+            }
+        errors = tidy.check_directory_files(dirs)
+        error_dir = os.path.join(base_path, "dir_check/webidl_plus")
+        self.assertEqual("Unexpected extension found for test.rs. We only expect files with webidl, test extensions in {0}".format(error_dir), errors.next()[2])
+        self.assertEqual("Unexpected extension found for test2.rs. We only expect files with webidl, test extensions in {0}".format(error_dir), errors.next()[2])
+        self.assertNoMoreErrors(errors)
+
     def test_spaces_correctnes(self):
         errors = tidy.collect_errors_for_files(iterFile('wrong_space.rs'), [], [tidy.check_by_line], print_text=False)
         self.assertEqual('trailing whitespace', errors.next()[2])

servo-tidy.toml

@@ -46,3 +46,8 @@ directories = [
   "./target",
   "./ports/cef",
 ]
+
+# Directories that are checked for correct file extension
+[check_ext]
+# directory, list of expected file extensions
+"components/script/dom/webidls" = [".webidl"]