Add flake8 to the tidy process for Python files

Fixes #6236

Also included in this commit are the changes need to make flake8 pass
for the existing python file

python/licenseck.py

@@ -59,4 +59,4 @@ licenses = [
 # except according to those terms.
 """,
 
-]
+]  # noqa: Indicate to flake8 that we do not want to check indentation here

python/mach_bootstrap.py

@@ -98,7 +98,7 @@ def bootstrap(topdir):
 
     for category, meta in CATEGORIES.items():
         mach.define_category(category, meta['short'], meta['long'],
-            meta['priority'])
+                             meta['priority'])
 
     for path in MACH_MODULES:
         mach.load_commands_from_file(os.path.join(topdir, path))

python/servo/bootstrap_commands.py

@@ -40,7 +40,8 @@ def download(desc, src, dst):
         with open(dst, 'wb') as fd:
             while True:
                 chunk = resp.read(chunk_size)
-                if not chunk: break
+                if not chunk:
+                    break
                 recved += len(chunk)
                 if not dumb:
                     pct = recved * 100.0 / fsize
@@ -61,6 +62,7 @@ def download(desc, src, dst):
 
         sys.exit(1)
 
+
 def extract(src, dst, movedir=None):
     tarfile.open(src).extractall(dst)
 

python/servo/build_commands.py

@@ -58,13 +58,14 @@ def notify_win(title, text):
     FLASHW_TIMERNOFG = 0x0C
 
     params = FLASHWINDOW(sizeof(FLASHWINDOW),
-                        windll.kernel32.GetConsoleWindow(),
-                        FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0)
+                         windll.kernel32.GetConsoleWindow(),
+                         FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0)
     FlashWindowEx(params)
 
 
 def notify_darwin(title, text):
-    import Foundation, objc
+    import Foundation
+    import objc
 
     NSUserNotification = objc.lookUpClass("NSUserNotification")
     NSUserNotificationCenter = objc.lookUpClass("NSUserNotificationCenter")
@@ -291,7 +292,7 @@ class MachCommands(CommandBase):
             opts += ["--release"]
 
         opts += ["--target", "arm-linux-androideabi"]
-        env=self.build_env(gonk=True)
+        env = self.build_env(gonk=True)
         build_start = time()
         with cd(path.join("ports", "gonk")):
             ret = subprocess.call(["cargo", "build"] + opts, env=env)
@@ -304,7 +305,6 @@ class MachCommands(CommandBase):
 
         return ret
 
-
     @Command('build-tests',
              description='Build the Servo test suites',
              category='build')
@@ -329,7 +329,6 @@ class MachCommands(CommandBase):
     @CommandArgument('--verbose', '-v',
                      action='store_true',
                      help='Print verbose output')
-
     @CommandArgument('params', nargs='...',
                      help="Command-line arguments to be passed through to Cargo")
     def clean(self, manifest_path, params, verbose=False):

python/servo/command_base.py

@@ -165,7 +165,7 @@ class CommandBase(object):
 
         print("The %s profile is not built. Please run './mach build%s' "
               "and try again." % ("release" if release else "dev",
-                                   " --release" if release else ""))
+                                  " --release" if release else ""))
         sys.exit()
 
     def build_env(self, gonk=False, hosts_file_path=None):

python/servo/devenv_commands.py

@@ -36,8 +36,8 @@ class MachCommands(CommandBase):
 
         if self.context.topdir == getcwd():
             with cd(path.join('components', 'servo')):
-                return subprocess.call(["cargo"] + params,
-                               env=self.build_env())
+                return subprocess.call(
+                    ["cargo"] + params, env=self.build_env())
         return subprocess.call(['cargo'] + params,
                                env=self.build_env())
 
@@ -51,7 +51,7 @@ class MachCommands(CommandBase):
         '--package', '-p', default=None,
         help='Updates selected package')
     @CommandArgument(
-        '--all-packages','-a',action='store_true',
+        '--all-packages', '-a', action='store_true',
         help='Updates all packages')
     def cargo_update(self, params=None, package=None, all_packages=None):
         self.update_cargo(params, package, all_packages)
@@ -63,10 +63,10 @@ class MachCommands(CommandBase):
         'params', default=None, nargs='...',
         help='Command-line arguments to be passed through to cargo update')
     @CommandArgument(
-        '--package','-p',default=None,
+        '--package', '-p', default=None,
         help='Updates selected package')
     @CommandArgument(
-        '--all-packages','-a',action='store_true',
+        '--all-packages', '-a', action='store_true',
         help='Updates all packages')
     def update_cargo(self, params=None, package=None, all_packages=None):
         if not params:

python/servo/post_build_commands.py

@@ -12,7 +12,6 @@ from __future__ import print_function, unicode_literals
 import os
 import os.path as path
 import subprocess
-import sys
 from shutil import copytree, rmtree, copy2
 
 from mach.registrar import Registrar
@@ -49,7 +48,7 @@ class MachCommands(CommandBase):
                           'debugger being used. The following arguments '
                           'have no effect without this.')
     @CommandArgument('--debugger', default=None, type=str,
-        help='Name of debugger to use.')
+                     help='Name of debugger to use.')
     @CommandArgument(
         'params', nargs='...',
         help="Command-line arguments to be passed through to Servo")
@@ -74,8 +73,8 @@ class MachCommands(CommandBase):
                 return 1
 
             # Prepend the debugger args.
-            args = ([self.debuggerInfo.path] + self.debuggerInfo.args
-                    + args + params)
+            args = ([self.debuggerInfo.path] + self.debuggerInfo.args +
+                    args + params)
         else:
             args = args + params
 
@@ -84,7 +83,7 @@ class MachCommands(CommandBase):
         except OSError as e:
             if e.errno == 2:
                 print("Servo Binary can't be found! Run './mach build'"
-                     " and try again!")
+                      " and try again!")
             else:
                 raise e
 
@@ -132,8 +131,7 @@ class MachCommands(CommandBase):
         help="Command-line arguments to be passed through to cargo doc")
     def doc(self, params):
         self.ensure_bootstrapped()
-        if not path.exists(path.join(
-               self.config["tools"]["rust-root"], "doc")):
+        if not path.exists(path.join(self.config["tools"]["rust-root"], "doc")):
             Registrar.dispatch("bootstrap-rust-docs", context=self.context)
         rust_docs = path.join(self.config["tools"]["rust-root"], "doc")
         docs = path.join(

python/servo/testing_commands.py

@@ -281,7 +281,6 @@ class MachCommands(CommandBase):
         execfile(run_file, run_globals)
         return run_globals["update_tests"](**kwargs)
 
-
     def ensure_wpt_virtualenv(self):
         virtualenv_path = path.join("tests", "wpt", "_virtualenv")
         python = self.get_exec("python2", "python")
@@ -295,8 +294,8 @@ class MachCommands(CommandBase):
         execfile(activate_path, dict(__file__=activate_path))
 
         try:
-            import wptrunner
-            from wptrunner.browsers import servo
+            import wptrunner  # noqa
+            from wptrunner.browsers import servo  # noqa
         except ImportError:
             subprocess.check_call(["pip", "install", "-r",
                                    path.join("tests", "wpt", "harness", "requirements.txt")])
@@ -311,7 +310,7 @@ class MachCommands(CommandBase):
         # before the virtualenv is initalised it doesn't see the blessings module so we don't
         # get coloured output. Setting the blessings global explicitly fixes that.
         from mozlog.structured.formatters import machformatter
-        import blessings
+        import blessings  # noqa
         machformatter.blessings = blessings
 
     def get_exec(self, name, default=None):

python/tidy.py

@@ -7,17 +7,21 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-#!/usr/bin/env python
-
 import os
 import fnmatch
 import itertools
 import re
+import sys
 from licenseck import licenses
 
 filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".py"]
 reftest_directories = ["tests/ref"]
 reftest_filetype = ".list"
+python_dependencies = [
+    "./python/dependencies/flake8-2.4.1-py2.py3-none-any.whl",
+    "./python/dependencies/pep8-1.5.7-py2.py3-none-any.whl",
+    "./python/dependencies/pyflakes-0.9.0-py2.py3-none-any.whl",
+]
 
 ignored_files = [
     # Upstream
@@ -80,12 +84,14 @@ def check_length(idx, line):
     if len(line) >= 120:
         yield (idx + 1, "(much) overlong line")
 
+
 def check_whatwg_url(idx, line):
     match = re.search(r"https://html\.spec\.whatwg\.org/multipage/[\w-]+\.html#([\w\:-]+)", line)
     if match is not None:
         preferred_link = "https://html.spec.whatwg.org/multipage/#{}".format(match.group(1))
         yield (idx + 1, "link to WHATWG may break in the future, use this format instead: {}".format(preferred_link))
 
+
 def check_whitespace(idx, line):
     if line[-1] == "\n":
         line = line[:-1]
@@ -101,6 +107,7 @@ def check_whitespace(idx, line):
     if "\r" in line:
         yield (idx + 1, "CR on line")
 
+
 def check_by_line(contents):
     lines = contents.splitlines(True)
     for idx, line in enumerate(lines):
@@ -113,6 +120,25 @@ def check_by_line(contents):
             yield error
 
 
+def check_flake8(file_paths):
+    from flake8.main import check_file
+
+    ignore = {
+        "W291",  # trailing whitespace; the standard tidy process will enforce no trailing whitespace
+        "E501",  # 80 character line length; the standard tidy process will enforce line length
+    }
+
+    num_errors = 0
+
+    for file_path in file_paths:
+        if os.path.splitext(file_path)[-1].lower() != ".py":
+            continue
+
+        num_errors += check_file(file_path, ignore=ignore)
+
+    return num_errors
+
+
 def collect_errors_for_files(files_to_check, checking_functions):
     for file_name in files_to_check:
         with open(file_name, "r") as fp:
@@ -129,7 +155,7 @@ def check_reftest_order(files_to_check):
             split_lines = fp.read().splitlines()
             lines = filter(lambda l: len(l) > 0 and l[0] != '#', split_lines)
             for idx, line in enumerate(lines[:-1]):
-                next_line = lines[idx+1]
+                next_line = lines[idx + 1]
                 current = get_reftest_names(line)
                 next = get_reftest_names(next_line)
                 if current is not None and next is not None and current > next:
@@ -146,9 +172,13 @@ def get_reftest_names(line):
 
 
 def scan():
+    sys.path += python_dependencies
+
     all_files = collect_file_names()
     files_to_check = filter(should_check, all_files)
 
+    num_flake8_errors = check_flake8(files_to_check)
+
     checking_functions = [check_license, check_by_line]
     errors = collect_errors_for_files(files_to_check, checking_functions)
 
@@ -158,7 +188,7 @@ def scan():
 
     errors = list(itertools.chain(errors, r_errors))
 
-    if errors:
+    if errors or num_flake8_errors:
         for error in errors:
             print("{}:{}: {}".format(*error))
         return 1