Add flake8 to the tidy process for Python files

Fixes #6236

Also included in this commit are the changes need to make flake8 pass
for the existing python file

etc/servo_gdb.py

@@ -7,19 +7,22 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-# A set of simple pretty printers for gdb to make debugging Servo a bit easier.
+"""
+A set of simple pretty printers for gdb to make debugging Servo a bit easier.
 
-# To load these, you need to add something like the following
+To load these, you need to add something like the following to your .gdbinit file:
-# to your .gdbinit file.
+
-#python
+python
-#import sys
+import sys
-#sys.path.insert(0, '/home/<path to git checkout>/servo/src/etc')
+sys.path.insert(0, '/home/<path to git checkout>/servo/src/etc')
-#import servo_gdb
+import servo_gdb
-#servo_gdb.register_printers(None)
+servo_gdb.register_printers(None)
-#end
+end
+"""
 
 import gdb
 
+
 # Print Au in both raw value and CSS pixels
 class AuPrinter:
     def __init__(self, val):
@@ -27,9 +30,10 @@ class AuPrinter:
 
     def to_string(self):
         i32_type = gdb.lookup_type("i32")
-        au = self.val.cast(i32_type);
+        au = self.val.cast(i32_type)
         return "{0}px".format(au / 60.0)
 
+
 # Print a U8 bitfield as binary
 class BitFieldU8Printer:
     def __init__(self, val):
@@ -37,9 +41,10 @@ class BitFieldU8Printer:
 
     def to_string(self):
         u8_type = gdb.lookup_type("u8")
-        value = self.val.cast(u8_type);
+        value = self.val.cast(u8_type)
         return "[{0:#010b}]".format(int(value))
 
+
 # Print a struct with fields as children
 class ChildPrinter:
     def __init__(self, val):
@@ -54,6 +59,7 @@ class ChildPrinter:
     def to_string(self):
         return None
 
+
 # Allow a trusted node to be dereferenced in the debugger
 class TrustedNodeAddressPrinter:
     def __init__(self, val):
@@ -67,6 +73,7 @@ class TrustedNodeAddressPrinter:
     def to_string(self):
         return self.val.address
 
+
 # Extract a node type ID from enum
 class NodeTypeIdPrinter:
     def __init__(self, val):
@@ -75,9 +82,10 @@ class NodeTypeIdPrinter:
     def to_string(self):
         u8_ptr_type = gdb.lookup_type("u8").pointer()
         enum_0 = self.val.address.cast(u8_ptr_type).dereference()
-        enum_type = self.val.type.fields()[int(enum_0)].type;
+        enum_type = self.val.type.fields()[int(enum_0)].type
         return str(enum_type).lstrip('struct ')
 
+
 # Printer for std::Option<>
 class OptionPrinter:
     def __init__(self, val):
@@ -111,6 +119,7 @@ class OptionPrinter:
     def to_string(self):
         return None
 
+
 # Useful for debugging when type is unknown
 class TestPrinter:
     def __init__(self, val):
@@ -129,6 +138,7 @@ type_map = [
     ('Option', OptionPrinter),
 ]
 
+
 def lookup_servo_type(val):
     val_type = str(val.type)
     for (type_name, printer) in type_map:
@@ -137,5 +147,6 @@ def lookup_servo_type (val):
     return None
     # return TestPrinter(val)
 
+
 def register_printers(obj):
     gdb.pretty_printers.append(lookup_servo_type)

python/licenseck.py

@@ -59,4 +59,4 @@ licenses = [
 # except according to those terms.
 """,
 
-]
+]  # noqa: Indicate to flake8 that we do not want to check indentation here

python/servo/bootstrap_commands.py

@@ -40,7 +40,8 @@ def download(desc, src, dst):
         with open(dst, 'wb') as fd:
             while True:
                 chunk = resp.read(chunk_size)
-                if not chunk: break
+                if not chunk:
+                    break
                 recved += len(chunk)
                 if not dumb:
                     pct = recved * 100.0 / fsize
@@ -61,6 +62,7 @@ def download(desc, src, dst):
 
         sys.exit(1)
 
+
 def extract(src, dst, movedir=None):
     tarfile.open(src).extractall(dst)
 

python/servo/build_commands.py

@@ -64,7 +64,8 @@ def notify_win(title, text):
 
 
 def notify_darwin(title, text):
-    import Foundation, objc
+    import Foundation
+    import objc
 
     NSUserNotification = objc.lookUpClass("NSUserNotification")
     NSUserNotificationCenter = objc.lookUpClass("NSUserNotificationCenter")
@@ -304,7 +305,6 @@ class MachCommands(CommandBase):
 
         return ret
 
-
     @Command('build-tests',
              description='Build the Servo test suites',
              category='build')
@@ -329,7 +329,6 @@ class MachCommands(CommandBase):
     @CommandArgument('--verbose', '-v',
                      action='store_true',
                      help='Print verbose output')
-
     @CommandArgument('params', nargs='...',
                      help="Command-line arguments to be passed through to Cargo")
     def clean(self, manifest_path, params, verbose=False):

python/servo/devenv_commands.py

@@ -36,8 +36,8 @@ class MachCommands(CommandBase):
 
         if self.context.topdir == getcwd():
             with cd(path.join('components', 'servo')):
-                return subprocess.call(["cargo"] + params,
+                return subprocess.call(
-                               env=self.build_env())
+                    ["cargo"] + params, env=self.build_env())
         return subprocess.call(['cargo'] + params,
                                env=self.build_env())
 

python/servo/post_build_commands.py

@@ -12,7 +12,6 @@ from __future__ import print_function, unicode_literals
 import os
 import os.path as path
 import subprocess
-import sys
 from shutil import copytree, rmtree, copy2
 
 from mach.registrar import Registrar
@@ -74,8 +73,8 @@ class MachCommands(CommandBase):
                 return 1
 
             # Prepend the debugger args.
-            args = ([self.debuggerInfo.path] + self.debuggerInfo.args
+            args = ([self.debuggerInfo.path] + self.debuggerInfo.args +
-                    + args + params)
+                    args + params)
         else:
             args = args + params
 
@@ -132,8 +131,7 @@ class MachCommands(CommandBase):
         help="Command-line arguments to be passed through to cargo doc")
     def doc(self, params):
         self.ensure_bootstrapped()
-        if not path.exists(path.join(
+        if not path.exists(path.join(self.config["tools"]["rust-root"], "doc")):
-               self.config["tools"]["rust-root"], "doc")):
             Registrar.dispatch("bootstrap-rust-docs", context=self.context)
         rust_docs = path.join(self.config["tools"]["rust-root"], "doc")
         docs = path.join(

python/servo/testing_commands.py

@@ -281,7 +281,6 @@ class MachCommands(CommandBase):
         execfile(run_file, run_globals)
         return run_globals["update_tests"](**kwargs)
 
-
     def ensure_wpt_virtualenv(self):
         virtualenv_path = path.join("tests", "wpt", "_virtualenv")
         python = self.get_exec("python2", "python")
@@ -295,8 +294,8 @@ class MachCommands(CommandBase):
         execfile(activate_path, dict(__file__=activate_path))
 
         try:
-            import wptrunner
+            import wptrunner  # noqa
-            from wptrunner.browsers import servo
+            from wptrunner.browsers import servo  # noqa
         except ImportError:
             subprocess.check_call(["pip", "install", "-r",
                                    path.join("tests", "wpt", "harness", "requirements.txt")])
@@ -311,7 +310,7 @@ class MachCommands(CommandBase):
         # before the virtualenv is initalised it doesn't see the blessings module so we don't
         # get coloured output. Setting the blessings global explicitly fixes that.
         from mozlog.structured.formatters import machformatter
-        import blessings
+        import blessings  # noqa
         machformatter.blessings = blessings
 
     def get_exec(self, name, default=None):

python/tidy.py

@@ -7,17 +7,21 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-#!/usr/bin/env python
-
 import os
 import fnmatch
 import itertools
 import re
+import sys
 from licenseck import licenses
 
 filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".py"]
 reftest_directories = ["tests/ref"]
 reftest_filetype = ".list"
+python_dependencies = [
+    "./python/dependencies/flake8-2.4.1-py2.py3-none-any.whl",
+    "./python/dependencies/pep8-1.5.7-py2.py3-none-any.whl",
+    "./python/dependencies/pyflakes-0.9.0-py2.py3-none-any.whl",
+]
 
 ignored_files = [
     # Upstream
@@ -80,12 +84,14 @@ def check_length(idx, line):
     if len(line) >= 120:
         yield (idx + 1, "(much) overlong line")
 
+
 def check_whatwg_url(idx, line):
     match = re.search(r"https://html\.spec\.whatwg\.org/multipage/[\w-]+\.html#([\w\:-]+)", line)
     if match is not None:
         preferred_link = "https://html.spec.whatwg.org/multipage/#{}".format(match.group(1))
         yield (idx + 1, "link to WHATWG may break in the future, use this format instead: {}".format(preferred_link))
 
+
 def check_whitespace(idx, line):
     if line[-1] == "\n":
         line = line[:-1]
@@ -101,6 +107,7 @@ def check_whitespace(idx, line):
     if "\r" in line:
         yield (idx + 1, "CR on line")
 
+
 def check_by_line(contents):
     lines = contents.splitlines(True)
     for idx, line in enumerate(lines):
@@ -113,6 +120,25 @@ def check_by_line(contents):
             yield error
 
 
+def check_flake8(file_paths):
+    from flake8.main import check_file
+
+    ignore = {
+        "W291",  # trailing whitespace; the standard tidy process will enforce no trailing whitespace
+        "E501",  # 80 character line length; the standard tidy process will enforce line length
+    }
+
+    num_errors = 0
+
+    for file_path in file_paths:
+        if os.path.splitext(file_path)[-1].lower() != ".py":
+            continue
+
+        num_errors += check_file(file_path, ignore=ignore)
+
+    return num_errors
+
+
 def collect_errors_for_files(files_to_check, checking_functions):
     for file_name in files_to_check:
         with open(file_name, "r") as fp:
@@ -146,9 +172,13 @@ def get_reftest_names(line):
 
 
 def scan():
+    sys.path += python_dependencies
+
     all_files = collect_file_names()
     files_to_check = filter(should_check, all_files)
 
+    num_flake8_errors = check_flake8(files_to_check)
+
     checking_functions = [check_license, check_by_line]
     errors = collect_errors_for_files(files_to_check, checking_functions)
 
@@ -158,7 +188,7 @@ def scan():
 
     errors = list(itertools.chain(errors, r_errors))
 
-    if errors:
+    if errors or num_flake8_errors:
         for error in errors:
             print("{}:{}: {}".format(*error))
         return 1

tests/jquery/run_jquery.py

@@ -48,6 +48,7 @@ TEST_SERVER_PORT = 8192
 # A regex for matching console.log output lines from the test runner.
 REGEX_PATTERN = "^\[jQuery test\] \[([0-9]+)/([0-9]+)/([0-9]+)] (.*)"
 
+
 # The result of a single test group.
 class TestResult:
     def __init__(self, success, fail, total, text):
@@ -107,26 +108,31 @@ def run_servo(servo_exe, module):
         except AttributeError:
             pass
 
+
 # Build the filename for an expected results file.
 def module_filename(module):
     return 'expected_{0}.txt'.format(module)
 
+
 # Read an existing set of expected results to compare against.
 def read_existing_results(module):
     with open(module_filename(module), 'r') as file:
         buffer = file.read()
         return parse_string_to_results(buffer)
 
+
 # Write a set of results to file
 def write_results(module, results):
     with open(module_filename(module), 'w') as file:
         for result in test_results.itervalues():
             file.write(result.text + '\n')
 
+
 # Print usage if command line args are incorrect
 def print_usage():
     print("USAGE: {0} servo_binary jquery_base_dir test|update".format(sys.argv[0]))
 
+
 # Run a simple HTTP server to serve up the jQuery test suite
 def run_http_server():
     class ThreadingSimpleServer(SocketServer.ThreadingMixIn,