Auto merge of #10590 - edunham:package-tidy, r=larsbergstrom

Package tidy

This fixes https://github.com/servo/servo/issues/861.

@askeing, I've copied your work from https://github.com/askeing/servo_tidy and attributed the commit to you. My commit in this PR is Git housekeeping to preserve `tidy`'s history. If you'd like to make additional changes, I've given you and @shinglyu push access to my fork of Servo. Apologies if this is already familiar, but the workflow for pushing to my branch is:

```
$ git remote add edunham git@github.com:edunham/servo.git
$ git checkout -b package-tidy
$ git pull edunham package-tidy
$ git push edunham package-tidy
```

Once this lands, I'll look at how to publish it to PyPI and automate that process.

Please don't merge this yet; we still need to discuss how the change should work around https://github.com/servo/servo/blob/master/python/servo/testing_commands.py#L33 , as I've yet to figure out how to get the egg to actually expose its tests.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/10590)
<!-- Reviewable:end -->

python/README.md

@@ -10,3 +10,8 @@ harnesses.
 
 servo-specific python code e.g. implementations of mach commands. This
 is the canonical repository for this code.
+
+= tidy =
+
+servo-tidy is used to check licenses, line lengths, whitespace, flake8 on
+Python files, lock file versions, and more.

python/mach_bootstrap.py

@@ -13,6 +13,7 @@ from pipes import quote
 
 SEARCH_PATHS = [
     os.path.join("python", "mach"),
+    os.path.join("python", "tidy"),
     os.path.join("tests", "wpt"),
     os.path.join("tests", "wpt", "harness"),
 ]

python/requirements.txt

@@ -13,3 +13,5 @@ pyflakes == 0.8.0
 
 # For test-webidl
 ply == 3.8
+
+-e python/tidy

python/servo/testing_commands.py

@@ -29,8 +29,8 @@ from mach.decorators import (
 from servo.command_base import CommandBase, call, check_call
 from wptrunner import wptcommandline
 from update import updatecommandline
-import tidy
+from servo_tidy import tidy
-from tidy_self_test import tidy_self_test
+from servo_tidy_tests import test_tidy
 
 SCRIPT_PATH = os.path.split(__file__)[0]
 PROJECT_TOPLEVEL_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, "..", ".."))
@@ -279,7 +279,7 @@ class MachCommands(CommandBase):
                      help="Run unit tests for tidy")
     def test_tidy(self, faster, no_progress, self_test):
         if self_test:
-            return tidy_self_test.do_tests()
+            return test_tidy.do_tests()
         else:
             return tidy.scan(faster, not no_progress)
 

python/tidy/HISTORY.rst

@@ -0,0 +1,6 @@
+Release History
+---------------
+
+0.0.1 (2016-04-12)
+++++++++++++++++++
+- Package Tidy

python/tidy/Makefile

@@ -0,0 +1,26 @@
+PYTHON := python
+VENV := env-$(PYTHON)
+
+# for travis
+
+$(VENV)/bin/python:
+	[ -d $(VENV) ] || $(PYTHON) -m virtualenv $(VENV) || virtualenv $(VENV)
+	$(VENV)/bin/pip install --upgrade setuptools
+	$(VENV)/bin/python setup.py develop
+
+
+.PHONY: dev-env
+dev-env: $(VENV)/bin/python
+
+
+# for testing
+.PHONY: test
+test: dev-env
+	$(VENV)/bin/python -m unittest discover -s servo_tidy_tests -v
+
+
+.PHONY: clean
+clean:
+	find . -name "*.pyc" -type f -delete
+
+

python/tidy/README.rst

@@ -0,0 +1,12 @@
+servo_tidy
+==========
+
+Servo's code and license checker.
+
+Install from PyPI (coming soon) or
+
+```
+pip install -e git+https://github.com/servo/servo.git#egg=servo_tidy&subdirectory=python/tidy
+```
+
+

python/tidy/servo_tidy/licenseck.py

@@ -0,0 +1,62 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+
+# These licenses are valid for use in Servo
+licenses = [
+
+"""\
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+""",
+
+"""\
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""",
+
+"""\
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""",
+
+"""\
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+""",
+
+"""\
+// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+""",
+
+"""\
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+""",
+
+]  # noqa: Indicate to flake8 that we do not want to check indentation here

python/tidy/servo_tidy/tidy.py

@@ -56,7 +56,7 @@ ignored_dirs = [
     os.path.join(".", "tests", "wpt", "sync"),
     os.path.join(".", "tests", "wpt", "sync_css"),
     os.path.join(".", "python", "mach"),
-    os.path.join(".", "python", "tidy_self_test"),
+    os.path.join(".", "python", "tidy", "servo_tidy_tests"),
     os.path.join(".", "components", "script", "dom", "bindings", "codegen", "parser"),
     os.path.join(".", "components", "script", "dom", "bindings", "codegen", "ply"),
     os.path.join(".", "python", "_virtualenv"),
@@ -69,8 +69,6 @@ ignored_dirs = [
     os.path.join(".", "."),
 ]
 
-spec_base_path = "components/script/dom/"
-
 
 def is_iter_empty(iterator):
     try:
@@ -223,7 +221,7 @@ def check_lock(file_name, contents):
         raise StopIteration
 
     # package names to be neglected (as named by cargo)
-    exceptions = ["bitflags", "xml-rs"]
+    exceptions = ["bitflags", "xml-rs", "byteorder"]
 
     import toml
     content = toml.loads(contents)
@@ -527,9 +525,10 @@ def check_json(filename, contents):
 
 
 def check_spec(file_name, lines):
-    if spec_base_path not in file_name:
+    base_path = "components/script/dom/"
+    if base_path not in file_name:
         raise StopIteration
-    file_name = os.path.relpath(os.path.splitext(file_name)[0], spec_base_path)
+    file_name = os.path.relpath(os.path.splitext(file_name)[0], base_path)
     patt = re.compile("^\s*\/\/.+")
 
     # Pattern representing a line with a macro

python/tidy/servo_tidy_tests/__init__.py

@@ -0,0 +1,8 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.

python/tidy/servo_tidy_tests/test_tidy.py

@@ -0,0 +1,75 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+import unittest
+from servo_tidy import tidy
+
+
+def iterFile(name):
+    path = 'servo_tidy_tests/' if os.path.exists('servo_tidy_tests/') else 'python/tidy/servo_tidy_tests/'
+    return iter([os.path.join(path, name)])
+
+
+class CheckTidiness(unittest.TestCase):
+    def test_spaces_correctnes(self):
+        errors = tidy.collect_errors_for_files(iterFile('wrong_space.rs'), [], [tidy.check_by_line])
+        self.assertEqual('trailing whitespace', errors.next()[2])
+        self.assertEqual('no newline at EOF', errors.next()[2])
+        self.assertEqual('tab on line', errors.next()[2])
+        self.assertEqual('CR on line', errors.next()[2])
+
+    def test_long_line(self):
+        errors = tidy.collect_errors_for_files(iterFile('long_line.rs'), [], [tidy.check_by_line])
+        self.assertEqual('Line is longer than 120 characters', errors.next()[2])
+
+    def test_whatwg_link(self):
+        errors = tidy.collect_errors_for_files(iterFile('whatwg_link.rs'), [], [tidy.check_by_line])
+        self.assertTrue('link to WHATWG may break in the future, use this format instead:' in errors.next()[2])
+        self.assertTrue('links to WHATWG single-page url, change to multi page:' in errors.next()[2])
+
+    def test_licence(self):
+        errors = tidy.collect_errors_for_files(iterFile('incorrect_license.rs'), [], [tidy.check_license])
+        self.assertEqual('incorrect license', errors.next()[2])
+
+    def test_rust(self):
+        errors = tidy.collect_errors_for_files(iterFile('rust_tidy.rs'), [], [tidy.check_rust])
+        self.assertEqual('use statement spans multiple lines', errors.next()[2])
+        self.assertEqual('missing space before }', errors.next()[2])
+        self.assertTrue('use statement is not in alphabetical order' in errors.next()[2])
+        self.assertEqual('encountered whitespace following a use statement', errors.next()[2])
+        self.assertTrue('mod declaration is not in alphabetical order' in errors.next()[2])
+        self.assertEqual('mod declaration spans multiple lines', errors.next()[2])
+        self.assertTrue('extern crate declaration is not in alphabetical order' in errors.next()[2])
+        self.assertEqual('missing space before ->', errors.next()[2])
+        self.assertEqual('missing space after ->', errors.next()[2])
+        self.assertEqual('missing space after :', errors.next()[2])
+        self.assertEqual('missing space before {', errors.next()[2])
+        self.assertEqual('missing space before =', errors.next()[2])
+        self.assertEqual('missing space after =', errors.next()[2])
+        self.assertEqual('missing space before -', errors.next()[2])
+        self.assertEqual('missing space before *', errors.next()[2])
+        self.assertEqual('missing space after =>', errors.next()[2])
+        self.assertEqual('extra space before :', errors.next()[2])
+        self.assertEqual('extra space before :', errors.next()[2])
+        self.assertEqual('use &[T] instead of &Vec<T>', errors.next()[2])
+        self.assertEqual('use &str instead of &String', errors.next()[2])
+
+    def test_webidl(self):
+        errors = tidy.collect_errors_for_files(iterFile('spec.webidl'), [tidy.check_webidl_spec], [])
+        self.assertEqual('No specification link found.', errors.next()[2])
+
+    def test_toml(self):
+        errors = tidy.collect_errors_for_files(iterFile('test.toml'), [tidy.check_toml], [])
+        self.assertEqual('found asterisk instead of minimum version number', errors.next()[2])
+
+
+def do_tests():
+    suite = unittest.TestLoader().loadTestsFromTestCase(CheckTidiness)
+    unittest.TextTestRunner(verbosity=2).run(suite)

python/tidy/setup.py

@@ -0,0 +1,50 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+import os
+from setuptools import setup, find_packages
+
+
+VERSION = '0.0.1'
+
+install_requires = [
+]
+
+here = os.path.dirname(os.path.abspath(__file__))
+# get documentation from the README and HISTORY
+try:
+    with open(os.path.join(here, 'README.rst')) as doc:
+        readme = doc.read()
+except:
+    readme = ''
+
+try:
+    with open(os.path.join(here, 'HISTORY.rst')) as doc:
+        history = doc.read()
+except:
+    history = ''
+
+long_description = readme + '\n\n' + history
+
+if __name__ == '__main__':
+    setup(
+        name='servo_tidy',
+        version=VERSION,
+        description='The servo-tidy is used to check licenses, '
+                    'line lengths, whitespace, flake8 on Python files, lock file versions, and more.',
+        long_description=long_description,
+        keywords='mozilla servo tidy ',
+        author='The Servo Project Developers',
+        author_email='dev-servo@lists.mozilla.org',
+        url='https://github.com/servo/servo',
+        packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
+        package_data={},
+        install_requires=install_requires,
+        zip_safe=False,
+    )