Use one Python virtual environment for all mach commands

Prior to this commit:

* Our Python dependency story was a bit of a mess. We had complete
 Python packages (wheels and directories) living in-tree, despite
 not having any changes from upstream. This is particularly bad because
 `setup.py` never gets run on these packages which could (sometimes
 silently) unintended breakage.
* Python virtual environments (virtualenv) were only utilized for
 testing web-platform tests

After this commit:

* A single virtualenv (`python/_virtualenv`) is activated upon *every*
 call to mach
* A requirements file (`python/requirements.txt`) is added to describe
 the dependencies needed by Python modules in `python/`. The child
 commit immediately following this will remove all the dependencies
 no longer needed in-tree (for the sake of keeping this commit
 readable).

Relevant to https://github.com/servo/servo/issues/861

Fixes https://github.com/servo/servo/issues/6999

.gitignore

@@ -6,7 +6,7 @@
 /ports/android/libs
 /ports/android/local.properties
 /ports/android/obj
-/tests/wpt/_virtualenv
+/python/_virtualenv
 *~
 *#
 *.o

python/mach_bootstrap.py

@@ -6,7 +6,9 @@ from __future__ import print_function, unicode_literals
 
 import os
 import platform
+import subprocess
 import sys
+from distutils.spawn import find_executable
 
 SEARCH_PATHS = [
     "python/mach",
@@ -73,6 +75,39 @@ CATEGORIES = {
 }
 
 
+def _get_exec(name, default=None):
+    path = find_executable(name)
+    if not path:
+        return default
+    return path
+
+
+def _activate_virtualenv(topdir):
+    virtualenv_path = os.path.join(topdir, "python", "_virtualenv")
+    python = _get_exec("python2", "python")
+
+    if not os.path.exists(virtualenv_path):
+        virtualenv = _get_exec("virtualenv2", "virtualenv")
+        subprocess.check_call([virtualenv, "-p", python, virtualenv_path])
+
+    activate_path = os.path.join(virtualenv_path, "bin", "activate_this.py")
+    execfile(activate_path, dict(__file__=activate_path))
+
+    # TODO: Right now, we iteratively install all the requirements by invoking
+    # `pip install` each time. If it were the case that there were conflicting
+    # requirements, we wouldn't know about them. Once
+    # https://github.com/pypa/pip/issues/988 is addressed, then we can just
+    # chain each of the requirements files into the same `pip install` call
+    # and it will check for conflicts.
+    requirements_paths = [
+        os.path.join(topdir, "python", "requirements.txt"),
+        os.path.join(topdir, "tests", "wpt", "harness", "requirements.txt"),
+        os.path.join(topdir, "tests", "wpt", "harness", "requirements_servo.txt"),
+    ]
+    for path in requirements_paths:
+        subprocess.check_call(["pip", "install", "-q", "-r", path])
+
+
 def bootstrap(topdir):
     topdir = os.path.abspath(topdir)
 
@@ -84,6 +119,8 @@ def bootstrap(topdir):
         print('You are running Python', platform.python_version())
         sys.exit(1)
 
+    _activate_virtualenv(topdir)
+
     def populate_context(context, key=None):
         if key is None:
             return

python/requirements.txt

@@ -0,0 +1,12 @@
+# 'mach' is not listed here because a new version hasn't been published to PyPi in a while
+
+blessings == 1.6
+mozdebug == 0.1
+mozinfo == 0.8
+mozlog == 3.0
+toml == 0.9.1
+
+# For Python linting
+flake8 == 2.4.1
+pep8 == 1.5.7
+pyflakes == 0.8.0

python/servo/testing_commands.py

@@ -15,7 +15,6 @@ import os
 import os.path as path
 import subprocess
 from collections import OrderedDict
-from distutils.spawn import find_executable
 from time import time
 
 from mach.registrar import Registrar
@@ -237,7 +236,6 @@ class MachCommands(CommandBase):
                      help="Run with a release build of servo")
     def test_wpt(self, **kwargs):
         self.ensure_bootstrapped()
-        self.ensure_wpt_virtualenv()
         hosts_file_path = path.join('tests', 'wpt', 'hosts')
 
         os.environ["hosts_file_path"] = hosts_file_path
@@ -255,7 +253,6 @@ class MachCommands(CommandBase):
              parser=updatecommandline.create_parser())
     def update_wpt(self, **kwargs):
         self.ensure_bootstrapped()
-        self.ensure_wpt_virtualenv()
         run_file = path.abspath(path.join("tests", "wpt", "update.py"))
         run_globals = {"__file__": run_file}
         execfile(run_file, run_globals)
@@ -301,7 +298,6 @@ class MachCommands(CommandBase):
                      help="Run with a release build of servo")
     def test_css(self, **kwargs):
         self.ensure_bootstrapped()
-        self.ensure_wpt_virtualenv()
 
         run_file = path.abspath(path.join("tests", "wpt", "run_css.py"))
         run_globals = {"__file__": run_file}
@@ -320,45 +316,6 @@ class MachCommands(CommandBase):
         execfile(run_file, run_globals)
         return run_globals["update_tests"](**kwargs)
 
-    def ensure_wpt_virtualenv(self):
-        virtualenv_path = path.join("tests", "wpt", "_virtualenv")
-        python = self.get_exec("python2", "python")
-
-        if not os.path.exists(virtualenv_path):
-            virtualenv = self.get_exec("virtualenv2", "virtualenv")
-            subprocess.check_call([virtualenv, "-p", python, virtualenv_path])
-
-        activate_path = path.join(virtualenv_path, "bin", "activate_this.py")
-
-        execfile(activate_path, dict(__file__=activate_path))
-
-        try:
-            import wptrunner  # noqa
-            from wptrunner.browsers import servo  # noqa
-        except ImportError:
-            subprocess.check_call(["pip", "install", "-r",
-                                   path.join("tests", "wpt", "harness", "requirements.txt")])
-            subprocess.check_call(["pip", "install", "-r",
-                                   path.join("tests", "wpt", "harness", "requirements_servo.txt")])
-        try:
-            import blessings
-        except ImportError:
-            subprocess.check_call(["pip", "install", "blessings"])
-
-        # This is an unfortunate hack. Because mozlog gets imported by wptcommandline
-        # before the virtualenv is initalised it doesn't see the blessings module so we don't
-        # get coloured output. Setting the blessings global explicitly fixes that.
-        from mozlog.structured.formatters import machformatter
-        import blessings  # noqa
-        machformatter.blessings = blessings
-
-    def get_exec(self, name, default=None):
-        path = find_executable(name)
-        if not path:
-            return default
-
-        return path
-
     def jquery_test_runner(self, cmd, release, dev):
         self.ensure_bootstrapped()
         base_dir = path.abspath(path.join("tests", "jquery"))

python/tidy.py

@@ -19,11 +19,6 @@ from licenseck import licenses
 filetypes_to_check = [".rs", ".rc", ".cpp", ".c", ".h", ".py", ".toml", ".webidl"]
 reftest_directories = ["tests/ref"]
 reftest_filetype = ".list"
-python_dependencies = [
-    "./python/dependencies/flake8-2.4.1-py2.py3-none-any.whl",
-    "./python/dependencies/pep8-1.5.7-py2.py3-none-any.whl",
-    "./python/dependencies/pyflakes-0.9.0-py2.py3-none-any.whl",
-]
 
 ignored_files = [
     # Upstream
@@ -36,6 +31,7 @@ ignored_files = [
     "python/toml/*",
     "components/script/dom/bindings/codegen/parser/*",
     "components/script/dom/bindings/codegen/ply/*",
+    "python/_virtualenv/*",
 
     # Generated and upstream code combined with our own. Could use cleanup
     "target/*",
@@ -272,8 +268,6 @@ def get_reftest_names(line):
 
 
 def scan():
-    sys.path += python_dependencies
-
     all_files = collect_file_names()
     files_to_check = filter(should_check, all_files)