Vendor mach-1.0.0.

python/mach/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 1.1
+Name: mach
+Version: 1.0.0
+Summary: Generic command line command dispatching framework.
+Home-page: https://developer.mozilla.org/en-US/docs/Developer_Guide/mach
+Author: Gregory Szorc
+Author-email: gregory.szorc@gmail.com
+License: MPL 2.0
+Description: ====
+        mach
+        ====
+        
+        Mach (German for *do*) is a generic command dispatcher for the command
+        line.
+        
+        To use mach, you install the mach core (a Python package), create an
+        executable *driver* script (named whatever you want), and write mach
+        commands. When the *driver* is executed, mach dispatches to the
+        requested command handler automatically.
+        
+        To learn more, read the docs in ``docs/``.
+        
+Platform: UNKNOWN
+Classifier: Environment :: Console
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.5

python/mach/README.rst

@@ -0,0 +1,13 @@
+====
+mach
+====
+
+Mach (German for *do*) is a generic command dispatcher for the command
+line.
+
+To use mach, you install the mach core (a Python package), create an
+executable *driver* script (named whatever you want), and write mach
+commands. When the *driver* is executed, mach dispatches to the
+requested command handler automatically.
+
+To learn more, read the docs in ``docs/``.

python/mach/mach.egg-info/PKG-INFO

@@ -0,0 +1,29 @@
+Metadata-Version: 1.1
+Name: mach
+Version: 1.0.1
+Summary: Generic command line command dispatching framework.
+Home-page: https://developer.mozilla.org/en-US/docs/Developer_Guide/mach
+Author: Gregory Szorc
+Author-email: gregory.szorc@gmail.com
+License: MPL 2.0
+Description: ====
+        mach
+        ====
+        
+        Mach (German for *do*) is a generic command dispatcher for the command
+        line.
+        
+        To use mach, you install the mach core (a Python package), create an
+        executable *driver* script (named whatever you want), and write mach
+        commands. When the *driver* is executed, mach dispatches to the
+        requested command handler automatically.
+        
+        To learn more, read the docs in ``docs/``.
+        
+Platform: UNKNOWN
+Classifier: Environment :: Console
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.5

python/mach/mach.egg-info/SOURCES.txt

@@ -0,0 +1,21 @@
+README.rst
+setup.cfg
+setup.py
+mach/__init__.py
+mach/base.py
+mach/config.py
+mach/decorators.py
+mach/dispatcher.py
+mach/logging.py
+mach/main.py
+mach/registrar.py
+mach/terminal.py
+mach/util.py
+mach.egg-info/PKG-INFO
+mach.egg-info/SOURCES.txt
+mach.egg-info/dependency_links.txt
+mach.egg-info/requires.txt
+mach.egg-info/top_level.txt
+mach/mixin/__init__.py
+mach/mixin/logging.py
+mach/mixin/process.py

python/mach/mach.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

python/mach/mach.egg-info/requires.txt

@@ -0,0 +1,4 @@
+blessings
+mozfile
+mozprocess
+six

python/mach/mach.egg-info/top_level.txt

@@ -0,0 +1 @@
+mach

python/mach/mach/base.py

@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+
+class CommandContext(object):
+    """Holds run-time state so it can easily be passed to command providers."""
+    def __init__(self, cwd=None, settings=None, log_manager=None, commands=None, **kwargs):
+        self.cwd = cwd
+        self.settings = settings
+        self.log_manager = log_manager
+        self.commands = commands
+        self.command_attrs = {}
+
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+
+class MachError(Exception):
+    """Base class for all errors raised by mach itself."""
+
+
+class NoCommandError(MachError):
+    """No command was passed into mach."""
+
+
+class UnknownCommandError(MachError):
+    """Raised when we attempted to execute an unknown command."""
+
+    def __init__(self, command, verb, suggested_commands=None):
+        MachError.__init__(self)
+
+        self.command = command
+        self.verb = verb
+        self.suggested_commands = suggested_commands or []
+
+
+class UnrecognizedArgumentError(MachError):
+    """Raised when an unknown argument is passed to mach."""
+
+    def __init__(self, command, arguments):
+        MachError.__init__(self)
+
+        self.command = command
+        self.arguments = arguments
+
+
+class FailedCommandError(Exception):
+    """Raised by commands to signal a handled failure to be printed by mach
+
+    When caught by mach a FailedCommandError will print message and exit
+    with ''exit_code''. The optional ''reason'' is a string in cases where
+    other scripts may wish to handle the exception, though this is generally
+    intended to communicate failure to mach.
+    """
+
+    def __init__(self, message, exit_code=1, reason=''):
+        Exception.__init__(self, message)
+        self.exit_code = exit_code
+        self.reason = reason
+
+
+class MissingFileError(MachError):
+    """Attempted to load a mach commands file that doesn't exist."""

python/mach/mach/config.py

@@ -0,0 +1,419 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+r"""
+This file defines classes for representing config data/settings.
+
+Config data is modeled as key-value pairs. Keys are grouped together into named
+sections. Individual config settings (options) have metadata associated with
+them. This metadata includes type, default value, valid values, etc.
+
+The main interface to config data is the ConfigSettings class. 1 or more
+ConfigProvider classes are associated with ConfigSettings and define what
+settings are available.
+"""
+
+from __future__ import absolute_import, unicode_literals
+
+import collections
+import os
+import sys
+import six
+from functools import wraps
+from six.moves.configparser import RawConfigParser, NoSectionError
+from six import string_types
+
+
+class ConfigException(Exception):
+    pass
+
+
+class ConfigType(object):
+    """Abstract base class for config values."""
+
+    @staticmethod
+    def validate(value):
+        """Validates a Python value conforms to this type.
+
+        Raises a TypeError or ValueError if it doesn't conform. Does not do
+        anything if the value is valid.
+        """
+
+    @staticmethod
+    def from_config(config, section, option):
+        """Obtain the value of this type from a RawConfigParser.
+
+        Receives a RawConfigParser instance, a str section name, and the str
+        option in that section to retrieve.
+
+        The implementation may assume the option exists in the RawConfigParser
+        instance.
+
+        Implementations are not expected to validate the value. But, they
+        should return the appropriate Python type.
+        """
+
+    @staticmethod
+    def to_config(value):
+        return value
+
+
+class StringType(ConfigType):
+    @staticmethod
+    def validate(value):
+        if not isinstance(value, string_types):
+            raise TypeError()
+
+    @staticmethod
+    def from_config(config, section, option):
+        return config.get(section, option)
+
+
+class BooleanType(ConfigType):
+    @staticmethod
+    def validate(value):
+        if not isinstance(value, bool):
+            raise TypeError()
+
+    @staticmethod
+    def from_config(config, section, option):
+        return config.getboolean(section, option)
+
+    @staticmethod
+    def to_config(value):
+        return 'true' if value else 'false'
+
+
+class IntegerType(ConfigType):
+    @staticmethod
+    def validate(value):
+        if not isinstance(value, int):
+            raise TypeError()
+
+    @staticmethod
+    def from_config(config, section, option):
+        return config.getint(section, option)
+
+
+class PositiveIntegerType(IntegerType):
+    @staticmethod
+    def validate(value):
+        if not isinstance(value, int):
+            raise TypeError()
+
+        if value < 0:
+            raise ValueError()
+
+
+class PathType(StringType):
+    @staticmethod
+    def validate(value):
+        if not isinstance(value, string_types):
+            raise TypeError()
+
+    @staticmethod
+    def from_config(config, section, option):
+        return config.get(section, option)
+
+
+TYPE_CLASSES = {
+    'string': StringType,
+    'boolean': BooleanType,
+    'int': IntegerType,
+    'pos_int': PositiveIntegerType,
+    'path': PathType,
+}
+
+
+class DefaultValue(object):
+    pass
+
+
+def reraise_attribute_error(func):
+    """Used to make sure __getattr__ wrappers around __getitem__
+    raise AttributeError instead of KeyError.
+    """
+    @wraps(func)
+    def _(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except KeyError:
+            exc_class, exc, tb = sys.exc_info()
+            six.reraise(AttributeError().__class__, exc, tb)
+    return _
+
+
+class ConfigSettings(collections.Mapping):
+    """Interface for configuration settings.
+
+    This is the main interface to the configuration.
+
+    A configuration is a collection of sections. Each section contains
+    key-value pairs.
+
+    When an instance is created, the caller first registers ConfigProvider
+    instances with it. This tells the ConfigSettings what individual settings
+    are available and defines extra metadata associated with those settings.
+    This is used for validation, etc.
+
+    Once ConfigProvider instances are registered, a config is populated. It can
+    be loaded from files or populated by hand.
+
+    ConfigSettings instances are accessed like dictionaries or by using
+    attributes. e.g. the section "foo" is accessed through either
+    settings.foo or settings['foo'].
+
+    Sections are modeled by the ConfigSection class which is defined inside
+    this one. They look just like dicts or classes with attributes. To access
+    the "bar" option in the "foo" section:
+
+        value = settings.foo.bar
+        value = settings['foo']['bar']
+        value = settings.foo['bar']
+
+    Assignment is similar:
+
+        settings.foo.bar = value
+        settings['foo']['bar'] = value
+        settings['foo'].bar = value
+
+    You can even delete user-assigned values:
+
+        del settings.foo.bar
+        del settings['foo']['bar']
+
+    If there is a default, it will be returned.
+
+    When settings are mutated, they are validated against the registered
+    providers. Setting unknown settings or setting values to illegal values
+    will result in exceptions being raised.
+    """
+
+    class ConfigSection(collections.MutableMapping, object):
+        """Represents an individual config section."""
+        def __init__(self, config, name, settings):
+            object.__setattr__(self, '_config', config)
+            object.__setattr__(self, '_name', name)
+            object.__setattr__(self, '_settings', settings)
+
+            wildcard = any(s == '*' for s in self._settings)
+            object.__setattr__(self, '_wildcard', wildcard)
+
+        @property
+        def options(self):
+            try:
+                return self._config.options(self._name)
+            except NoSectionError:
+                return []
+
+        def get_meta(self, option):
+            if option in self._settings:
+                return self._settings[option]
+            if self._wildcard:
+                return self._settings['*']
+            raise KeyError('Option not registered with provider: %s' % option)
+
+        def _validate(self, option, value):
+            meta = self.get_meta(option)
+            meta['type_cls'].validate(value)
+
+            if 'choices' in meta and value not in meta['choices']:
+                raise ValueError("Value '%s' must be one of: %s" % (
+                                 value, ', '.join(sorted(meta['choices']))))
+
+        # MutableMapping interface
+        def __len__(self):
+            return len(self.options)
+
+        def __iter__(self):
+            return iter(self.options)
+
+        def __contains__(self, k):
+            return self._config.has_option(self._name, k)
+
+        def __getitem__(self, k):
+            meta = self.get_meta(k)
+
+            if self._config.has_option(self._name, k):
+                v = meta['type_cls'].from_config(self._config, self._name, k)
+            else:
+                v = meta.get('default', DefaultValue)
+
+            if v == DefaultValue:
+                raise KeyError('No default value registered: %s' % k)
+
+            self._validate(k, v)
+            return v
+
+        def __setitem__(self, k, v):
+            self._validate(k, v)
+            meta = self.get_meta(k)
+
+            if not self._config.has_section(self._name):
+                self._config.add_section(self._name)
+
+            self._config.set(self._name, k, meta['type_cls'].to_config(v))
+
+        def __delitem__(self, k):
+            self._config.remove_option(self._name, k)
+
+            # Prune empty sections.
+            if not len(self._config.options(self._name)):
+                self._config.remove_section(self._name)
+
+        @reraise_attribute_error
+        def __getattr__(self, k):
+            return self.__getitem__(k)
+
+        @reraise_attribute_error
+        def __setattr__(self, k, v):
+            self.__setitem__(k, v)
+
+        @reraise_attribute_error
+        def __delattr__(self, k):
+            self.__delitem__(k)
+
+    def __init__(self):
+        self._config = RawConfigParser()
+        self._config.optionxform = str
+
+        self._settings = {}
+        self._sections = {}
+        self._finalized = False
+        self.loaded_files = set()
+
+    def load_file(self, filename):
+        self.load_files([filename])
+
+    def load_files(self, filenames):
+        """Load a config from files specified by their paths.
+
+        Files are loaded in the order given. Subsequent files will overwrite
+        values from previous files. If a file does not exist, it will be
+        ignored.
+        """
+        filtered = [f for f in filenames if os.path.exists(f)]
+
+        fps = [open(f, 'rt') for f in filtered]
+        self.load_fps(fps)
+        self.loaded_files.update(set(filtered))
+        for fp in fps:
+            fp.close()
+
+    def load_fps(self, fps):
+        """Load config data by reading file objects."""
+
+        for fp in fps:
+            self._config.readfp(fp)
+
+    def write(self, fh):
+        """Write the config to a file object."""
+        self._config.write(fh)
+
+    @classmethod
+    def _format_metadata(cls, provider, section, option, type_cls, description,
+                         default=DefaultValue, extra=None):
+        """Formats and returns the metadata for a setting.
+
+        Each setting must have:
+
+            section -- str section to which the setting belongs. This is how
+                settings are grouped.
+
+            option -- str id for the setting. This must be unique within the
+                section it appears.
+
+            type -- a ConfigType-derived type defining the type of the setting.
+
+            description -- str describing how to use the setting and where it
+                applies.
+
+        Each setting has the following optional parameters:
+
+            default -- The default value for the setting. If None (the default)
+                there is no default.
+
+            extra -- A dict of additional key/value pairs to add to the
+                setting metadata.
+        """
+        if isinstance(type_cls, string_types):
+            type_cls = TYPE_CLASSES[type_cls]
+
+        meta = {
+            'description': description,
+            'type_cls': type_cls,
+        }
+
+        if default != DefaultValue:
+            meta['default'] = default
+
+        if extra:
+            meta.update(extra)
+
+        return meta
+
+    def register_provider(self, provider):
+        """Register a SettingsProvider with this settings interface."""
+
+        if self._finalized:
+            raise ConfigException('Providers cannot be registered after finalized.')
+
+        settings = provider.config_settings
+        if callable(settings):
+            settings = settings()
+
+        config_settings = collections.defaultdict(dict)
+        for setting in settings:
+            section, option = setting[0].split('.')
+
+            if option in config_settings[section]:
+                raise ConfigException('Setting has already been registered: %s.%s' % (
+                                section, option))
+
+            meta = self._format_metadata(provider, section, option, *setting[1:])
+            config_settings[section][option] = meta
+
+        for section_name, settings in config_settings.items():
+            section = self._settings.get(section_name, {})
+
+            for k, v in settings.items():
+                if k in section:
+                    raise ConfigException('Setting already registered: %s.%s' %
+                                          (section_name, k))
+
+                section[k] = v
+
+            self._settings[section_name] = section
+
+    def _finalize(self):
+        if self._finalized:
+            return
+
+        for section, settings in self._settings.items():
+            s = ConfigSettings.ConfigSection(self._config, section, settings)
+            self._sections[section] = s
+
+        self._finalized = True
+
+    # Mapping interface.
+    def __len__(self):
+        return len(self._settings)
+
+    def __iter__(self):
+        self._finalize()
+
+        return iter(self._sections.keys())
+
+    def __contains__(self, k):
+        return k in self._settings
+
+    def __getitem__(self, k):
+        self._finalize()
+
+        return self._sections[k]
+
+    # Allow attribute access because it looks nice.
+    @reraise_attribute_error
+    def __getattr__(self, k):
+        return self.__getitem__(k)

python/mach/mach/decorators.py

@@ -0,0 +1,348 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import argparse
+import collections
+import inspect
+import sys
+import types
+
+from .base import MachError
+from .registrar import Registrar
+
+
+class _MachCommand(object):
+    """Container for mach command metadata.
+
+    Mach commands contain lots of attributes. This class exists to capture them
+    in a sane way so tuples, etc aren't used instead.
+    """
+    __slots__ = (
+        # Content from decorator arguments to define the command.
+        'name',
+        'subcommand',
+        'category',
+        'description',
+        'conditions',
+        '_parser',
+        'arguments',
+        'argument_group_names',
+
+        # Describes how dispatch is performed.
+
+        # The Python class providing the command. This is the class type not
+        # an instance of the class. Mach will instantiate a new instance of
+        # the class if the command is executed.
+        'cls',
+
+        # Whether the __init__ method of the class should receive a mach
+        # context instance. This should only affect the mach driver and how
+        # it instantiates classes.
+        'pass_context',
+
+        # The name of the method providing the command. In other words, this
+        # is the str name of the attribute on the class type corresponding to
+        # the name of the function.
+        'method',
+
+        # Dict of string to _MachCommand defining sub-commands for this
+        # command.
+        'subcommand_handlers',
+    )
+
+    def __init__(self, name=None, subcommand=None, category=None,
+                 description=None, conditions=None, parser=None):
+        self.name = name
+        self.subcommand = subcommand
+        self.category = category
+        self.description = description
+        self.conditions = conditions or []
+        self._parser = parser
+        self.arguments = []
+        self.argument_group_names = []
+
+        self.cls = None
+        self.pass_context = None
+        self.method = None
+        self.subcommand_handlers = {}
+
+    @property
+    def parser(self):
+        # Creating CLI parsers at command dispatch time can be expensive. Make
+        # it possible to lazy load them by using functions.
+        if callable(self._parser):
+            self._parser = self._parser()
+
+        return self._parser
+
+    @property
+    def docstring(self):
+        return self.cls.__dict__[self.method].__doc__
+
+    def __ior__(self, other):
+        if not isinstance(other, _MachCommand):
+            raise ValueError('can only operate on _MachCommand instances')
+
+        for a in self.__slots__:
+            if not getattr(self, a):
+                setattr(self, a, getattr(other, a))
+
+        return self
+
+
+def CommandProvider(cls):
+    """Class decorator to denote that it provides subcommands for Mach.
+
+    When this decorator is present, mach looks for commands being defined by
+    methods inside the class.
+    """
+
+    # The implementation of this decorator relies on the parse-time behavior of
+    # decorators. When the module is imported, the method decorators (like
+    # @Command and @CommandArgument) are called *before* this class decorator.
+    # The side-effect of the method decorators is to store specifically-named
+    # attributes on the function types. We just scan over all functions in the
+    # class looking for the side-effects of the method decorators.
+
+    # Tell mach driver whether to pass context argument to __init__.
+    pass_context = False
+
+    isfunc = inspect.ismethod if sys.version_info < (3, 0) else inspect.isfunction
+    if isfunc(cls.__init__):
+        spec = inspect.getargspec(cls.__init__)
+
+        if len(spec.args) > 2:
+            msg = 'Mach @CommandProvider class %s implemented incorrectly. ' + \
+                  '__init__() must take 1 or 2 arguments. From %s'
+            msg = msg % (cls.__name__, inspect.getsourcefile(cls))
+            raise MachError(msg)
+
+        if len(spec.args) == 2:
+            pass_context = True
+
+    seen_commands = set()
+
+    # We scan __dict__ because we only care about the classes own attributes,
+    # not inherited ones. If we did inherited attributes, we could potentially
+    # define commands multiple times. We also sort keys so commands defined in
+    # the same class are grouped in a sane order.
+    for attr in sorted(cls.__dict__.keys()):
+        value = cls.__dict__[attr]
+
+        if not isinstance(value, types.FunctionType):
+            continue
+
+        command = getattr(value, '_mach_command', None)
+        if not command:
+            continue
+
+        # Ignore subcommands for now: we handle them later.
+        if command.subcommand:
+            continue
+
+        seen_commands.add(command.name)
+
+        if not command.conditions and Registrar.require_conditions:
+            continue
+
+        msg = 'Mach command \'%s\' implemented incorrectly. ' + \
+              'Conditions argument must take a list ' + \
+              'of functions. Found %s instead.'
+
+        if not isinstance(command.conditions, collections.Iterable):
+            msg = msg % (command.name, type(command.conditions))
+            raise MachError(msg)
+
+        for c in command.conditions:
+            if not hasattr(c, '__call__'):
+                msg = msg % (command.name, type(c))
+                raise MachError(msg)
+
+        command.cls = cls
+        command.method = attr
+        command.pass_context = pass_context
+
+        Registrar.register_command_handler(command)
+
+    # Now do another pass to get sub-commands. We do this in two passes so
+    # we can check the parent command existence without having to hold
+    # state and reconcile after traversal.
+    for attr in sorted(cls.__dict__.keys()):
+        value = cls.__dict__[attr]
+
+        if not isinstance(value, types.FunctionType):
+            continue
+
+        command = getattr(value, '_mach_command', None)
+        if not command:
+            continue
+
+        # It is a regular command.
+        if not command.subcommand:
+            continue
+
+        if command.name not in seen_commands:
+            raise MachError('Command referenced by sub-command does not '
+                            'exist: %s' % command.name)
+
+        if command.name not in Registrar.command_handlers:
+            continue
+
+        command.cls = cls
+        command.method = attr
+        command.pass_context = pass_context
+        parent = Registrar.command_handlers[command.name]
+
+        if command.subcommand in parent.subcommand_handlers:
+            raise MachError('sub-command already defined: %s' % command.subcommand)
+
+        parent.subcommand_handlers[command.subcommand] = command
+
+    return cls
+
+
+class Command(object):
+    """Decorator for functions or methods that provide a mach command.
+
+    The decorator accepts arguments that define basic attributes of the
+    command. The following arguments are recognized:
+
+         category -- The string category to which this command belongs. Mach's
+             help will group commands by category.
+
+         description -- A brief description of what the command does.
+
+         parser -- an optional argparse.ArgumentParser instance or callable
+             that returns an argparse.ArgumentParser instance to use as the
+             basis for the command arguments.
+
+    For example:
+
+        @Command('foo', category='misc', description='Run the foo action')
+        def foo(self):
+            pass
+    """
+    def __init__(self, name, **kwargs):
+        self._mach_command = _MachCommand(name=name, **kwargs)
+
+    def __call__(self, func):
+        if not hasattr(func, '_mach_command'):
+            func._mach_command = _MachCommand()
+
+        func._mach_command |= self._mach_command
+
+        return func
+
+
+class SubCommand(object):
+    """Decorator for functions or methods that provide a sub-command.
+
+    Mach commands can have sub-commands. e.g. ``mach command foo`` or
+    ``mach command bar``. Each sub-command has its own parser and is
+    effectively its own mach command.
+
+    The decorator accepts arguments that define basic attributes of the
+    sub command:
+
+        command -- The string of the command this sub command should be
+        attached to.
+
+        subcommand -- The string name of the sub command to register.
+
+        description -- A textual description for this sub command.
+    """
+    def __init__(self, command, subcommand, description=None, parser=None):
+        self._mach_command = _MachCommand(name=command, subcommand=subcommand,
+                                          description=description, parser=parser)
+
+    def __call__(self, func):
+        if not hasattr(func, '_mach_command'):
+            func._mach_command = _MachCommand()
+
+        func._mach_command |= self._mach_command
+
+        return func
+
+
+class CommandArgument(object):
+    """Decorator for additional arguments to mach subcommands.
+
+    This decorator should be used to add arguments to mach commands. Arguments
+    to the decorator are proxied to ArgumentParser.add_argument().
+
+    For example:
+
+        @Command('foo', help='Run the foo action')
+        @CommandArgument('-b', '--bar', action='store_true', default=False,
+            help='Enable bar mode.')
+        def foo(self):
+            pass
+    """
+    def __init__(self, *args, **kwargs):
+        if kwargs.get('nargs') == argparse.REMAINDER:
+            # These are the assertions we make in dispatcher.py about
+            # those types of CommandArguments.
+            assert len(args) == 1
+            assert all(k in ('default', 'nargs', 'help', 'group') for k in kwargs)
+        self._command_args = (args, kwargs)
+
+    def __call__(self, func):
+        if not hasattr(func, '_mach_command'):
+            func._mach_command = _MachCommand()
+
+        func._mach_command.arguments.insert(0, self._command_args)
+
+        return func
+
+
+class CommandArgumentGroup(object):
+    """Decorator for additional argument groups to mach commands.
+
+    This decorator should be used to add arguments groups to mach commands.
+    Arguments to the decorator are proxied to
+    ArgumentParser.add_argument_group().
+
+    For example:
+
+        @Command('foo', helps='Run the foo action')
+        @CommandArgumentGroup('group1')
+        @CommandArgument('-b', '--bar', group='group1', action='store_true',
+            default=False, help='Enable bar mode.')
+        def foo(self):
+            pass
+
+    The name should be chosen so that it makes sense as part of the phrase
+    'Command Arguments for <name>' because that's how it will be shown in the
+    help message.
+    """
+    def __init__(self, group_name):
+        self._group_name = group_name
+
+    def __call__(self, func):
+        if not hasattr(func, '_mach_command'):
+            func._mach_command = _MachCommand()
+
+        func._mach_command.argument_group_names.insert(0, self._group_name)
+
+        return func
+
+
+def SettingsProvider(cls):
+    """Class decorator to denote that this class provides Mach settings.
+
+    When this decorator is encountered, the underlying class will automatically
+    be registered with the Mach registrar and will (likely) be hooked up to the
+    mach driver.
+    """
+    if not hasattr(cls, 'config_settings'):
+        raise MachError('@SettingsProvider must contain a config_settings attribute. It '
+                        'may either be a list of tuples, or a callable that returns a list '
+                        'of tuples. Each tuple must be of the form:\n'
+                        '(<section>.<option>, <type_cls>, <description>, <default>, <choices>)\n'
+                        'as specified by ConfigSettings._format_metadata.')
+
+    Registrar.register_settings_provider(cls)
+    return cls

python/mach/mach/dispatcher.py

@@ -0,0 +1,468 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+from __future__ import absolute_import, unicode_literals
+
+import argparse
+import difflib
+import shlex
+import sys
+
+from operator import itemgetter
+
+from .base import (
+    NoCommandError,
+    UnknownCommandError,
+    UnrecognizedArgumentError,
+)
+from .decorators import SettingsProvider
+
+
+@SettingsProvider
+class DispatchSettings():
+    config_settings = [
+        ('alias.*', 'string', """
+Create a command alias of the form `<alias>=<command> <args>`.
+Aliases can also be used to set default arguments:
+<command>=<command> <args>
+""".strip()),
+    ]
+
+
+class CommandFormatter(argparse.HelpFormatter):
+    """Custom formatter to format just a subcommand."""
+
+    def add_usage(self, *args):
+        pass
+
+
+class CommandAction(argparse.Action):
+    """An argparse action that handles mach commands.
+
+    This class is essentially a reimplementation of argparse's sub-parsers
+    feature. We first tried to use sub-parsers. However, they were missing
+    features like grouping of commands (http://bugs.python.org/issue14037).
+
+    The way this works involves light magic and a partial understanding of how
+    argparse works.
+
+    Arguments registered with an argparse.ArgumentParser have an action
+    associated with them. An action is essentially a class that when called
+    does something with the encountered argument(s). This class is one of those
+    action classes.
+
+    An instance of this class is created doing something like:
+
+        parser.add_argument('command', action=CommandAction, registrar=r)
+
+    Note that a mach.registrar.Registrar instance is passed in. The Registrar
+    holds information on all the mach commands that have been registered.
+
+    When this argument is registered with the ArgumentParser, an instance of
+    this class is instantiated. One of the subtle but important things it does
+    is tell the argument parser that it's interested in *all* of the remaining
+    program arguments. So, when the ArgumentParser calls this action, we will
+    receive the command name plus all of its arguments.
+
+    For more, read the docs in __call__.
+    """
+    def __init__(self, option_strings, dest, required=True, default=None,
+                 registrar=None, context=None):
+        # A proper API would have **kwargs here. However, since we are a little
+        # hacky, we intentionally omit it as a way of detecting potentially
+        # breaking changes with argparse's implementation.
+        #
+        # In a similar vein, default is passed in but is not needed, so we drop
+        # it.
+        argparse.Action.__init__(self, option_strings, dest, required=required,
+                                 help=argparse.SUPPRESS, nargs=argparse.REMAINDER)
+
+        self._mach_registrar = registrar
+        self._context = context
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        """This is called when the ArgumentParser has reached our arguments.
+
+        Since we always register ourselves with nargs=argparse.REMAINDER,
+        values should be a list of remaining arguments to parse. The first
+        argument should be the name of the command to invoke and all remaining
+        arguments are arguments for that command.
+
+        The gist of the flow is that we look at the command being invoked. If
+        it's *help*, we handle that specially (because argparse's default help
+        handler isn't satisfactory). Else, we create a new, independent
+        ArgumentParser instance for just the invoked command (based on the
+        information contained in the command registrar) and feed the arguments
+        into that parser. We then merge the results with the main
+        ArgumentParser.
+        """
+        if namespace.help:
+            # -h or --help is in the global arguments.
+            self._handle_main_help(parser, namespace.verbose)
+            sys.exit(0)
+        elif values:
+            command = values[0].lower()
+            args = values[1:]
+            if command == 'help':
+                if args and args[0] not in ['-h', '--help']:
+                    # Make sure args[0] is indeed a command.
+                    self._handle_command_help(parser, args[0], args, namespace.print_command)
+                else:
+                    self._handle_main_help(parser, namespace.verbose)
+                sys.exit(0)
+            elif '-h' in args or '--help' in args:
+                # -h or --help is in the command arguments.
+                if '--' in args:
+                    # -- is in command arguments
+                    if '-h' in args[:args.index('--')] or '--help' in args[:args.index('--')]:
+                        # Honor -h or --help only if it appears before --
+                        self._handle_command_help(parser, command, args, namespace.print_command)
+                        sys.exit(0)
+                else:
+                    self._handle_command_help(parser, command, args, namespace.print_command)
+                    sys.exit(0)
+        else:
+            raise NoCommandError()
+
+        # First see if the this is a user-defined alias
+        if command in self._context.settings.alias:
+            alias = self._context.settings.alias[command]
+            defaults = shlex.split(alias)
+            command = defaults.pop(0)
+            args = defaults + args
+
+        if command not in self._mach_registrar.command_handlers:
+            # Try to find similar commands, may raise UnknownCommandError.
+            command = self._suggest_command(command)
+
+        # This is used by the `mach` driver to find the command name amidst
+        # global arguments.
+        if namespace.print_command:
+            print(command)
+            sys.exit(0)
+
+        handler = self._mach_registrar.command_handlers.get(command)
+
+        usage = '%(prog)s [global arguments] ' + command + \
+            ' [command arguments]'
+
+        subcommand = None
+
+        # If there are sub-commands, parse the intent out immediately.
+        if handler.subcommand_handlers and args:
+            # mach <command> help <subcommand>
+            if set(args).intersection(('help', '--help')):
+                self._handle_subcommand_help(parser, handler, args)
+                sys.exit(0)
+            # mach <command> <subcommand> ...
+            elif args[0] in handler.subcommand_handlers:
+                subcommand = args[0]
+                handler = handler.subcommand_handlers[subcommand]
+                usage = '%(prog)s [global arguments] ' + command + ' ' + \
+                    subcommand + ' [command arguments]'
+                args.pop(0)
+
+        # We create a new parser, populate it with the command's arguments,
+        # then feed all remaining arguments to it, merging the results
+        # with ourselves. This is essentially what argparse subparsers
+        # do.
+
+        parser_args = {
+            'add_help': False,
+            'usage': usage,
+        }
+
+        remainder = None
+
+        if handler.parser:
+            subparser = handler.parser
+            subparser.context = self._context
+            for arg in subparser._actions[:]:
+                if arg.nargs == argparse.REMAINDER:
+                    subparser._actions.remove(arg)
+                    remainder = (arg.dest,), {'default': arg.default,
+                                              'nargs': arg.nargs,
+                                              'help': arg.help}
+        else:
+            subparser = argparse.ArgumentParser(**parser_args)
+
+        for arg in handler.arguments:
+            # Remove our group keyword; it's not needed here.
+            group_name = arg[1].get('group')
+            if group_name:
+                del arg[1]['group']
+
+            if arg[1].get('nargs') == argparse.REMAINDER:
+                # parse_known_args expects all argparse.REMAINDER ('...')
+                # arguments to be all stuck together. Instead, we want them to
+                # pick any extra argument, wherever they are.
+                # Assume a limited CommandArgument for those arguments.
+                assert len(arg[0]) == 1
+                assert all(k in ('default', 'nargs', 'help') for k in arg[1])
+                remainder = arg
+            else:
+                subparser.add_argument(*arg[0], **arg[1])
+
+        # We define the command information on the main parser result so as to
+        # not interfere with arguments passed to the command.
+        setattr(namespace, 'mach_handler', handler)
+        setattr(namespace, 'command', command)
+        setattr(namespace, 'subcommand', subcommand)
+
+        command_namespace, extra = subparser.parse_known_args(args)
+        setattr(namespace, 'command_args', command_namespace)
+        if remainder:
+            (name,), options = remainder
+            # parse_known_args usefully puts all arguments after '--' in
+            # extra, but also puts '--' there. We don't want to pass it down
+            # to the command handler. Note that if multiple '--' are on the
+            # command line, only the first one is removed, so that subsequent
+            # ones are passed down.
+            if '--' in extra:
+                extra.remove('--')
+
+            # Commands with argparse.REMAINDER arguments used to force the
+            # other arguments to be '+' prefixed. If a user now passes such
+            # an argument, if will silently end up in extra. So, check if any
+            # of the allowed arguments appear in a '+' prefixed form, and error
+            # out if that's the case.
+            for args, _ in handler.arguments:
+                for arg in args:
+                    arg = arg.replace('-', '+', 1)
+                    if arg in extra:
+                        raise UnrecognizedArgumentError(command, [arg])
+
+            if extra:
+                setattr(command_namespace, name, extra)
+            else:
+                setattr(command_namespace, name, options.get('default', []))
+        elif extra and handler.cls.__name__ != 'DeprecatedCommands':
+            raise UnrecognizedArgumentError(command, extra)
+
+    def _handle_main_help(self, parser, verbose):
+        # Since we don't need full sub-parser support for the main help output,
+        # we create groups in the ArgumentParser and populate each group with
+        # arguments corresponding to command names. This has the side-effect
+        # that argparse renders it nicely.
+        r = self._mach_registrar
+        disabled_commands = []
+
+        cats = [(k, v[2]) for k, v in r.categories.items()]
+        sorted_cats = sorted(cats, key=itemgetter(1), reverse=True)
+        for category, priority in sorted_cats:
+            group = None
+
+            for command in sorted(r.commands_by_category[category]):
+                handler = r.command_handlers[command]
+
+                # Instantiate a handler class to see if it should be filtered
+                # out for the current context or not. Condition functions can be
+                # applied to the command's decorator.
+                if handler.conditions:
+                    if handler.pass_context:
+                        instance = handler.cls(self._context)
+                    else:
+                        instance = handler.cls()
+
+                    is_filtered = False
+                    for c in handler.conditions:
+                        if not c(instance):
+                            is_filtered = True
+                            break
+                    if is_filtered:
+                        description = handler.description
+                        disabled_command = {'command': command, 'description': description}
+                        disabled_commands.append(disabled_command)
+                        continue
+
+                if group is None:
+                    title, description, _priority = r.categories[category]
+                    group = parser.add_argument_group(title, description)
+
+                description = handler.description
+                group.add_argument(command, help=description,
+                                   action='store_true')
+
+        if disabled_commands and 'disabled' in r.categories:
+            title, description, _priority = r.categories['disabled']
+            group = parser.add_argument_group(title, description)
+            if verbose:
+                for c in disabled_commands:
+                    group.add_argument(c['command'], help=c['description'],
+                                       action='store_true')
+
+        parser.print_help()
+
+    def _populate_command_group(self, parser, handler, group):
+        extra_groups = {}
+        for group_name in handler.argument_group_names:
+            group_full_name = 'Command Arguments for ' + group_name
+            extra_groups[group_name] = \
+                parser.add_argument_group(group_full_name)
+
+        for arg in handler.arguments:
+            # Apply our group keyword.
+            group_name = arg[1].get('group')
+            if group_name:
+                del arg[1]['group']
+                group = extra_groups[group_name]
+            group.add_argument(*arg[0], **arg[1])
+
+    def _handle_command_help(self, parser, command, args, print_command):
+        handler = self._mach_registrar.command_handlers.get(command)
+
+        if not handler:
+            raise UnknownCommandError(command, 'query')
+
+        if print_command:
+            print(command)
+            sys.exit(0)
+
+        if handler.subcommand_handlers:
+            self._handle_subcommand_help(parser, handler, args)
+            return
+
+        # This code is worth explaining. Because we are doing funky things with
+        # argument registration to allow the same option in both global and
+        # command arguments, we can't simply put all arguments on the same
+        # parser instance because argparse would complain. We can't register an
+        # argparse subparser here because it won't properly show help for
+        # global arguments. So, we employ a strategy similar to command
+        # execution where we construct a 2nd, independent ArgumentParser for
+        # just the command data then supplement the main help's output with
+        # this 2nd parser's. We use a custom formatter class to ignore some of
+        # the help output.
+        parser_args = {
+            'formatter_class': CommandFormatter,
+            'add_help': False,
+        }
+
+        if handler.parser:
+            c_parser = handler.parser
+            c_parser.context = self._context
+            c_parser.formatter_class = NoUsageFormatter
+            # Accessing _action_groups is a bit shady. We are highly dependent
+            # on the argparse implementation not changing. We fail fast to
+            # detect upstream changes so we can intelligently react to them.
+            group = c_parser._action_groups[1]
+
+            # By default argparse adds two groups called "positional arguments"
+            # and "optional arguments". We want to rename these to reflect standard
+            # mach terminology.
+            c_parser._action_groups[0].title = 'Command Parameters'
+            c_parser._action_groups[1].title = 'Command Arguments'
+
+            if not handler.description:
+                handler.description = c_parser.description
+                c_parser.description = None
+        else:
+            c_parser = argparse.ArgumentParser(**parser_args)
+            group = c_parser.add_argument_group('Command Arguments')
+
+        self._populate_command_group(c_parser, handler, group)
+
+        # Set the long help of the command to the docstring (if present) or
+        # the command decorator description argument (if present).
+        if handler.docstring:
+            parser.description = format_docstring(handler.docstring)
+        elif handler.description:
+            parser.description = handler.description
+
+        parser.usage = '%(prog)s [global arguments] ' + command + \
+            ' [command arguments]'
+
+        # This is needed to preserve line endings in the description field,
+        # which may be populated from a docstring.
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.print_help()
+        print('')
+        c_parser.print_help()
+
+    def _handle_subcommand_main_help(self, parser, handler):
+        parser.usage = '%(prog)s [global arguments] ' + handler.name + \
+            ' subcommand [subcommand arguments]'
+        group = parser.add_argument_group('Sub Commands')
+
+        for subcommand, subhandler in sorted(handler.subcommand_handlers.items()):
+            group.add_argument(subcommand, help=subhandler.description,
+                               action='store_true')
+
+        if handler.docstring:
+            parser.description = format_docstring(handler.docstring)
+
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+
+        parser.print_help()
+
+    def _handle_subcommand_help(self, parser, handler, args):
+        subcommand = set(args).intersection(list(handler.subcommand_handlers.keys()))
+        if not subcommand:
+            return self._handle_subcommand_main_help(parser, handler)
+
+        subcommand = subcommand.pop()
+        subhandler = handler.subcommand_handlers[subcommand]
+
+        c_parser = subhandler.parser or argparse.ArgumentParser(add_help=False)
+        c_parser.formatter_class = CommandFormatter
+
+        group = c_parser.add_argument_group('Sub Command Arguments')
+        self._populate_command_group(c_parser, subhandler, group)
+
+        if subhandler.docstring:
+            parser.description = format_docstring(subhandler.docstring)
+
+        parser.formatter_class = argparse.RawDescriptionHelpFormatter
+        parser.usage = '%(prog)s [global arguments] ' + handler.name + \
+            ' ' + subcommand + ' [command arguments]'
+
+        parser.print_help()
+        print('')
+        c_parser.print_help()
+
+    def _suggest_command(self, command):
+        # Make sure we don't suggest any deprecated commands.
+        names = [h.name for h in self._mach_registrar.command_handlers.values()
+                 if h.cls.__name__ != 'DeprecatedCommands']
+
+        # Bug 1577908 - We used to automatically re-execute the suggested
+        # command with the proper spelling. But because the `mach` driver now
+        # uses a whitelist to determine which command to run with Python 2, all
+        # misspellings are automatically run with Python 3 (and would fail if
+        # we were to correct a Python 2 command here). So we now suggest the
+        # command instead. Once the Python 3 migration has completed, we can
+        # turn autosuggestions back on. We could alternatively figure out a way
+        # to compare the suggested command against the mach whitelist.
+        suggested_commands = set(difflib.get_close_matches(command, names, cutoff=0.5))
+        suggested_commands |= {cmd for cmd in names if cmd.startswith(command)}
+        raise UnknownCommandError(command, 'run', suggested_commands)
+
+
+class NoUsageFormatter(argparse.HelpFormatter):
+    def _format_usage(self, *args, **kwargs):
+        return ""
+
+
+def format_docstring(docstring):
+    """Format a raw docstring into something suitable for presentation.
+
+    This function is based on the example function in PEP-0257.
+    """
+    if not docstring:
+        return ''
+    lines = docstring.expandtabs().splitlines()
+    indent = sys.maxsize
+    for line in lines[1:]:
+        stripped = line.lstrip()
+        if stripped:
+            indent = min(indent, len(line) - len(stripped))
+    trimmed = [lines[0].strip()]
+    if indent < sys.maxsize:
+        for line in lines[1:]:
+            trimmed.append(line[indent:].rstrip())
+    while trimmed and not trimmed[-1]:
+        trimmed.pop()
+    while trimmed and not trimmed[0]:
+        trimmed.pop(0)
+    return '\n'.join(trimmed)

python/mach/mach/logging.py

@@ -0,0 +1,298 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains logging functionality for mach. It essentially provides
+# support for a structured logging framework built on top of Python's built-in
+# logging framework.
+
+from __future__ import absolute_import, unicode_literals
+
+try:
+    import blessings
+except ImportError:
+    blessings = None
+
+import json
+import logging
+import sys
+import time
+
+
+def format_seconds(total):
+    """Format number of seconds to MM:SS.DD form."""
+
+    minutes, seconds = divmod(total, 60)
+
+    return '%2d:%05.2f' % (minutes, seconds)
+
+
+class ConvertToStructuredFilter(logging.Filter):
+    """Filter that converts unstructured records into structured ones."""
+    def filter(self, record):
+        if hasattr(record, 'action') and hasattr(record, 'params'):
+            return True
+
+        record.action = 'unstructured'
+        record.params = {'msg': record.getMessage()}
+        record.msg = '{msg}'
+
+        return True
+
+
+class StructuredJSONFormatter(logging.Formatter):
+    """Log formatter that writes a structured JSON entry."""
+
+    def format(self, record):
+        action = getattr(record, 'action', 'UNKNOWN')
+        params = getattr(record, 'params', {})
+
+        return json.dumps([record.created, action, params])
+
+
+class StructuredHumanFormatter(logging.Formatter):
+    """Log formatter that writes structured messages for humans.
+
+    It is important that this formatter never be added to a logger that
+    produces unstructured/classic log messages. If it is, the call to format()
+    could fail because the string could contain things (like JSON) that look
+    like formatting character sequences.
+
+    Because of this limitation, format() will fail with a KeyError if an
+    unstructured record is passed or if the structured message is malformed.
+    """
+    def __init__(self, start_time, write_interval=False, write_times=True):
+        self.start_time = start_time
+        self.write_interval = write_interval
+        self.write_times = write_times
+        self.last_time = None
+
+    def format(self, record):
+        f = record.msg.format(**record.params)
+
+        if not self.write_times:
+            return f
+
+        elapsed = self._time(record)
+
+        return '%s %s' % (format_seconds(elapsed), f)
+
+    def _time(self, record):
+        t = record.created - self.start_time
+
+        if self.write_interval and self.last_time is not None:
+            t = record.created - self.last_time
+
+        self.last_time = record.created
+
+        return t
+
+
+class StructuredTerminalFormatter(StructuredHumanFormatter):
+    """Log formatter for structured messages writing to a terminal."""
+
+    def set_terminal(self, terminal):
+        self.terminal = terminal
+        self._sgr0 = terminal.normal if terminal and blessings else ''
+
+    def format(self, record):
+        f = record.msg.format(**record.params)
+
+        if not self.write_times:
+            return f
+
+        t = self.terminal.blue(format_seconds(self._time(record)))
+
+        # Some processes (notably Clang) don't reset terminal attributes after
+        # printing newlines. This can lead to terminal attributes getting in a
+        # wonky state. Work around this by sending the sgr0 sequence after every
+        # line to reset all attributes. For programs that rely on the next line
+        # inheriting the same attributes, this will prevent that from happening.
+        # But that's better than "corrupting" the terminal.
+        return '%s %s%s' % (t, self._colorize(f), self._sgr0)
+
+    def _colorize(self, s):
+        if not self.terminal:
+            return s
+
+        result = s
+
+        reftest = s.startswith('REFTEST ')
+        if reftest:
+            s = s[8:]
+
+        if s.startswith('TEST-PASS'):
+            result = self.terminal.green(s[0:9]) + s[9:]
+        elif s.startswith('TEST-UNEXPECTED'):
+            result = self.terminal.red(s[0:20]) + s[20:]
+        elif s.startswith('TEST-START'):
+            result = self.terminal.yellow(s[0:10]) + s[10:]
+        elif s.startswith('TEST-INFO'):
+            result = self.terminal.yellow(s[0:9]) + s[9:]
+
+        if reftest:
+            result = 'REFTEST ' + result
+
+        return result
+
+
+class LoggingManager(object):
+    """Holds and controls global logging state.
+
+    An application should instantiate one of these and configure it as needed.
+
+    This class provides a mechanism to configure the output of logging data
+    both from mach and from the overall logging system (e.g. from other
+    modules).
+    """
+
+    def __init__(self):
+        self.start_time = time.time()
+
+        self.json_handlers = []
+        self.terminal_handler = None
+        self.terminal_formatter = None
+
+        self.root_logger = logging.getLogger()
+        self.root_logger.setLevel(logging.DEBUG)
+
+        # Installing NullHandler on the root logger ensures that *all* log
+        # messages have at least one handler. This prevents Python from
+        # complaining about "no handlers could be found for logger XXX."
+        self.root_logger.addHandler(logging.NullHandler())
+
+        mach_logger = logging.getLogger('mach')
+        mach_logger.setLevel(logging.DEBUG)
+
+        self.structured_filter = ConvertToStructuredFilter()
+
+        self.structured_loggers = [mach_logger]
+
+        self._terminal = None
+
+    @property
+    def terminal(self):
+        if not self._terminal and blessings:
+            # Sometimes blessings fails to set up the terminal. In that case,
+            # silently fail.
+            try:
+                terminal = blessings.Terminal(stream=sys.stdout)
+
+                if terminal.is_a_tty:
+                    self._terminal = terminal
+            except Exception:
+                pass
+
+        return self._terminal
+
+    def add_json_handler(self, fh):
+        """Enable JSON logging on the specified file object."""
+
+        # Configure the consumer of structured messages.
+        handler = logging.StreamHandler(stream=fh)
+        handler.setFormatter(StructuredJSONFormatter())
+        handler.setLevel(logging.DEBUG)
+
+        # And hook it up.
+        for logger in self.structured_loggers:
+            logger.addHandler(handler)
+
+        self.json_handlers.append(handler)
+
+    def add_terminal_logging(self, fh=sys.stdout, level=logging.INFO,
+                             write_interval=False, write_times=True):
+        """Enable logging to the terminal."""
+
+        formatter = StructuredHumanFormatter(self.start_time,
+                                             write_interval=write_interval,
+                                             write_times=write_times)
+
+        if self.terminal:
+            formatter = StructuredTerminalFormatter(self.start_time,
+                                                    write_interval=write_interval,
+                                                    write_times=write_times)
+            formatter.set_terminal(self.terminal)
+
+        handler = logging.StreamHandler(stream=fh)
+        handler.setFormatter(formatter)
+        handler.setLevel(level)
+
+        for logger in self.structured_loggers:
+            logger.addHandler(handler)
+
+        self.terminal_handler = handler
+        self.terminal_formatter = formatter
+
+    def replace_terminal_handler(self, handler):
+        """Replace the installed terminal handler.
+
+        Returns the old handler or None if none was configured.
+        If the new handler is None, removes any existing handler and disables
+        logging to the terminal.
+        """
+        old = self.terminal_handler
+
+        if old:
+            for logger in self.structured_loggers:
+                logger.removeHandler(old)
+
+        if handler:
+            for logger in self.structured_loggers:
+                logger.addHandler(handler)
+
+        self.terminal_handler = handler
+
+        return old
+
+    def enable_unstructured(self):
+        """Enable logging of unstructured messages."""
+        if self.terminal_handler:
+            self.terminal_handler.addFilter(self.structured_filter)
+            self.root_logger.addHandler(self.terminal_handler)
+
+    def disable_unstructured(self):
+        """Disable logging of unstructured messages."""
+        if self.terminal_handler:
+            self.terminal_handler.removeFilter(self.structured_filter)
+            self.root_logger.removeHandler(self.terminal_handler)
+
+    def register_structured_logger(self, logger, terminal=True, json=True):
+        """Register a structured logger.
+
+        This needs to be called for all structured loggers that don't chain up
+        to the mach logger in order for their output to be captured.
+        """
+        self.structured_loggers.append(logger)
+
+        if terminal and self.terminal_handler:
+            logger.addHandler(self.terminal_handler)
+
+        if json:
+            for handler in self.json_handlers:
+                logger.addHandler(handler)
+
+    def enable_all_structured_loggers(self, terminal=True, json=True):
+        """Enable logging of all structured messages from all loggers.
+
+        ``terminal`` and ``json`` determine which log handlers to operate
+        on. By default, all known handlers are operated on.
+        """
+        # Remove current handlers from all loggers so we don't double
+        # register handlers.
+        for logger in self.root_logger.manager.loggerDict.values():
+            # Some entries might be logging.PlaceHolder.
+            if not isinstance(logger, logging.Logger):
+                continue
+
+            if terminal:
+                logger.removeHandler(self.terminal_handler)
+
+            if json:
+                for handler in self.json_handlers:
+                    logger.removeHandler(handler)
+
+        # Wipe out existing registered structured loggers since they
+        # all propagate to root logger.
+        self.structured_loggers = []
+        self.register_structured_logger(self.root_logger, terminal=terminal,
+                                        json=json)

python/mach/mach/main.py

@@ -0,0 +1,613 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides functionality for the command-line build tool
+# (mach). It is packaged as a module because everything is a library.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import codecs
+import errno
+import imp
+import logging
+import os
+import sys
+import traceback
+import uuid
+from collections import Iterable
+
+from six import string_types
+
+from .base import (
+    CommandContext,
+    MachError,
+    MissingFileError,
+    NoCommandError,
+    UnknownCommandError,
+    UnrecognizedArgumentError,
+    FailedCommandError,
+)
+from .config import ConfigSettings
+from .decorators import (
+    CommandProvider,
+)
+from .dispatcher import CommandAction
+from .logging import LoggingManager
+from .registrar import Registrar
+from .util import setenv
+
+SUGGEST_MACH_BUSTED = r'''
+You can invoke |./mach busted| to check if this issue is already on file. If it
+isn't, please use |./mach busted file| to report it. If |./mach busted| is
+misbehaving, you can also inspect the dependencies of bug 1543241.
+'''.lstrip()
+
+MACH_ERROR = r'''
+The error occurred in mach itself. This is likely a bug in mach itself or a
+fundamental problem with a loaded module.
+
+'''.lstrip() + SUGGEST_MACH_BUSTED
+
+ERROR_FOOTER = r'''
+If filing a bug, please include the full output of mach, including this error
+message.
+
+The details of the failure are as follows:
+'''.lstrip()
+
+COMMAND_ERROR = r'''
+The error occurred in the implementation of the invoked mach command.
+
+This should never occur and is likely a bug in the implementation of that
+command.
+'''.lstrip() + SUGGEST_MACH_BUSTED
+
+MODULE_ERROR = r'''
+The error occurred in code that was called by the mach command. This is either
+a bug in the called code itself or in the way that mach is calling it.
+'''.lstrip() + SUGGEST_MACH_BUSTED
+
+NO_COMMAND_ERROR = r'''
+It looks like you tried to run mach without a command.
+
+Run |mach help| to show a list of commands.
+'''.lstrip()
+
+UNKNOWN_COMMAND_ERROR = r'''
+It looks like you are trying to %s an unknown mach command: %s
+%s
+Run |mach help| to show a list of commands.
+'''.lstrip()
+
+SUGGESTED_COMMANDS_MESSAGE = r'''
+Did you want to %s any of these commands instead: %s?
+'''
+
+UNRECOGNIZED_ARGUMENT_ERROR = r'''
+It looks like you passed an unrecognized argument into mach.
+
+The %s command does not accept the arguments: %s
+'''.lstrip()
+
+INVALID_ENTRY_POINT = r'''
+Entry points should return a list of command providers or directories
+containing command providers. The following entry point is invalid:
+
+    %s
+
+You are seeing this because there is an error in an external module attempting
+to implement a mach command. Please fix the error, or uninstall the module from
+your system.
+'''.lstrip()
+
+class ArgumentParser(argparse.ArgumentParser):
+    """Custom implementation argument parser to make things look pretty."""
+
+    def error(self, message):
+        """Custom error reporter to give more helpful text on bad commands."""
+        if not message.startswith('argument command: invalid choice'):
+            argparse.ArgumentParser.error(self, message)
+            assert False
+
+        print('Invalid command specified. The list of commands is below.\n')
+        self.print_help()
+        sys.exit(1)
+
+    def format_help(self):
+        text = argparse.ArgumentParser.format_help(self)
+
+        # Strip out the silly command list that would preceed the pretty list.
+        #
+        # Commands:
+        #   {foo,bar}
+        #     foo  Do foo.
+        #     bar  Do bar.
+        search = 'Commands:\n  {'
+        start = text.find(search)
+
+        if start != -1:
+            end = text.find('}\n', start)
+            assert end != -1
+
+            real_start = start + len('Commands:\n')
+            real_end = end + len('}\n')
+
+            text = text[0:real_start] + text[real_end:]
+
+        return text
+
+
+class ContextWrapper(object):
+    def __init__(self, context, handler):
+        object.__setattr__(self, '_context', context)
+        object.__setattr__(self, '_handler', handler)
+
+    def __getattribute__(self, key):
+        try:
+            return getattr(object.__getattribute__(self, '_context'), key)
+        except AttributeError as e:
+            try:
+                ret = object.__getattribute__(self, '_handler')(self, key)
+            except (AttributeError, TypeError):
+                # TypeError is in case the handler comes from old code not
+                # taking a key argument.
+                raise e
+            setattr(self, key, ret)
+            return ret
+
+    def __setattr__(self, key, value):
+        setattr(object.__getattribute__(self, '_context'), key, value)
+
+
+@CommandProvider
+class Mach(object):
+    """Main mach driver type.
+
+    This type is responsible for holding global mach state and dispatching
+    a command from arguments.
+
+    The following attributes may be assigned to the instance to influence
+    behavior:
+
+        populate_context_handler -- If defined, it must be a callable. The
+            callable signature is the following:
+                populate_context_handler(context, key=None)
+            It acts as a fallback getter for the mach.base.CommandContext
+            instance.
+            This allows to augment the context instance with arbitrary data
+            for use in command handlers.
+            For backwards compatibility, it is also called before command
+            dispatch without a key, allowing the context handler to add
+            attributes to the context instance.
+
+        require_conditions -- If True, commands that do not have any condition
+            functions applied will be skipped. Defaults to False.
+
+        settings_paths -- A list of files or directories in which to search
+            for settings files to load.
+
+    """
+
+    USAGE = """%(prog)s [global arguments] command [command arguments]
+
+mach (German for "do") is the main interface to the Mozilla build system and
+common developer tasks.
+
+You tell mach the command you want to perform and it does it for you.
+
+Some common commands are:
+
+    %(prog)s build     Build/compile the source tree.
+    %(prog)s help      Show full help, including the list of all commands.
+
+To see more help for a specific command, run:
+
+  %(prog)s help <command>
+"""
+
+    def __init__(self, cwd):
+        assert os.path.isdir(cwd)
+
+        self.cwd = cwd
+        self.log_manager = LoggingManager()
+        self.logger = logging.getLogger(__name__)
+        self.settings = ConfigSettings()
+        self.settings_paths = []
+
+        if 'MACHRC' in os.environ:
+            self.settings_paths.append(os.environ['MACHRC'])
+
+        self.log_manager.register_structured_logger(self.logger)
+        self.global_arguments = []
+        self.populate_context_handler = None
+
+    def add_global_argument(self, *args, **kwargs):
+        """Register a global argument with the argument parser.
+
+        Arguments are proxied to ArgumentParser.add_argument()
+        """
+
+        self.global_arguments.append((args, kwargs))
+
+    def load_commands_from_directory(self, path):
+        """Scan for mach commands from modules in a directory.
+
+        This takes a path to a directory, loads the .py files in it, and
+        registers and found mach command providers with this mach instance.
+        """
+        for f in sorted(os.listdir(path)):
+            if not f.endswith('.py') or f == '__init__.py':
+                continue
+
+            full_path = os.path.join(path, f)
+            module_name = 'mach.commands.%s' % f[0:-3]
+
+            self.load_commands_from_file(full_path, module_name=module_name)
+
+    def load_commands_from_file(self, path, module_name=None):
+        """Scan for mach commands from a file.
+
+        This takes a path to a file and loads it as a Python module under the
+        module name specified. If no name is specified, a random one will be
+        chosen.
+        """
+        if module_name is None:
+            # Ensure parent module is present otherwise we'll (likely) get
+            # an error due to unknown parent.
+            if 'mach.commands' not in sys.modules:
+                mod = imp.new_module('mach.commands')
+                sys.modules['mach.commands'] = mod
+
+            module_name = 'mach.commands.%s' % uuid.uuid4().hex
+
+        try:
+            imp.load_source(module_name, path)
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+            raise MissingFileError('%s does not exist' % path)
+
+    def load_commands_from_entry_point(self, group='mach.providers'):
+        """Scan installed packages for mach command provider entry points. An
+        entry point is a function that returns a list of paths to files or
+        directories containing command providers.
+
+        This takes an optional group argument which specifies the entry point
+        group to use. If not specified, it defaults to 'mach.providers'.
+        """
+        try:
+            import pkg_resources
+        except ImportError:
+            print("Could not find setuptools, ignoring command entry points",
+                  file=sys.stderr)
+            return
+
+        for entry in pkg_resources.iter_entry_points(group=group, name=None):
+            paths = entry.load()()
+            if not isinstance(paths, Iterable):
+                print(INVALID_ENTRY_POINT % entry)
+                sys.exit(1)
+
+            for path in paths:
+                if os.path.isfile(path):
+                    self.load_commands_from_file(path)
+                elif os.path.isdir(path):
+                    self.load_commands_from_directory(path)
+                else:
+                    print("command provider '%s' does not exist" % path)
+
+    def define_category(self, name, title, description, priority=50):
+        """Provide a description for a named command category."""
+
+        Registrar.register_category(name, title, description, priority)
+
+    @property
+    def require_conditions(self):
+        return Registrar.require_conditions
+
+    @require_conditions.setter
+    def require_conditions(self, value):
+        Registrar.require_conditions = value
+
+    def run(self, argv, stdin=None, stdout=None, stderr=None):
+        """Runs mach with arguments provided from the command line.
+
+        Returns the integer exit code that should be used. 0 means success. All
+        other values indicate failure.
+        """
+
+        # If no encoding is defined, we default to UTF-8 because without this
+        # Python 2.7 will assume the default encoding of ASCII. This will blow
+        # up with UnicodeEncodeError as soon as it encounters a non-ASCII
+        # character in a unicode instance. We simply install a wrapper around
+        # the streams and restore once we have finished.
+        stdin = sys.stdin if stdin is None else stdin
+        stdout = sys.stdout if stdout is None else stdout
+        stderr = sys.stderr if stderr is None else stderr
+
+        orig_stdin = sys.stdin
+        orig_stdout = sys.stdout
+        orig_stderr = sys.stderr
+
+        sys.stdin = stdin
+        sys.stdout = stdout
+        sys.stderr = stderr
+
+        orig_env = dict(os.environ)
+
+        try:
+            if sys.version_info < (3, 0):
+                if stdin.encoding is None:
+                    sys.stdin = codecs.getreader('utf-8')(stdin)
+
+                if stdout.encoding is None:
+                    sys.stdout = codecs.getwriter('utf-8')(stdout)
+
+                if stderr.encoding is None:
+                    sys.stderr = codecs.getwriter('utf-8')(stderr)
+
+            # Allow invoked processes (which may not have a handle on the
+            # original stdout file descriptor) to know if the original stdout
+            # is a TTY. This provides a mechanism to allow said processes to
+            # enable emitting code codes, for example.
+            if os.isatty(orig_stdout.fileno()):
+                setenv('MACH_STDOUT_ISATTY', '1')
+
+            return self._run(argv)
+        except KeyboardInterrupt:
+            print('mach interrupted by signal or user action. Stopping.')
+            return 1
+
+        except Exception:
+            # _run swallows exceptions in invoked handlers and converts them to
+            # a proper exit code. So, the only scenario where we should get an
+            # exception here is if _run itself raises. If _run raises, that's a
+            # bug in mach (or a loaded command module being silly) and thus
+            # should be reported differently.
+            self._print_error_header(argv, sys.stdout)
+            print(MACH_ERROR)
+
+            exc_type, exc_value, exc_tb = sys.exc_info()
+            stack = traceback.extract_tb(exc_tb)
+
+            self._print_exception(sys.stdout, exc_type, exc_value, stack)
+
+            return 1
+
+        finally:
+            os.environ.clear()
+            os.environ.update(orig_env)
+
+            sys.stdin = orig_stdin
+            sys.stdout = orig_stdout
+            sys.stderr = orig_stderr
+
+    def _run(self, argv):
+        # Load settings as early as possible so things in dispatcher.py
+        # can use them.
+        for provider in Registrar.settings_providers:
+            self.settings.register_provider(provider)
+        self.load_settings(self.settings_paths)
+
+        context = CommandContext(cwd=self.cwd,
+                                 settings=self.settings, log_manager=self.log_manager,
+                                 commands=Registrar)
+
+        if self.populate_context_handler:
+            self.populate_context_handler(context)
+            context = ContextWrapper(context, self.populate_context_handler)
+
+        parser = self.get_argument_parser(context)
+
+        if not len(argv):
+            # We don't register the usage until here because if it is globally
+            # registered, argparse always prints it. This is not desired when
+            # running with --help.
+            parser.usage = Mach.USAGE
+            parser.print_usage()
+            return 0
+
+        try:
+            args = parser.parse_args(argv)
+        except NoCommandError:
+            print(NO_COMMAND_ERROR)
+            return 1
+        except UnknownCommandError as e:
+            suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (
+                e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else ''
+            print(UNKNOWN_COMMAND_ERROR %
+                  (e.verb, e.command, suggestion_message))
+            return 1
+        except UnrecognizedArgumentError as e:
+            print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command,
+                                                 ' '.join(e.arguments)))
+            return 1
+
+        if not hasattr(args, 'mach_handler'):
+            raise MachError('ArgumentParser result missing mach handler info.')
+
+        handler = getattr(args, 'mach_handler')
+
+        # Add JSON logging to a file if requested.
+        if args.logfile:
+            self.log_manager.add_json_handler(args.logfile)
+
+        # Up the logging level if requested.
+        log_level = logging.INFO
+        if args.verbose:
+            log_level = logging.DEBUG
+
+        self.log_manager.register_structured_logger(logging.getLogger('mach'))
+
+        write_times = True
+        if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ:
+            write_times = False
+
+        # Always enable terminal logging. The log manager figures out if we are
+        # actually in a TTY or are a pipe and does the right thing.
+        self.log_manager.add_terminal_logging(level=log_level,
+                                              write_interval=args.log_interval,
+                                              write_times=write_times)
+
+        if args.settings_file:
+            # Argument parsing has already happened, so settings that apply
+            # to command line handling (e.g alias, defaults) will be ignored.
+            self.load_settings(args.settings_file)
+
+        try:
+            return Registrar._run_command_handler(handler, context=context,
+                                                  debug_command=args.debug_command,
+                                                  **vars(args.command_args))
+        except KeyboardInterrupt as ki:
+            raise ki
+        except FailedCommandError as e:
+            print(e.message)
+            return e.exit_code
+        except Exception:
+            exc_type, exc_value, exc_tb = sys.exc_info()
+
+            # The first two frames are us and are never used.
+            stack = traceback.extract_tb(exc_tb)[2:]
+
+            # If we have nothing on the stack, the exception was raised as part
+            # of calling the @Command method itself. This likely means a
+            # mismatch between @CommandArgument and arguments to the method.
+            # e.g. there exists a @CommandArgument without the corresponding
+            # argument on the method. We handle that here until the module
+            # loader grows the ability to validate better.
+            if not len(stack):
+                print(COMMAND_ERROR)
+                self._print_exception(sys.stdout, exc_type, exc_value,
+                                      traceback.extract_tb(exc_tb))
+                return 1
+
+            # Split the frames into those from the module containing the
+            # command and everything else.
+            command_frames = []
+            other_frames = []
+
+            initial_file = stack[0][0]
+
+            for frame in stack:
+                if frame[0] == initial_file:
+                    command_frames.append(frame)
+                else:
+                    other_frames.append(frame)
+
+            # If the exception was in the module providing the command, it's
+            # likely the bug is in the mach command module, not something else.
+            # If there are other frames, the bug is likely not the mach
+            # command's fault.
+            self._print_error_header(argv, sys.stdout)
+
+            if len(other_frames):
+                print(MODULE_ERROR)
+            else:
+                print(COMMAND_ERROR)
+
+            self._print_exception(sys.stdout, exc_type, exc_value, stack)
+
+            return 1
+
+    def log(self, level, action, params, format_str):
+        """Helper method to record a structured log event."""
+        self.logger.log(level, format_str,
+                        extra={'action': action, 'params': params})
+
+    def _print_error_header(self, argv, fh):
+        fh.write('Error running mach:\n\n')
+        fh.write('    ')
+        fh.write(repr(argv))
+        fh.write('\n\n')
+
+    def _print_exception(self, fh, exc_type, exc_value, stack):
+        fh.write(ERROR_FOOTER)
+        fh.write('\n')
+
+        for l in traceback.format_exception_only(exc_type, exc_value):
+            fh.write(l)
+
+        fh.write('\n')
+        for l in traceback.format_list(stack):
+            fh.write(l)
+
+    def load_settings(self, paths):
+        """Load the specified settings files.
+
+        If a directory is specified, the following basenames will be
+        searched for in this order:
+
+            machrc, .machrc
+        """
+        if isinstance(paths, string_types):
+            paths = [paths]
+
+        valid_names = ('machrc', '.machrc')
+
+        def find_in_dir(base):
+            if os.path.isfile(base):
+                return base
+
+            for name in valid_names:
+                path = os.path.join(base, name)
+                if os.path.isfile(path):
+                    return path
+
+        files = map(find_in_dir, self.settings_paths)
+        files = filter(bool, files)
+
+        self.settings.load_files(files)
+
+    def get_argument_parser(self, context):
+        """Returns an argument parser for the command-line interface."""
+
+        parser = ArgumentParser(add_help=False,
+                                usage='%(prog)s [global arguments] '
+                                'command [command arguments]')
+
+        # Order is important here as it dictates the order the auto-generated
+        # help messages are printed.
+        global_group = parser.add_argument_group('Global Arguments')
+
+        global_group.add_argument('-v', '--verbose', dest='verbose',
+                                  action='store_true', default=False,
+                                  help='Print verbose output.')
+        global_group.add_argument('-l', '--log-file', dest='logfile',
+                                  metavar='FILENAME', type=argparse.FileType('ab'),
+                                  help='Filename to write log data to.')
+        global_group.add_argument('--log-interval', dest='log_interval',
+                                  action='store_true', default=False,
+                                  help='Prefix log line with interval from last message rather '
+                                  'than relative time. Note that this is NOT execution time '
+                                  'if there are parallel operations.')
+        suppress_log_by_default = False
+        if 'INSIDE_EMACS' in os.environ:
+            suppress_log_by_default = True
+        global_group.add_argument('--log-no-times', dest='log_no_times',
+                                  action='store_true', default=suppress_log_by_default,
+                                  help='Do not prefix log lines with times. By default, '
+                                  'mach will prefix each output line with the time since '
+                                  'command start.')
+        global_group.add_argument('-h', '--help', dest='help',
+                                  action='store_true', default=False,
+                                  help='Show this help message.')
+        global_group.add_argument('--debug-command', action='store_true',
+                                  help='Start a Python debugger when command is dispatched.')
+        global_group.add_argument('--settings', dest='settings_file',
+                                  metavar='FILENAME', default=None,
+                                  help='Path to settings file.')
+        global_group.add_argument('--print-command', action='store_true',
+                                  help=argparse.SUPPRESS)
+
+        for args, kwargs in self.global_arguments:
+            global_group.add_argument(*args, **kwargs)
+
+        # We need to be last because CommandAction swallows all remaining
+        # arguments and argparse parses arguments in the order they were added.
+        parser.add_argument('command', action=CommandAction,
+                            registrar=Registrar, context=context)
+
+        return parser

python/mach/mach/mixin/logging.py

@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+
+
+class LoggingMixin(object):
+    """Provides functionality to control logging."""
+
+    def populate_logger(self, name=None):
+        """Ensure this class instance has a logger associated with it.
+
+        Users of this mixin that call log() will need to ensure self._logger is
+        a logging.Logger instance before they call log(). This function ensures
+        self._logger is defined by populating it if it isn't.
+        """
+        if hasattr(self, '_logger'):
+            return
+
+        if name is None:
+            name = '.'.join([self.__module__, self.__class__.__name__])
+
+        self._logger = logging.getLogger(name)
+
+    def log(self, level, action, params, format_str):
+        """Log a structured log event.
+
+        A structured log event consists of a logging level, a string action, a
+        dictionary of attributes, and a formatting string.
+
+        The logging level is one of the logging.* constants, such as
+        logging.INFO.
+
+        The action string is essentially the enumeration of the event. Each
+        different type of logged event should have a different action.
+
+        The params dict is the metadata constituting the logged event.
+
+        The formatting string is used to convert the structured message back to
+        human-readable format. Conversion back to human-readable form is
+        performed by calling format() on this string, feeding into it the dict
+        of attributes constituting the event.
+
+        Example Usage
+        -------------
+
+        self.log(logging.DEBUG, 'login', {'username': 'johndoe'},
+            'User login: {username}')
+        """
+        self._logger.log(level, format_str,
+                         extra={'action': action, 'params': params})

python/mach/mach/mixin/process.py

@@ -0,0 +1,176 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides mixins to perform process execution.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import os
+import signal
+import subprocess
+import sys
+
+from mozbuild.util import ensure_subprocess_env
+from mozprocess.processhandler import ProcessHandlerMixin
+
+from .logging import LoggingMixin
+
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if 'SHELL' in os.environ:
+    _current_shell = os.environ['SHELL']
+elif 'MOZILLABUILD' in os.environ:
+    _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+elif 'COMSPEC' in os.environ:
+    _current_shell = os.environ['COMSPEC']
+else:
+    raise Exception('Could not detect environment shell!')
+
+_in_msys = False
+
+if os.environ.get('MSYSTEM', None) in ('MINGW32', 'MINGW64'):
+    _in_msys = True
+
+    if not _current_shell.lower().endswith('.exe'):
+        _current_shell += '.exe'
+
+
+class ProcessExecutionMixin(LoggingMixin):
+    """Mix-in that provides process execution functionality."""
+
+    def run_process(self, args=None, cwd=None, append_env=None,
+                    explicit_env=None, log_name=None, log_level=logging.INFO,
+                    line_handler=None, require_unix_environment=False,
+                    ensure_exit_code=0, ignore_children=False, pass_thru=False):
+        """Runs a single process to completion.
+
+        Takes a list of arguments to run where the first item is the
+        executable. Runs the command in the specified directory and
+        with optional environment variables.
+
+        append_env -- Dict of environment variables to append to the current
+            set of environment variables.
+        explicit_env -- Dict of environment variables to set for the new
+            process. Any existing environment variables will be ignored.
+
+        require_unix_environment if True will ensure the command is executed
+        within a UNIX environment. Basically, if we are on Windows, it will
+        execute the command via an appropriate UNIX-like shell.
+
+        ignore_children is proxied to mozprocess's ignore_children.
+
+        ensure_exit_code is used to ensure the exit code of a process matches
+        what is expected. If it is an integer, we raise an Exception if the
+        exit code does not match this value. If it is True, we ensure the exit
+        code is 0. If it is False, we don't perform any exit code validation.
+
+        pass_thru is a special execution mode where the child process inherits
+        this process's standard file handles (stdin, stdout, stderr) as well as
+        additional file descriptors. It should be used for interactive processes
+        where buffering from mozprocess could be an issue. pass_thru does not
+        use mozprocess. Therefore, arguments like log_name, line_handler,
+        and ignore_children have no effect.
+        """
+        args = self._normalize_command(args, require_unix_environment)
+
+        self.log(logging.INFO, 'new_process', {'args': ' '.join(args)}, '{args}')
+
+        def handleLine(line):
+            # Converts str to unicode on Python 2 and bytes to str on Python 3.
+            if isinstance(line, bytes):
+                line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
+
+            if line_handler:
+                line_handler(line)
+
+            if not log_name:
+                return
+
+            self.log(log_level, log_name, {'line': line.rstrip()}, '{line}')
+
+        use_env = {}
+        if explicit_env:
+            use_env = explicit_env
+        else:
+            use_env.update(os.environ)
+
+            if append_env:
+                use_env.update(append_env)
+
+        self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
+
+        use_env = ensure_subprocess_env(use_env)
+        if pass_thru:
+            proc = subprocess.Popen(args, cwd=cwd, env=use_env)
+            status = None
+            # Leave it to the subprocess to handle Ctrl+C. If it terminates as
+            # a result of Ctrl+C, proc.wait() will return a status code, and,
+            # we get out of the loop. If it doesn't, like e.g. gdb, we continue
+            # waiting.
+            while status is None:
+                try:
+                    status = proc.wait()
+                except KeyboardInterrupt:
+                    pass
+        else:
+            p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+                                    processOutputLine=[handleLine],
+                                    universal_newlines=True,
+                                    ignore_children=ignore_children)
+            p.run()
+            p.processOutput()
+            status = None
+            sig = None
+            while status is None:
+                try:
+                    if sig is None:
+                        status = p.wait()
+                    else:
+                        status = p.kill(sig=sig)
+                except KeyboardInterrupt:
+                    if sig is None:
+                        sig = signal.SIGINT
+                    elif sig == signal.SIGINT:
+                        # If we've already tried SIGINT, escalate.
+                        sig = signal.SIGKILL
+
+        if ensure_exit_code is False:
+            return status
+
+        if ensure_exit_code is True:
+            ensure_exit_code = 0
+
+        if status != ensure_exit_code:
+            raise Exception('Process executed with non-0 exit code %d: %s' % (status, args))
+
+        return status
+
+    def _normalize_command(self, args, require_unix_environment):
+        """Adjust command arguments to run in the necessary environment.
+
+        This exists mainly to facilitate execution of programs requiring a *NIX
+        shell when running on Windows. The caller specifies whether a shell
+        environment is required. If it is and we are running on Windows but
+        aren't running in the UNIX-like msys environment, then we rewrite the
+        command to execute via a shell.
+        """
+        assert isinstance(args, list) and len(args)
+
+        if not require_unix_environment or not _in_msys:
+            return args
+
+        # Always munge Windows-style into Unix style for the command.
+        prog = args[0].replace('\\', '/')
+
+        # PyMake removes the C: prefix. But, things seem to work here
+        # without it. Not sure what that's about.
+
+        # We run everything through the msys shell. We need to use
+        # '-c' and pass all the arguments as one argument because that is
+        # how sh works.
+        cline = subprocess.list2cmdline([prog] + args[1:])
+        return [_current_shell, '-c', cline]

python/mach/mach/registrar.py

@@ -0,0 +1,155 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import time
+
+import six
+
+from .base import MachError
+
+INVALID_COMMAND_CONTEXT = r'''
+It looks like you tried to run a mach command from an invalid context. The %s
+command failed to meet the following conditions: %s
+
+Run |mach help| to show a list of all commands available to the current context.
+'''.lstrip()
+
+
+class MachRegistrar(object):
+    """Container for mach command and config providers."""
+
+    def __init__(self):
+        self.command_handlers = {}
+        self.commands_by_category = {}
+        self.settings_providers = set()
+        self.categories = {}
+        self.require_conditions = False
+        self.command_depth = 0
+
+    def register_command_handler(self, handler):
+        name = handler.name
+
+        if not handler.category:
+            raise MachError('Cannot register a mach command without a '
+                            'category: %s' % name)
+
+        if handler.category not in self.categories:
+            raise MachError('Cannot register a command to an undefined '
+                            'category: %s -> %s' % (name, handler.category))
+
+        self.command_handlers[name] = handler
+        self.commands_by_category[handler.category].add(name)
+
+    def register_settings_provider(self, cls):
+        self.settings_providers.add(cls)
+
+    def register_category(self, name, title, description, priority=50):
+        self.categories[name] = (title, description, priority)
+        self.commands_by_category[name] = set()
+
+    @classmethod
+    def _condition_failed_message(cls, name, conditions):
+        msg = ['\n']
+        for c in conditions:
+            part = ['  %s' % c.__name__]
+            if c.__doc__ is not None:
+                part.append(c.__doc__)
+            msg.append(' - '.join(part))
+        return INVALID_COMMAND_CONTEXT % (name, '\n'.join(msg))
+
+    @classmethod
+    def _instance(_, handler, context, **kwargs):
+        cls = handler.cls
+
+        if handler.pass_context and not context:
+            raise Exception('mach command class requires context.')
+
+        if context:
+            prerun = getattr(context, 'pre_dispatch_handler', None)
+            if prerun:
+                prerun(context, handler, args=kwargs)
+
+        if handler.pass_context:
+            context.handler = handler
+            instance = cls(context)
+        else:
+            instance = cls()
+
+        return instance
+
+    @classmethod
+    def _fail_conditions(_, handler, instance):
+        fail_conditions = []
+        if handler.conditions:
+            for c in handler.conditions:
+                if not c(instance):
+                    fail_conditions.append(c)
+
+        return fail_conditions
+
+    def _run_command_handler(self, handler, context=None, debug_command=False, **kwargs):
+        instance = MachRegistrar._instance(handler, context, **kwargs)
+        fail_conditions = MachRegistrar._fail_conditions(handler, instance)
+        if fail_conditions:
+            print(MachRegistrar._condition_failed_message(handler.name, fail_conditions))
+            return 1
+
+        self.command_depth += 1
+        fn = getattr(instance, handler.method)
+
+        start_time = time.time()
+
+        if debug_command:
+            import pdb
+            result = pdb.runcall(fn, **kwargs)
+        else:
+            result = fn(**kwargs)
+
+        end_time = time.time()
+
+        result = result or 0
+        assert isinstance(result, six.integer_types)
+
+        if context and not debug_command:
+            postrun = getattr(context, 'post_dispatch_handler', None)
+            if postrun:
+                postrun(context, handler, instance, result,
+                        start_time, end_time, self.command_depth, args=kwargs)
+        self.command_depth -= 1
+
+        return result
+
+    def dispatch(self, name, context=None, argv=None, subcommand=None, **kwargs):
+        """Dispatch/run a command.
+
+        Commands can use this to call other commands.
+        """
+        handler = self.command_handlers[name]
+
+        if subcommand:
+            handler = handler.subcommand_handlers[subcommand]
+
+        if handler.parser:
+            parser = handler.parser
+
+            # save and restore existing defaults so **kwargs don't persist across
+            # subsequent invocations of Registrar.dispatch()
+            old_defaults = parser._defaults.copy()
+            parser.set_defaults(**kwargs)
+            kwargs, unknown = parser.parse_known_args(argv or [])
+            kwargs = vars(kwargs)
+            parser._defaults = old_defaults
+
+            if unknown:
+                if subcommand:
+                    name = '{} {}'.format(name, subcommand)
+                parser.error("unrecognized arguments for {}: {}".format(
+                    name, ', '.join(["'{}'".format(arg) for arg in unknown])))
+
+        return self._run_command_handler(handler, context=context, **kwargs)
+
+
+Registrar = MachRegistrar()

python/mach/mach/terminal.py

@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This file contains code for interacting with terminals.
+
+All the terminal interaction code is consolidated so the complexity can be in
+one place, away from code that is commonly looked at.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import sys
+
+from six.moves import range
+
+
+class LoggingHandler(logging.Handler):
+    """Custom logging handler that works with terminal window dressing.
+
+    This is alternative terminal logging handler which contains smarts for
+    emitting terminal control characters properly. Currently, it has generic
+    support for "footer" elements at the bottom of the screen. Functionality
+    can be added when needed.
+    """
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+        self.fh = sys.stdout
+        self.footer = None
+
+    def flush(self):
+        self.acquire()
+
+        try:
+            self.fh.flush()
+        finally:
+            self.release()
+
+    def emit(self, record):
+        msg = self.format(record)
+
+        if self.footer:
+            self.footer.clear()
+
+        self.fh.write(msg)
+        self.fh.write('\n')
+
+        if self.footer:
+            self.footer.draw()
+
+        # If we don't flush, the footer may not get drawn.
+        self.flush()
+
+
+class TerminalFooter(object):
+    """Represents something drawn on the bottom of a terminal."""
+    def __init__(self, terminal):
+        self.t = terminal
+        self.fh = sys.stdout
+
+    def _clear_lines(self, n):
+        for i in range(n):
+            self.fh.write(self.t.move_x(0))
+            self.fh.write(self.t.clear_eol())
+            self.fh.write(self.t.move_up())
+
+        self.fh.write(self.t.move_down())
+        self.fh.write(self.t.move_x(0))
+
+    def clear(self):
+        raise Exception('clear() must be implemented.')
+
+    def draw(self):
+        raise Exception('draw() must be implemented.')

python/mach/mach/util.py

@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+import sys
+
+from six import text_type
+
+
+def setenv(key, value):
+    """Compatibility shim to ensure the proper string type is used with
+    os.environ for the version of Python being used.
+    """
+    encoding = "mbcs" if sys.platform == "win32" else "utf-8"
+
+    if sys.version_info[0] == 2:
+        if isinstance(key, text_type):
+            key = key.encode(encoding)
+        if isinstance(value, text_type):
+            value = value.encode(encoding)
+    else:
+        if isinstance(key, bytes):
+            key = key.decode(encoding)
+        if isinstance(value, bytes):
+            value = value.decode(encoding)
+
+    os.environ[key] = value

python/mach/setup.cfg

@@ -0,0 +1,7 @@
+[bdist_wheel]
+universal = 1
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

python/mach/setup.py

@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+
+VERSION = '1.0.0'
+
+README = open('README.rst').read()
+
+setup(
+    name='mach',
+    description='Generic command line command dispatching framework.',
+    long_description=README,
+    license='MPL 2.0',
+    author='Gregory Szorc',
+    author_email='gregory.szorc@gmail.com',
+    url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach',
+    packages=['mach', 'mach.mixin'],
+    version=VERSION,
+    classifiers=[
+        'Environment :: Console',
+        'Development Status :: 5 - Production/Stable',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+        'Natural Language :: English',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.5',
+    ],
+    install_requires=[
+        'blessings',
+        'mozfile',
+        'mozprocess',
+        'six',
+    ],
+    tests_require=['mock'],
+)

python/mach_bootstrap.py

@@ -14,6 +14,7 @@ from tempfile import TemporaryFile
 
 SEARCH_PATHS = [
     os.path.join("python", "tidy"),
+    os.path.join("python", "mach"),
 ]
 
 # Individual files providing mach commands.

python/requirements.txt

@@ -3,7 +3,6 @@
 
 blessings == 1.7
 distro == 1.4
-mach == 1.0.0
 mozdebug == 0.3
 mozinfo == 1.2.1
 mozlog == 7.1.0