Mirrors/servo
1
0
Fork 0
mirror of https://github.com/servo/servo.git synced 2025-06-08 00:23:30 +00:00
Code Issues Projects Releases Packages Wiki Activity

Cargoify servo

Browse source
This commit is contained in:
Jack Moffitt 2014-08-28 09:34:23 -06:00
parent db2f642c32
commit c6ab60dbfc
1761 changed files with 8423 additions and 2294 deletions
Download patch file Download diff file Expand all files Collapse all files

75
python/licenseck.py Normal file
View file

@ -0,0 +1,75 @@
# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
license0="""\
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"""
license1="""\
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
license2="""\
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
license3 = """\
// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
"""
license4 = """\
# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
"""
licenses = [license0, license1, license2, license3, license4]
exceptions = [
"servo/dom/bindings/codegen/ply/ply/yacc.py", # BSD
"servo/dom/bindings/codegen/ply/ply/__init__.py", # BSD
"servo/dom/bindings/codegen/ply/ply/lex.py", # BSD
]
def check_license(name, contents):
valid_license = False
for a_valid_license in licenses:
if contents.startswith(a_valid_license):
valid_license = True
break
if valid_license:
return True
for exception in exceptions:
if name.endswith(exception):
return True
firstlineish = contents[:100]
if firstlineish.find("xfail-license") != -1:
return True
return False

328
python/mach/README.rst Normal file
View file

@ -0,0 +1,328 @@
====
mach
====
Mach (German for *do*) is a generic command dispatcher for the command
line.
To use mach, you install the mach core (a Python package), create an
executable *driver* script (named whatever you want), and write mach
commands. When the *driver* is executed, mach dispatches to the
requested command handler automatically.
Features
========
On a high level, mach is similar to using argparse with subparsers (for
command handling). When you dig deeper, mach offers a number of
additional features:
Distributed command definitions
With optparse/argparse, you have to define your commands on a central
parser instance. With mach, you annotate your command methods with
decorators and mach finds and dispatches to them automatically.
Command categories
Mach commands can be grouped into categories when displayed in help.
This is currently not possible with argparse.
Logging management
Mach provides a facility for logging (both classical text and
structured) that is available to any command handler.
Settings files
Mach provides a facility for reading settings from an ini-like file
format.
Components
==========
Mach is conceptually composed of the following components:
core
The mach core is the core code powering mach. This is a Python package
that contains all the business logic that makes mach work. The mach
core is common to all mach deployments.
commands
These are what mach dispatches to. Commands are simply Python methods
registered as command names. The set of commands is unique to the
environment mach is deployed in.
driver
The *driver* is the entry-point to mach. It is simply an executable
script that loads the mach core, tells it where commands can be found,
then asks the mach core to handle the current request. The driver is
unique to the deployed environment. But, it's usually based on an
example from this source tree.
Project State
=============
mach was originally written as a command dispatching framework to aid
Firefox development. While the code is mostly generic, there are still
some pieces that closely tie it to Mozilla/Firefox. The goal is for
these to eventually be removed and replaced with generic features so
mach is suitable for anybody to use. Until then, mach may not be the
best fit for you.
Implementing Commands
---------------------
Mach commands are defined via Python decorators.
All the relevant decorators are defined in the *mach.decorators* module.
The important decorators are as follows:
CommandProvider
A class decorator that denotes that a class contains mach
commands. The decorator takes no arguments.
Command
A method decorator that denotes that the method should be called when
the specified command is requested. The decorator takes a command name
as its first argument and a number of additional arguments to
configure the behavior of the command.
CommandArgument
A method decorator that defines an argument to the command. Its
arguments are essentially proxied to ArgumentParser.add_argument()
Classes with the *@CommandProvider* decorator *must* have an *__init__*
method that accepts 1 or 2 arguments. If it accepts 2 arguments, the
2nd argument will be a *MachCommandContext* instance. This is just a named
tuple containing references to objects provided by the mach driver.
Here is a complete example::
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
@CommandProvider
class MyClass(object):
@Command('doit', help='Do ALL OF THE THINGS.')
@CommandArgument('--force', '-f', action='store_true',
help='Force doing it.')
def doit(self, force=False):
# Do stuff here.
When the module is loaded, the decorators tell mach about all handlers.
When mach runs, it takes the assembled metadata from these handlers and
hooks it up to the command line driver. Under the hood, arguments passed
to the decorators are being used to help mach parse command arguments,
formulate arguments to the methods, etc. See the documentation in the
*mach.base* module for more.
The Python modules defining mach commands do not need to live inside the
main mach source tree.
Conditionally Filtering Commands
--------------------------------
Sometimes it might only make sense to run a command given a certain
context. For example, running tests only makes sense if the product
they are testing has been built, and said build is available. To make
sure a command is only runnable from within a correct context, you can
define a series of conditions on the *Command* decorator.
A condition is simply a function that takes an instance of the
*CommandProvider* class as an argument, and returns True or False. If
any of the conditions defined on a command return False, the command
will not be runnable. The doc string of a condition function is used in
error messages, to explain why the command cannot currently be run.
Here is an example:
from mach.decorators import (
CommandProvider,
Command,
)
def build_available(cls):
"""The build needs to be available."""
return cls.build_path is not None
@CommandProvider
class MyClass(MachCommandBase):
def __init__(self, build_path=None):
self.build_path = build_path
@Command('run_tests', conditions=[build_available])
def run_tests(self):
# Do stuff here.
It is important to make sure that any state needed by the condition is
available to instances of the command provider.
By default all commands without any conditions applied will be runnable,
but it is possible to change this behaviour by setting *require_conditions*
to True:
m = mach.main.Mach()
m.require_conditions = True
Minimizing Code in Commands
---------------------------
Mach command modules, classes, and methods work best when they are
minimal dispatchers. The reason is import bloat. Currently, the mach
core needs to import every Python file potentially containing mach
commands for every command invocation. If you have dozens of commands or
commands in modules that import a lot of Python code, these imports
could slow mach down and waste memory.
It is thus recommended that mach modules, classes, and methods do as
little work as possible. Ideally the module should only import from
the *mach* package. If you need external modules, you should import them
from within the command method.
To keep code size small, the body of a command method should be limited
to:
1. Obtaining user input (parsing arguments, prompting, etc)
2. Calling into some other Python package
3. Formatting output
Of course, these recommendations can be ignored if you want to risk
slower performance.
In the future, the mach driver may cache the dispatching information or
have it intelligently loaded to facilitate lazy loading.
Logging
=======
Mach configures a built-in logging facility so commands can easily log
data.
What sets the logging facility apart from most loggers you've seen is
that it encourages structured logging. Instead of conventional logging
where simple strings are logged, the internal logging mechanism logs all
events with the following pieces of information:
* A string *action*
* A dict of log message fields
* A formatting string
Essentially, instead of assembling a human-readable string at
logging-time, you create an object holding all the pieces of data that
will constitute your logged event. For each unique type of logged event,
you assign an *action* name.
Depending on how logging is configured, your logged event could get
written a couple of different ways.
JSON Logging
------------
Where machines are the intended target of the logging data, a JSON
logger is configured. The JSON logger assembles an array consisting of
the following elements:
* Decimal wall clock time in seconds since UNIX epoch
* String *action* of message
* Object with structured message data
The JSON-serialized array is written to a configured file handle.
Consumers of this logging stream can just perform a readline() then feed
that into a JSON deserializer to reconstruct the original logged
message. They can key off the *action* element to determine how to
process individual events. There is no need to invent a parser.
Convenient, isn't it?
Logging for Humans
------------------
Where humans are the intended consumer of a log message, the structured
log message are converted to more human-friendly form. This is done by
utilizing the *formatting* string provided at log time. The logger
simply calls the *format* method of the formatting string, passing the
dict containing the message's fields.
When *mach* is used in a terminal that supports it, the logging facility
also supports terminal features such as colorization. This is done
automatically in the logging layer - there is no need to control this at
logging time.
In addition, messages intended for humans typically prepends every line
with the time passed since the application started.
Logging HOWTO
-------------
Structured logging piggybacks on top of Python's built-in logging
infrastructure provided by the *logging* package. We accomplish this by
taking advantage of *logging.Logger.log()*'s *extra* argument. To this
argument, we pass a dict with the fields *action* and *params*. These
are the string *action* and dict of message fields, respectively. The
formatting string is passed as the *msg* argument, like normal.
If you were logging to a logger directly, you would do something like:
logger.log(logging.INFO, 'My name is {name}',
extra={'action': 'my_name', 'params': {'name': 'Gregory'}})
The JSON logging would produce something like:
[1339985554.306338, "my_name", {"name": "Gregory"}]
Human logging would produce something like:
0.52 My name is Gregory
Since there is a lot of complexity using logger.log directly, it is
recommended to go through a wrapping layer that hides part of the
complexity for you. The easiest way to do this is by utilizing the
LoggingMixin:
import logging
from mach.mixin.logging import LoggingMixin
class MyClass(LoggingMixin):
def foo(self):
self.log(logging.INFO, 'foo_start', {'bar': True},
'Foo performed. Bar: {bar}')
Entry Points
============
It is possible to use setuptools' entry points to load commands
directly from python packages. A mach entry point is a function which
returns a list of files or directories containing mach command
providers. e.g.::
def list_providers():
providers = []
here = os.path.abspath(os.path.dirname(__file__))
for p in os.listdir(here):
if p.endswith('.py'):
providers.append(os.path.join(here, p))
return providers
See http://pythonhosted.org/setuptools/setuptools.html#dynamic-discovery-of-services-and-plugins
for more information on creating an entry point. To search for entry
point plugins, you can call *load_commands_from_entry_point*. This
takes a single parameter called *group*. This is the name of the entry
point group to load and defaults to ``mach.providers``. e.g.::
mach.load_commands_from_entry_point("mach.external.providers")
Adding Global Arguments
=======================
Arguments to mach commands are usually command-specific. However,
mach ships with a handful of global arguments that apply to all
commands.
It is possible to extend the list of global arguments. In your
*mach driver*, simply call ``add_global_argument()`` on your
``mach.main.Mach`` instance. e.g.::
mach = mach.main.Mach(os.getcwd())
# Will allow --example to be specified on every mach command.
mach.add_global_argument('--example', action='store_true',
help='Demonstrate an example global argument.')

29
python/mach/bash-completion.sh Normal file
View file

@ -0,0 +1,29 @@
function _mach()
{
local cur cmds c subcommand
COMPREPLY=()
# Load the list of commands
cmds=`"${COMP_WORDS[0]}" mach-commands`
# Look for the subcommand.
cur="${COMP_WORDS[COMP_CWORD]}"
subcommand=""
c=1
while [ $c -lt $COMP_CWORD ]; do
word="${COMP_WORDS[c]}"
for cmd in $cmds; do
if [ "$cmd" = "$word" ]; then
subcommand="$word"
fi
done
c=$((++c))
done
if [[ "$subcommand" == "help" || -z "$subcommand" ]]; then
COMPREPLY=( $(compgen -W "$cmds" -- ${cur}) )
fi
return 0
}
complete -o default -F _mach mach

0
python/mach/mach/__init__.py Normal file
View file

110
python/mach/mach/base.py Normal file
View file

@ -0,0 +1,110 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
class CommandContext(object):
"""Holds run-time state so it can easily be passed to command providers."""
def __init__(self, cwd=None, settings=None, log_manager=None,
commands=None, **kwargs):
self.cwd = cwd
self.settings = settings
self.log_manager = log_manager
self.commands = commands
for k,v in kwargs.items():
setattr(self, k, v)
class MachError(Exception):
"""Base class for all errors raised by mach itself."""
class NoCommandError(MachError):
"""No command was passed into mach."""
class UnknownCommandError(MachError):
"""Raised when we attempted to execute an unknown command."""
def __init__(self, command, verb, suggested_commands=None):
MachError.__init__(self)
self.command = command
self.verb = verb
self.suggested_commands = suggested_commands or []
class UnrecognizedArgumentError(MachError):
"""Raised when an unknown argument is passed to mach."""
def __init__(self, command, arguments):
MachError.__init__(self)
self.command = command
self.arguments = arguments
class MethodHandler(object):
"""Describes a Python method that implements a mach command.
Instances of these are produced by mach when it processes classes
defining mach commands.
"""
__slots__ = (
# The Python class providing the command. This is the class type not
# an instance of the class. Mach will instantiate a new instance of
# the class if the command is executed.
'cls',
# Whether the __init__ method of the class should receive a mach
# context instance. This should only affect the mach driver and how
# it instantiates classes.
'pass_context',
# The name of the method providing the command. In other words, this
# is the str name of the attribute on the class type corresponding to
# the name of the function.
'method',
# The name of the command.
'name',
# String category this command belongs to.
'category',
# Description of the purpose of this command.
'description',
# Whether to allow all arguments from the parser.
'allow_all_arguments',
# Functions used to 'skip' commands if they don't meet the conditions
# in a given context.
'conditions',
# argparse.ArgumentParser instance to use as the basis for command
# arguments.
'parser',
# Arguments added to this command's parser. This is a 2-tuple of
# positional and named arguments, respectively.
'arguments',
)
def __init__(self, cls, method, name, category=None, description=None,
allow_all_arguments=False, conditions=None, parser=None, arguments=None,
pass_context=False):
self.cls = cls
self.method = method
self.name = name
self.category = category
self.description = description
self.allow_all_arguments = allow_all_arguments
self.conditions = conditions or []
self.parser = parser
self.arguments = arguments or []
self.pass_context = pass_context

0
python/mach/mach/commands/__init__.py Normal file
View file

41
python/mach/mach/commands/commandinfo.py Normal file
View file

@ -0,0 +1,41 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
from mach.decorators import (
CommandProvider,
Command,
)
@CommandProvider
class BuiltinCommands(object):
def __init__(self, context):
self.context = context
@Command('mach-commands', category='misc',
description='List all mach commands.')
def commands(self):
print("\n".join(self.context.commands.command_handlers.keys()))
@Command('mach-debug-commands', category='misc',
description='Show info about available mach commands.')
def debug_commands(self):
import inspect
handlers = self.context.commands.command_handlers
for command in sorted(handlers.keys()):
handler = handlers[command]
cls = handler.cls
method = getattr(cls, getattr(handler, 'method'))
print(command)
print('=' * len(command))
print('')
print('File: %s' % inspect.getsourcefile(method))
print('Class: %s' % cls.__name__)
print('Method: %s' % handler.method)
print('')

50
python/mach/mach/commands/settings.py Normal file
View file

@ -0,0 +1,50 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
from textwrap import TextWrapper
from mach.decorators import (
CommandProvider,
Command,
)
#@CommandProvider
class Settings(object):
"""Interact with settings for mach.
Currently, we only provide functionality to view what settings are
available. In the future, this module will be used to modify settings, help
people create configs via a wizard, etc.
"""
def __init__(self, context):
self.settings = context.settings
@Command('settings-list', category='devenv',
description='Show available config settings.')
def list_settings(self):
"""List available settings in a concise list."""
for section in sorted(self.settings):
for option in sorted(self.settings[section]):
short, full = self.settings.option_help(section, option)
print('%s.%s -- %s' % (section, option, short))
@Command('settings-create', category='devenv',
description='Print a new settings file with usage info.')
def create(self):
"""Create an empty settings file with full documentation."""
wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ')
for section in sorted(self.settings):
print('[%s]' % section)
print('')
for option in sorted(self.settings[section]):
short, full = self.settings.option_help(section, option)
print(wrapper.fill(full))
print(';%s =' % option)
print('')

488
python/mach/mach/config.py Normal file
View file

@ -0,0 +1,488 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
r"""
This file defines classes for representing config data/settings.
Config data is modeled as key-value pairs. Keys are grouped together into named
sections. Individual config settings (options) have metadata associated with
them. This metadata includes type, default value, valid values, etc.
The main interface to config data is the ConfigSettings class. 1 or more
ConfigProvider classes are associated with ConfigSettings and define what
settings are available.
Descriptions of individual config options can be translated to multiple
languages using gettext. Each option has associated with it a domain and locale
directory. By default, the domain is the section the option is in and the
locale directory is the "locale" directory beneath the directory containing the
module that defines it.
People implementing ConfigProvider instances are expected to define a complete
gettext .po and .mo file for the en-US locale. You can use the gettext-provided
msgfmt binary to perform this conversion. Generation of the original .po file
can be done via the write_pot() of ConfigSettings.
"""
from __future__ import unicode_literals
import collections
import gettext
import os
import sys
if sys.version_info[0] == 3:
from configparser import RawConfigParser
str_type = str
else:
from ConfigParser import RawConfigParser
str_type = basestring
class ConfigType(object):
"""Abstract base class for config values."""
@staticmethod
def validate(value):
"""Validates a Python value conforms to this type.
Raises a TypeError or ValueError if it doesn't conform. Does not do
anything if the value is valid.
"""
@staticmethod
def from_config(config, section, option):
"""Obtain the value of this type from a RawConfigParser.
Receives a RawConfigParser instance, a str section name, and the str
option in that section to retrieve.
The implementation may assume the option exists in the RawConfigParser
instance.
Implementations are not expected to validate the value. But, they
should return the appropriate Python type.
"""
@staticmethod
def to_config(value):
return value
class StringType(ConfigType):
@staticmethod
def validate(value):
if not isinstance(value, str_type):
raise TypeError()
@staticmethod
def from_config(config, section, option):
return config.get(section, option)
class BooleanType(ConfigType):
@staticmethod
def validate(value):
if not isinstance(value, bool):
raise TypeError()
@staticmethod
def from_config(config, section, option):
return config.getboolean(section, option)
@staticmethod
def to_config(value):
return 'true' if value else 'false'
class IntegerType(ConfigType):
@staticmethod
def validate(value):
if not isinstance(value, int):
raise TypeError()
@staticmethod
def from_config(config, section, option):
return config.getint(section, option)
class PositiveIntegerType(IntegerType):
@staticmethod
def validate(value):
if not isinstance(value, int):
raise TypeError()
if value < 0:
raise ValueError()
class PathType(StringType):
@staticmethod
def validate(value):
if not isinstance(value, str_type):
raise TypeError()
@staticmethod
def from_config(config, section, option):
return config.get(section, option)
class AbsolutePathType(PathType):
@staticmethod
def validate(value):
if not isinstance(value, str_type):
raise TypeError()
if not os.path.isabs(value):
raise ValueError()
class RelativePathType(PathType):
@staticmethod
def validate(value):
if not isinstance(value, str_type):
raise TypeError()
if os.path.isabs(value):
raise ValueError()
class DefaultValue(object):
pass
class ConfigProvider(object):
"""Abstract base class for an object providing config settings.
Classes implementing this interface expose configurable settings. Settings
are typically only relevant to that component itself. But, nothing says
settings can't be shared by multiple components.
"""
@classmethod
def register_settings(cls):
"""Registers config settings.
This is called automatically. Child classes should likely not touch it.
See _register_settings() instead.
"""
if hasattr(cls, '_settings_registered'):
return
cls._settings_registered = True
cls.config_settings = {}
ourdir = os.path.dirname(__file__)
cls.config_settings_locale_directory = os.path.join(ourdir, 'locale')
cls._register_settings()
@classmethod
def _register_settings(cls):
"""The actual implementation of register_settings().
This is what child classes should implement. They should not touch
register_settings().
Implementations typically make 1 or more calls to _register_setting().
"""
raise NotImplemented('%s must implement _register_settings.' %
__name__)
@classmethod
def register_setting(cls, section, option, type_cls, default=DefaultValue,
choices=None, domain=None):
"""Register a config setting with this type.
This is a convenience method to populate available settings. It is
typically called in the class's _register_settings() implementation.
Each setting must have:
section -- str section to which the setting belongs. This is how
settings are grouped.
option -- str id for the setting. This must be unique within the
section it appears.
type -- a ConfigType-derived type defining the type of the setting.
Each setting has the following optional parameters:
default -- The default value for the setting. If None (the default)
there is no default.
choices -- A set of values this setting can hold. Values not in
this set are invalid.
domain -- Translation domain for this setting. By default, the
domain is the same as the section name.
"""
if not section in cls.config_settings:
cls.config_settings[section] = {}
if option in cls.config_settings[section]:
raise Exception('Setting has already been registered: %s.%s' % (
section, option))
domain = domain if domain is not None else section
meta = {
'short': '%s.short' % option,
'full': '%s.full' % option,
'type_cls': type_cls,
'domain': domain,
'localedir': cls.config_settings_locale_directory,
}
if default != DefaultValue:
meta['default'] = default
if choices is not None:
meta['choices'] = choices
cls.config_settings[section][option] = meta
class ConfigSettings(collections.Mapping):
"""Interface for configuration settings.
This is the main interface to the configuration.
A configuration is a collection of sections. Each section contains
key-value pairs.
When an instance is created, the caller first registers ConfigProvider
instances with it. This tells the ConfigSettings what individual settings
are available and defines extra metadata associated with those settings.
This is used for validation, etc.
Once ConfigProvider instances are registered, a config is populated. It can
be loaded from files or populated by hand.
ConfigSettings instances are accessed like dictionaries or by using
attributes. e.g. the section "foo" is accessed through either
settings.foo or settings['foo'].
Sections are modeled by the ConfigSection class which is defined inside
this one. They look just like dicts or classes with attributes. To access
the "bar" option in the "foo" section:
value = settings.foo.bar
value = settings['foo']['bar']
value = settings.foo['bar']
Assignment is similar:
settings.foo.bar = value
settings['foo']['bar'] = value
settings['foo'].bar = value
You can even delete user-assigned values:
del settings.foo.bar
del settings['foo']['bar']
If there is a default, it will be returned.
When settings are mutated, they are validated against the registered
providers. Setting unknown settings or setting values to illegal values
will result in exceptions being raised.
"""
class ConfigSection(collections.MutableMapping, object):
"""Represents an individual config section."""
def __init__(self, config, name, settings):
object.__setattr__(self, '_config', config)
object.__setattr__(self, '_name', name)
object.__setattr__(self, '_settings', settings)
# MutableMapping interface
def __len__(self):
return len(self._settings)
def __iter__(self):
return iter(self._settings.keys())
def __contains__(self, k):
return k in self._settings
def __getitem__(self, k):
if k not in self._settings:
raise KeyError('Option not registered with provider: %s' % k)
meta = self._settings[k]
if self._config.has_option(self._name, k):
return meta['type_cls'].from_config(self._config, self._name, k)
if not 'default' in meta:
raise KeyError('No default value registered: %s' % k)
return meta['default']
def __setitem__(self, k, v):
if k not in self._settings:
raise KeyError('Option not registered with provider: %s' % k)
meta = self._settings[k]
meta['type_cls'].validate(v)
if not self._config.has_section(self._name):
self._config.add_section(self._name)
self._config.set(self._name, k, meta['type_cls'].to_config(v))
def __delitem__(self, k):
self._config.remove_option(self._name, k)
# Prune empty sections.
if not len(self._config.options(self._name)):
self._config.remove_section(self._name)
def __getattr__(self, k):
return self.__getitem__(k)
def __setattr__(self, k, v):
self.__setitem__(k, v)
def __delattr__(self, k):
self.__delitem__(k)
def __init__(self):
self._config = RawConfigParser()
self._settings = {}
self._sections = {}
self._finalized = False
self._loaded_filenames = set()
def load_file(self, filename):
self.load_files([filename])
def load_files(self, filenames):
"""Load a config from files specified by their paths.
Files are loaded in the order given. Subsequent files will overwrite
values from previous files. If a file does not exist, it will be
ignored.
"""
filtered = [f for f in filenames if os.path.exists(f)]
fps = [open(f, 'rt') for f in filtered]
self.load_fps(fps)
self._loaded_filenames.update(set(filtered))
for fp in fps:
fp.close()
def load_fps(self, fps):
"""Load config data by reading file objects."""
for fp in fps:
self._config.readfp(fp)
def loaded_files(self):
return self._loaded_filenames
def write(self, fh):
"""Write the config to a file object."""
self._config.write(fh)
def validate(self):
"""Ensure that the current config passes validation.
This is a generator of tuples describing any validation errors. The
elements of the tuple are:
(bool) True if error is fatal. False if just a warning.
(str) Type of validation issue. Can be one of ('unknown-section',
'missing-required', 'type-error')
"""
def register_provider(self, provider):
"""Register a ConfigProvider with this settings interface."""
if self._finalized:
raise Exception('Providers cannot be registered after finalized.')
provider.register_settings()
for section_name, settings in provider.config_settings.items():
section = self._settings.get(section_name, {})
for k, v in settings.items():
if k in section:
raise Exception('Setting already registered: %s.%s' %
section_name, k)
section[k] = v
self._settings[section_name] = section
def write_pot(self, fh):
"""Write a pot gettext translation file."""
for section in sorted(self):
fh.write('# Section %s\n\n' % section)
for option in sorted(self[section]):
fh.write('msgid "%s.%s.short"\n' % (section, option))
fh.write('msgstr ""\n\n')
fh.write('msgid "%s.%s.full"\n' % (section, option))
fh.write('msgstr ""\n\n')
fh.write('# End of section %s\n\n' % section)
def option_help(self, section, option):
"""Obtain the translated help messages for an option."""
meta = self[section]._settings[option]
# Providers should always have an en-US translation. If they don't,
# they are coded wrong and this will raise.
default = gettext.translation(meta['domain'], meta['localedir'],
['en-US'])
t = gettext.translation(meta['domain'], meta['localedir'],
fallback=True)
t.add_fallback(default)
short = t.ugettext('%s.%s.short' % (section, option))
full = t.ugettext('%s.%s.full' % (section, option))
return (short, full)
def _finalize(self):
if self._finalized:
return
for section, settings in self._settings.items():
s = ConfigSettings.ConfigSection(self._config, section, settings)
self._sections[section] = s
self._finalized = True
# Mapping interface.
def __len__(self):
return len(self._settings)
def __iter__(self):
self._finalize()
return iter(self._sections.keys())
def __contains__(self, k):
return k in self._settings
def __getitem__(self, k):
self._finalize()
return self._sections[k]
# Allow attribute access because it looks nice.
def __getattr__(self, k):
return self.__getitem__(k)

176
python/mach/mach/decorators.py Normal file
View file

@ -0,0 +1,176 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import collections
import inspect
import types
from .base import (
MachError,
MethodHandler
)
from .config import ConfigProvider
from .registrar import Registrar
def CommandProvider(cls):
"""Class decorator to denote that it provides subcommands for Mach.
When this decorator is present, mach looks for commands being defined by
methods inside the class.
"""
# The implementation of this decorator relies on the parse-time behavior of
# decorators. When the module is imported, the method decorators (like
# @Command and @CommandArgument) are called *before* this class decorator.
# The side-effect of the method decorators is to store specifically-named
# attributes on the function types. We just scan over all functions in the
# class looking for the side-effects of the method decorators.
# Tell mach driver whether to pass context argument to __init__.
pass_context = False
if inspect.ismethod(cls.__init__):
spec = inspect.getargspec(cls.__init__)
if len(spec.args) > 2:
msg = 'Mach @CommandProvider class %s implemented incorrectly. ' + \
'__init__() must take 1 or 2 arguments. From %s'
msg = msg % (cls.__name__, inspect.getsourcefile(cls))
raise MachError(msg)
if len(spec.args) == 2:
pass_context = True
# We scan __dict__ because we only care about the classes own attributes,
# not inherited ones. If we did inherited attributes, we could potentially
# define commands multiple times. We also sort keys so commands defined in
# the same class are grouped in a sane order.
for attr in sorted(cls.__dict__.keys()):
value = cls.__dict__[attr]
if not isinstance(value, types.FunctionType):
continue
command_name, category, description, allow_all, conditions, parser = getattr(
value, '_mach_command', (None, None, None, None, None, None))
if command_name is None:
continue
if conditions is None and Registrar.require_conditions:
continue
msg = 'Mach command \'%s\' implemented incorrectly. ' + \
'Conditions argument must take a list ' + \
'of functions. Found %s instead.'
conditions = conditions or []
if not isinstance(conditions, collections.Iterable):
msg = msg % (command_name, type(conditions))
raise MachError(msg)
for c in conditions:
if not hasattr(c, '__call__'):
msg = msg % (command_name, type(c))
raise MachError(msg)
arguments = getattr(value, '_mach_command_args', None)
handler = MethodHandler(cls, attr, command_name, category=category,
description=description, allow_all_arguments=allow_all,
conditions=conditions, parser=parser, arguments=arguments,
pass_context=pass_context)
Registrar.register_command_handler(handler)
return cls
class Command(object):
"""Decorator for functions or methods that provide a mach subcommand.
The decorator accepts arguments that define basic attributes of the
command. The following arguments are recognized:
category -- The string category to which this command belongs. Mach's
help will group commands by category.
description -- A brief description of what the command does.
allow_all_args -- Bool indicating whether to allow unknown arguments
through to the command.
parser -- an optional argparse.ArgumentParser instance to use as
the basis for the command arguments.
For example:
@Command('foo', category='misc', description='Run the foo action')
def foo(self):
pass
"""
def __init__(self, name, category=None, description=None,
allow_all_args=False, conditions=None, parser=None):
self._name = name
self._category = category
self._description = description
self._allow_all_args = allow_all_args
self._conditions = conditions
self._parser = parser
def __call__(self, func):
func._mach_command = (self._name, self._category, self._description,
self._allow_all_args, self._conditions, self._parser)
return func
class CommandArgument(object):
"""Decorator for additional arguments to mach subcommands.
This decorator should be used to add arguments to mach commands. Arguments
to the decorator are proxied to ArgumentParser.add_argument().
For example:
@Command('foo', help='Run the foo action')
@CommandArgument('-b', '--bar', action='store_true', default=False,
help='Enable bar mode.')
def foo(self):
pass
"""
def __init__(self, *args, **kwargs):
self._command_args = (args, kwargs)
def __call__(self, func):
command_args = getattr(func, '_mach_command_args', [])
command_args.insert(0, self._command_args)
func._mach_command_args = command_args
return func
def SettingsProvider(cls):
"""Class decorator to denote that this class provides Mach settings.
When this decorator is encountered, the underlying class will automatically
be registered with the Mach registrar and will (likely) be hooked up to the
mach driver.
This decorator is only allowed on mach.config.ConfigProvider classes.
"""
if not issubclass(cls, ConfigProvider):
raise MachError('@SettingsProvider encountered on class that does ' +
'not derived from mach.config.ConfigProvider.')
Registrar.register_settings_provider(cls)
return cls

277
python/mach/mach/dispatcher.py Normal file
View file

@ -0,0 +1,277 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import argparse
import difflib
import sys
from operator import itemgetter
from .base import (
NoCommandError,
UnknownCommandError,
UnrecognizedArgumentError,
)
class CommandFormatter(argparse.HelpFormatter):
"""Custom formatter to format just a subcommand."""
def add_usage(self, *args):
pass
class CommandAction(argparse.Action):
"""An argparse action that handles mach commands.
This class is essentially a reimplementation of argparse's sub-parsers
feature. We first tried to use sub-parsers. However, they were missing
features like grouping of commands (http://bugs.python.org/issue14037).
The way this works involves light magic and a partial understanding of how
argparse works.
Arguments registered with an argparse.ArgumentParser have an action
associated with them. An action is essentially a class that when called
does something with the encountered argument(s). This class is one of those
action classes.
An instance of this class is created doing something like:
parser.add_argument('command', action=CommandAction, registrar=r)
Note that a mach.registrar.Registrar instance is passed in. The Registrar
holds information on all the mach commands that have been registered.
When this argument is registered with the ArgumentParser, an instance of
this class is instantiated. One of the subtle but important things it does
is tell the argument parser that it's interested in *all* of the remaining
program arguments. So, when the ArgumentParser calls this action, we will
receive the command name plus all of its arguments.
For more, read the docs in __call__.
"""
def __init__(self, option_strings, dest, required=True, default=None,
registrar=None, context=None):
# A proper API would have **kwargs here. However, since we are a little
# hacky, we intentionally omit it as a way of detecting potentially
# breaking changes with argparse's implementation.
#
# In a similar vein, default is passed in but is not needed, so we drop
# it.
argparse.Action.__init__(self, option_strings, dest, required=required,
help=argparse.SUPPRESS, nargs=argparse.REMAINDER)
self._mach_registrar = registrar
self._context = context
def __call__(self, parser, namespace, values, option_string=None):
"""This is called when the ArgumentParser has reached our arguments.
Since we always register ourselves with nargs=argparse.REMAINDER,
values should be a list of remaining arguments to parse. The first
argument should be the name of the command to invoke and all remaining
arguments are arguments for that command.
The gist of the flow is that we look at the command being invoked. If
it's *help*, we handle that specially (because argparse's default help
handler isn't satisfactory). Else, we create a new, independent
ArgumentParser instance for just the invoked command (based on the
information contained in the command registrar) and feed the arguments
into that parser. We then merge the results with the main
ArgumentParser.
"""
if namespace.help:
# -h or --help is in the global arguments.
self._handle_main_help(parser, namespace.verbose)
sys.exit(0)
elif values:
command = values[0].lower()
args = values[1:]
if command == 'help':
if args and args[0] not in ['-h', '--help']:
# Make sure args[0] is indeed a command.
self._handle_subcommand_help(parser, args[0])
else:
self._handle_main_help(parser, namespace.verbose)
sys.exit(0)
elif '-h' in args or '--help' in args:
# -h or --help is in the command arguments.
self._handle_subcommand_help(parser, command)
sys.exit(0)
else:
raise NoCommandError()
# Command suggestion
if command not in self._mach_registrar.command_handlers:
# We first try to look for a valid command that is very similar to the given command.
suggested_commands = difflib.get_close_matches(command, self._mach_registrar.command_handlers.keys(), cutoff=0.8)
# If we find more than one matching command, or no command at all, we give command suggestions instead
# (with a lower matching threshold). All commands that start with the given command (for instance: 'mochitest-plain',
# 'mochitest-chrome', etc. for 'mochitest-') are also included.
if len(suggested_commands) != 1:
suggested_commands = set(difflib.get_close_matches(command, self._mach_registrar.command_handlers.keys(), cutoff=0.5))
suggested_commands |= {cmd for cmd in self._mach_registrar.command_handlers if cmd.startswith(command)}
raise UnknownCommandError(command, 'run', suggested_commands)
sys.stderr.write("We're assuming the '%s' command is '%s' and we're executing it for you.\n\n" % (command, suggested_commands[0]))
command = suggested_commands[0]
handler = self._mach_registrar.command_handlers.get(command)
# FUTURE
# If we wanted to conditionally enable commands based on whether
# it's possible to run them given the current state of system, here
# would be a good place to hook that up.
# We create a new parser, populate it with the command's arguments,
# then feed all remaining arguments to it, merging the results
# with ourselves. This is essentially what argparse subparsers
# do.
parser_args = {
'add_help': False,
'usage': '%(prog)s [global arguments] ' + command +
' command arguments]',
}
if handler.allow_all_arguments:
parser_args['prefix_chars'] = '+'
if handler.parser:
subparser = handler.parser
else:
subparser = argparse.ArgumentParser(**parser_args)
for arg in handler.arguments:
subparser.add_argument(*arg[0], **arg[1])
# We define the command information on the main parser result so as to
# not interfere with arguments passed to the command.
setattr(namespace, 'mach_handler', handler)
setattr(namespace, 'command', command)
command_namespace, extra = subparser.parse_known_args(args)
setattr(namespace, 'command_args', command_namespace)
if extra:
raise UnrecognizedArgumentError(command, extra)
def _handle_main_help(self, parser, verbose):
# Since we don't need full sub-parser support for the main help output,
# we create groups in the ArgumentParser and populate each group with
# arguments corresponding to command names. This has the side-effect
# that argparse renders it nicely.
r = self._mach_registrar
disabled_commands = []
cats = [(k, v[2]) for k, v in r.categories.items()]
sorted_cats = sorted(cats, key=itemgetter(1), reverse=True)
for category, priority in sorted_cats:
group = None
for command in sorted(r.commands_by_category[category]):
handler = r.command_handlers[command]
# Instantiate a handler class to see if it should be filtered
# out for the current context or not. Condition functions can be
# applied to the command's decorator.
if handler.conditions:
if handler.pass_context:
instance = handler.cls(self._context)
else:
instance = handler.cls()
is_filtered = False
for c in handler.conditions:
if not c(instance):
is_filtered = True
break
if is_filtered:
description = handler.description
disabled_command = {'command': command, 'description': description}
disabled_commands.append(disabled_command)
continue
if group is None:
title, description, _priority = r.categories[category]
group = parser.add_argument_group(title, description)
description = handler.description
group.add_argument(command, help=description,
action='store_true')
if disabled_commands and 'disabled' in r.categories:
title, description, _priority = r.categories['disabled']
group = parser.add_argument_group(title, description)
if verbose == True:
for c in disabled_commands:
group.add_argument(c['command'], help=c['description'],
action='store_true')
parser.print_help()
def _handle_subcommand_help(self, parser, command):
handler = self._mach_registrar.command_handlers.get(command)
if not handler:
raise UnknownCommandError(command, 'query')
# This code is worth explaining. Because we are doing funky things with
# argument registration to allow the same option in both global and
# command arguments, we can't simply put all arguments on the same
# parser instance because argparse would complain. We can't register an
# argparse subparser here because it won't properly show help for
# global arguments. So, we employ a strategy similar to command
# execution where we construct a 2nd, independent ArgumentParser for
# just the command data then supplement the main help's output with
# this 2nd parser's. We use a custom formatter class to ignore some of
# the help output.
parser_args = {
'formatter_class': CommandFormatter,
'add_help': False,
}
if handler.allow_all_arguments:
parser_args['prefix_chars'] = '+'
if handler.parser:
c_parser = handler.parser
c_parser.formatter_class = NoUsageFormatter
# Accessing _action_groups is a bit shady. We are highly dependent
# on the argparse implementation not changing. We fail fast to
# detect upstream changes so we can intelligently react to them.
group = c_parser._action_groups[1]
# By default argparse adds two groups called "positional arguments"
# and "optional arguments". We want to rename these to reflect standard
# mach terminology.
c_parser._action_groups[0].title = 'Command Parameters'
c_parser._action_groups[1].title = 'Command Arguments'
if not handler.description:
handler.description = c_parser.description
c_parser.description = None
else:
c_parser = argparse.ArgumentParser(**parser_args)
group = c_parser.add_argument_group('Command Arguments')
for arg in handler.arguments:
group.add_argument(*arg[0], **arg[1])
# This will print the description of the command below the usage.
description = handler.description
if description:
parser.description = description
parser.usage = '%(prog)s [global arguments] ' + command + \
' [command arguments]'
parser.print_help()
print('')
c_parser.print_help()
class NoUsageFormatter(argparse.HelpFormatter):
def _format_usage(self, *args, **kwargs):
return ""

256
python/mach/mach/logging.py Normal file
View file

@ -0,0 +1,256 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# This file contains logging functionality for mach. It essentially provides
# support for a structured logging framework built on top of Python's built-in
# logging framework.
from __future__ import absolute_import, unicode_literals
try:
import blessings
except ImportError:
blessings = None
import json
import logging
import sys
import time
def format_seconds(total):
"""Format number of seconds to MM:SS.DD form."""
minutes, seconds = divmod(total, 60)
return '%2d:%05.2f' % (minutes, seconds)
class ConvertToStructuredFilter(logging.Filter):
"""Filter that converts unstructured records into structured ones."""
def filter(self, record):
if hasattr(record, 'action') and hasattr(record, 'params'):
return True
record.action = 'unstructured'
record.params = {'msg': record.getMessage()}
record.msg = '{msg}'
return True
class StructuredJSONFormatter(logging.Formatter):
"""Log formatter that writes a structured JSON entry."""
def format(self, record):
action = getattr(record, 'action', 'UNKNOWN')
params = getattr(record, 'params', {})
return json.dumps([record.created, action, params])
class StructuredHumanFormatter(logging.Formatter):
"""Log formatter that writes structured messages for humans.
It is important that this formatter never be added to a logger that
produces unstructured/classic log messages. If it is, the call to format()
could fail because the string could contain things (like JSON) that look
like formatting character sequences.
Because of this limitation, format() will fail with a KeyError if an
unstructured record is passed or if the structured message is malformed.
"""
def __init__(self, start_time, write_interval=False, write_times=True):
self.start_time = start_time
self.write_interval = write_interval
self.write_times = write_times
self.last_time = None
def format(self, record):
f = record.msg.format(**record.params)
if not self.write_times:
return f
elapsed = self._time(record)
return '%s %s' % (format_seconds(elapsed), f)
def _time(self, record):
t = record.created - self.start_time
if self.write_interval and self.last_time is not None:
t = record.created - self.last_time
self.last_time = record.created
return t
class StructuredTerminalFormatter(StructuredHumanFormatter):
"""Log formatter for structured messages writing to a terminal."""
def set_terminal(self, terminal):
self.terminal = terminal
def format(self, record):
f = record.msg.format(**record.params)
if not self.write_times:
return f
t = self.terminal.blue(format_seconds(self._time(record)))
return '%s %s' % (t, self._colorize(f))
def _colorize(self, s):
if not self.terminal:
return s
result = s
reftest = s.startswith('REFTEST ')
if reftest:
s = s[8:]
if s.startswith('TEST-PASS'):
result = self.terminal.green(s[0:9]) + s[9:]
elif s.startswith('TEST-UNEXPECTED'):
result = self.terminal.red(s[0:20]) + s[20:]
elif s.startswith('TEST-START'):
result = self.terminal.yellow(s[0:10]) + s[10:]
elif s.startswith('TEST-INFO'):
result = self.terminal.yellow(s[0:9]) + s[9:]
if reftest:
result = 'REFTEST ' + result
return result
class LoggingManager(object):
"""Holds and controls global logging state.
An application should instantiate one of these and configure it as needed.
This class provides a mechanism to configure the output of logging data
both from mach and from the overall logging system (e.g. from other
modules).
"""
def __init__(self):
self.start_time = time.time()
self.json_handlers = []
self.terminal_handler = None
self.terminal_formatter = None
self.root_logger = logging.getLogger()
self.root_logger.setLevel(logging.DEBUG)
# Installing NullHandler on the root logger ensures that *all* log
# messages have at least one handler. This prevents Python from
# complaining about "no handlers could be found for logger XXX."
self.root_logger.addHandler(logging.NullHandler())
self.mach_logger = logging.getLogger('mach')
self.mach_logger.setLevel(logging.DEBUG)
self.structured_filter = ConvertToStructuredFilter()
self.structured_loggers = [self.mach_logger]
self._terminal = None
@property
def terminal(self):
if not self._terminal and blessings:
# Sometimes blessings fails to set up the terminal. In that case,
# silently fail.
try:
terminal = blessings.Terminal(stream=sys.stdout)
if terminal.is_a_tty:
self._terminal = terminal
except Exception:
pass
return self._terminal
def add_json_handler(self, fh):
"""Enable JSON logging on the specified file object."""
# Configure the consumer of structured messages.
handler = logging.StreamHandler(stream=fh)
handler.setFormatter(StructuredJSONFormatter())
handler.setLevel(logging.DEBUG)
# And hook it up.
for logger in self.structured_loggers:
logger.addHandler(handler)
self.json_handlers.append(handler)
def add_terminal_logging(self, fh=sys.stdout, level=logging.INFO,
write_interval=False, write_times=True):
"""Enable logging to the terminal."""
formatter = StructuredHumanFormatter(self.start_time,
write_interval=write_interval, write_times=write_times)
if self.terminal:
formatter = StructuredTerminalFormatter(self.start_time,
write_interval=write_interval, write_times=write_times)
formatter.set_terminal(self.terminal)
handler = logging.StreamHandler(stream=fh)
handler.setFormatter(formatter)
handler.setLevel(level)
for logger in self.structured_loggers:
logger.addHandler(handler)
self.terminal_handler = handler
self.terminal_formatter = formatter
def replace_terminal_handler(self, handler):
"""Replace the installed terminal handler.
Returns the old handler or None if none was configured.
If the new handler is None, removes any existing handler and disables
logging to the terminal.
"""
old = self.terminal_handler
if old:
for logger in self.structured_loggers:
logger.removeHandler(old)
if handler:
for logger in self.structured_loggers:
logger.addHandler(handler)
self.terminal_handler = handler
return old
def enable_unstructured(self):
"""Enable logging of unstructured messages."""
if self.terminal_handler:
self.terminal_handler.addFilter(self.structured_filter)
self.root_logger.addHandler(self.terminal_handler)
def disable_unstructured(self):
"""Disable logging of unstructured messages."""
if self.terminal_handler:
self.terminal_handler.removeFilter(self.structured_filter)
self.root_logger.removeHandler(self.terminal_handler)
def register_structured_logger(self, logger):
"""Register a structured logger.
This needs to be called for all structured loggers that don't chain up
to the mach logger in order for their output to be captured.
"""
self.structured_loggers.append(logger)

615
python/mach/mach/main.py Normal file
View file

@ -0,0 +1,615 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This module provides functionality for the command-line build tool
# (mach). It is packaged as a module because everything is a library.
from __future__ import absolute_import, print_function, unicode_literals
from collections import Iterable
import argparse
import codecs
import imp
import logging
import os
import sys
import traceback
import uuid
import sys
from .base import (
CommandContext,
MachError,
NoCommandError,
UnknownCommandError,
UnrecognizedArgumentError,
)
from .decorators import (
CommandArgument,
CommandProvider,
Command,
)
from .config import ConfigSettings
from .dispatcher import CommandAction
from .logging import LoggingManager
from .registrar import Registrar
MACH_ERROR = r'''
The error occurred in mach itself. This is likely a bug in mach itself or a
fundamental problem with a loaded module.
Please consider filing a bug against mach by going to the URL:
https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=mach
'''.lstrip()
ERROR_FOOTER = r'''
If filing a bug, please include the full output of mach, including this error
message.
The details of the failure are as follows:
'''.lstrip()
COMMAND_ERROR = r'''
The error occurred in the implementation of the invoked mach command.
This should never occur and is likely a bug in the implementation of that
command. Consider filing a bug for this issue.
'''.lstrip()
MODULE_ERROR = r'''
The error occurred in code that was called by the mach command. This is either
a bug in the called code itself or in the way that mach is calling it.
You should consider filing a bug for this issue.
'''.lstrip()
NO_COMMAND_ERROR = r'''
It looks like you tried to run mach without a command.
Run |mach help| to show a list of commands.
'''.lstrip()
UNKNOWN_COMMAND_ERROR = r'''
It looks like you are trying to %s an unknown mach command: %s
%s
Run |mach help| to show a list of commands.
'''.lstrip()
SUGGESTED_COMMANDS_MESSAGE = r'''
Did you want to %s any of these commands instead: %s?
'''
UNRECOGNIZED_ARGUMENT_ERROR = r'''
It looks like you passed an unrecognized argument into mach.
The %s command does not accept the arguments: %s
'''.lstrip()
INVALID_COMMAND_CONTEXT = r'''
It looks like you tried to run a mach command from an invalid context. The %s
command failed to meet the following conditions: %s
Run |mach help| to show a list of all commands available to the current context.
'''.lstrip()
INVALID_ENTRY_POINT = r'''
Entry points should return a list of command providers or directories
containing command providers. The following entry point is invalid:
%s
You are seeing this because there is an error in an external module attempting
to implement a mach command. Please fix the error, or uninstall the module from
your system.
'''.lstrip()
class ArgumentParser(argparse.ArgumentParser):
"""Custom implementation argument parser to make things look pretty."""
def error(self, message):
"""Custom error reporter to give more helpful text on bad commands."""
if not message.startswith('argument command: invalid choice'):
argparse.ArgumentParser.error(self, message)
assert False
print('Invalid command specified. The list of commands is below.\n')
self.print_help()
sys.exit(1)
def format_help(self):
text = argparse.ArgumentParser.format_help(self)
# Strip out the silly command list that would preceed the pretty list.
#
# Commands:
# {foo,bar}
# foo Do foo.
# bar Do bar.
search = 'Commands:\n {'
start = text.find(search)
if start != -1:
end = text.find('}\n', start)
assert end != -1
real_start = start + len('Commands:\n')
real_end = end + len('}\n')
text = text[0:real_start] + text[real_end:]
return text
class ContextWrapper(object):
def __init__(self, context, handler):
object.__setattr__(self, '_context', context)
object.__setattr__(self, '_handler', handler)
def __getattribute__(self, key):
try:
return getattr(object.__getattribute__(self, '_context'), key)
except AttributeError as e:
try:
ret = object.__getattribute__(self, '_handler')(self, key)
except AttributeError, TypeError:
# TypeError is in case the handler comes from old code not
# taking a key argument.
raise e
setattr(self, key, ret)
return ret
def __setattr__(self, key, value):
setattr(object.__getattribute__(self, '_context'), key, value)
@CommandProvider
class Mach(object):
"""Main mach driver type.
This type is responsible for holding global mach state and dispatching
a command from arguments.
The following attributes may be assigned to the instance to influence
behavior:
populate_context_handler -- If defined, it must be a callable. The
callable signature is the following:
populate_context_handler(context, key=None)
It acts as a fallback getter for the mach.base.CommandContext
instance.
This allows to augment the context instance with arbitrary data
for use in command handlers.
For backwards compatibility, it is also called before command
dispatch without a key, allowing the context handler to add
attributes to the context instance.
require_conditions -- If True, commands that do not have any condition
functions applied will be skipped. Defaults to False.
"""
USAGE = """%(prog)s [global arguments] command [command arguments]
mach (German for "do") is the main interface to the Mozilla build system and
common developer tasks.
You tell mach the command you want to perform and it does it for you.
Some common commands are:
%(prog)s build Build/compile the source tree.
%(prog)s help Show full help, including the list of all commands.
To see more help for a specific command, run:
%(prog)s help <command>
"""
def __init__(self, cwd):
assert os.path.isdir(cwd)
self.cwd = cwd
self.log_manager = LoggingManager()
self.logger = logging.getLogger(__name__)
self.settings = ConfigSettings()
self.log_manager.register_structured_logger(self.logger)
self.global_arguments = []
self.populate_context_handler = None
def add_global_argument(self, *args, **kwargs):
"""Register a global argument with the argument parser.
Arguments are proxied to ArgumentParser.add_argument()
"""
self.global_arguments.append((args, kwargs))
def load_commands_from_directory(self, path):
"""Scan for mach commands from modules in a directory.
This takes a path to a directory, loads the .py files in it, and
registers and found mach command providers with this mach instance.
"""
for f in sorted(os.listdir(path)):
if not f.endswith('.py') or f == '__init__.py':
continue
full_path = os.path.join(path, f)
module_name = 'mach.commands.%s' % f[0:-3]
self.load_commands_from_file(full_path, module_name=module_name)
def load_commands_from_file(self, path, module_name=None):
"""Scan for mach commands from a file.
This takes a path to a file and loads it as a Python module under the
module name specified. If no name is specified, a random one will be
chosen.
"""
if module_name is None:
# Ensure parent module is present otherwise we'll (likely) get
# an error due to unknown parent.
if b'mach.commands' not in sys.modules:
mod = imp.new_module(b'mach.commands')
sys.modules[b'mach.commands'] = mod
module_name = 'mach.commands.%s' % uuid.uuid1().get_hex()
imp.load_source(module_name, path)
def load_commands_from_entry_point(self, group='mach.providers'):
"""Scan installed packages for mach command provider entry points. An
entry point is a function that returns a list of paths to files or
directories containing command providers.
This takes an optional group argument which specifies the entry point
group to use. If not specified, it defaults to 'mach.providers'.
"""
try:
import pkg_resources
except ImportError:
print("Could not find setuptools, ignoring command entry points",
file=sys.stderr)
return
for entry in pkg_resources.iter_entry_points(group=group, name=None):
paths = entry.load()()
if not isinstance(paths, Iterable):
print(INVALID_ENTRY_POINT % entry)
sys.exit(1)
for path in paths:
if os.path.isfile(path):
self.load_commands_from_file(path)
elif os.path.isdir(path):
self.load_commands_from_directory(path)
else:
print("command provider '%s' does not exist" % path)
def define_category(self, name, title, description, priority=50):
"""Provide a description for a named command category."""
Registrar.register_category(name, title, description, priority)
@property
def require_conditions(self):
return Registrar.require_conditions
@require_conditions.setter
def require_conditions(self, value):
Registrar.require_conditions = value
def run(self, argv, stdin=None, stdout=None, stderr=None):
"""Runs mach with arguments provided from the command line.
Returns the integer exit code that should be used. 0 means success. All
other values indicate failure.
"""
# If no encoding is defined, we default to UTF-8 because without this
# Python 2.7 will assume the default encoding of ASCII. This will blow
# up with UnicodeEncodeError as soon as it encounters a non-ASCII
# character in a unicode instance. We simply install a wrapper around
# the streams and restore once we have finished.
stdin = sys.stdin if stdin is None else stdin
stdout = sys.stdout if stdout is None else stdout
stderr = sys.stderr if stderr is None else stderr
orig_stdin = sys.stdin
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdin = stdin
sys.stdout = stdout
sys.stderr = stderr
try:
if stdin.encoding is None:
sys.stdin = codecs.getreader('utf-8')(stdin)
if stdout.encoding is None:
sys.stdout = codecs.getwriter('utf-8')(stdout)
if stderr.encoding is None:
sys.stderr = codecs.getwriter('utf-8')(stderr)
return self._run(argv)
except KeyboardInterrupt:
print('mach interrupted by signal or user action. Stopping.')
return 1
except Exception as e:
# _run swallows exceptions in invoked handlers and converts them to
# a proper exit code. So, the only scenario where we should get an
# exception here is if _run itself raises. If _run raises, that's a
# bug in mach (or a loaded command module being silly) and thus
# should be reported differently.
self._print_error_header(argv, sys.stdout)
print(MACH_ERROR)
exc_type, exc_value, exc_tb = sys.exc_info()
stack = traceback.extract_tb(exc_tb)
self._print_exception(sys.stdout, exc_type, exc_value, stack)
return 1
finally:
sys.stdin = orig_stdin
sys.stdout = orig_stdout
sys.stderr = orig_stderr
def _run(self, argv):
context = CommandContext(cwd=self.cwd,
settings=self.settings, log_manager=self.log_manager,
commands=Registrar)
if self.populate_context_handler:
self.populate_context_handler(context)
context = ContextWrapper(context, self.populate_context_handler)
parser = self.get_argument_parser(context)
if not len(argv):
# We don't register the usage until here because if it is globally
# registered, argparse always prints it. This is not desired when
# running with --help.
parser.usage = Mach.USAGE
parser.print_usage()
return 0
try:
args = parser.parse_args(argv)
except NoCommandError:
print(NO_COMMAND_ERROR)
return 1
except UnknownCommandError as e:
suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else ''
print(UNKNOWN_COMMAND_ERROR % (e.verb, e.command, suggestion_message))
return 1
except UnrecognizedArgumentError as e:
print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command,
' '.join(e.arguments)))
return 1
# Add JSON logging to a file if requested.
if args.logfile:
self.log_manager.add_json_handler(args.logfile)
# Up the logging level if requested.
log_level = logging.INFO
if args.verbose:
log_level = logging.DEBUG
self.log_manager.register_structured_logger(logging.getLogger('mach'))
write_times = True
if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ:
write_times = False
# Always enable terminal logging. The log manager figures out if we are
# actually in a TTY or are a pipe and does the right thing.
self.log_manager.add_terminal_logging(level=log_level,
write_interval=args.log_interval, write_times=write_times)
self.load_settings(args)
if not hasattr(args, 'mach_handler'):
raise MachError('ArgumentParser result missing mach handler info.')
handler = getattr(args, 'mach_handler')
cls = handler.cls
if handler.pass_context:
instance = cls(context)
else:
instance = cls()
if handler.conditions:
fail_conditions = []
for c in handler.conditions:
if not c(instance):
fail_conditions.append(c)
if fail_conditions:
print(self._condition_failed_message(handler.name, fail_conditions))
return 1
fn = getattr(instance, handler.method)
try:
result = fn(**vars(args.command_args))
if not result:
result = 0
assert isinstance(result, (int, long))
return result
except KeyboardInterrupt as ki:
raise ki
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
# The first frame is us and is never used.
stack = traceback.extract_tb(exc_tb)[1:]
# If we have nothing on the stack, the exception was raised as part
# of calling the @Command method itself. This likely means a
# mismatch between @CommandArgument and arguments to the method.
# e.g. there exists a @CommandArgument without the corresponding
# argument on the method. We handle that here until the module
# loader grows the ability to validate better.
if not len(stack):
print(COMMAND_ERROR)
self._print_exception(sys.stdout, exc_type, exc_value,
traceback.extract_tb(exc_tb))
return 1
# Split the frames into those from the module containing the
# command and everything else.
command_frames = []
other_frames = []
initial_file = stack[0][0]
for frame in stack:
if frame[0] == initial_file:
command_frames.append(frame)
else:
other_frames.append(frame)
# If the exception was in the module providing the command, it's
# likely the bug is in the mach command module, not something else.
# If there are other frames, the bug is likely not the mach
# command's fault.
self._print_error_header(argv, sys.stdout)
if len(other_frames):
print(MODULE_ERROR)
else:
print(COMMAND_ERROR)
self._print_exception(sys.stdout, exc_type, exc_value, stack)
return 1
def log(self, level, action, params, format_str):
"""Helper method to record a structured log event."""
self.logger.log(level, format_str,
extra={'action': action, 'params': params})
@classmethod
def _condition_failed_message(cls, name, conditions):
msg = ['\n']
for c in conditions:
part = [' %s' % c.__name__]
if c.__doc__ is not None:
part.append(c.__doc__)
msg.append(' - '.join(part))
return INVALID_COMMAND_CONTEXT % (name, '\n'.join(msg))
def _print_error_header(self, argv, fh):
fh.write('Error running mach:\n\n')
fh.write(' ')
fh.write(repr(argv))
fh.write('\n\n')
def _print_exception(self, fh, exc_type, exc_value, stack):
fh.write(ERROR_FOOTER)
fh.write('\n')
for l in traceback.format_exception_only(exc_type, exc_value):
fh.write(l)
fh.write('\n')
for l in traceback.format_list(stack):
fh.write(l)
def load_settings(self, args):
"""Determine which settings files apply and load them.
Currently, we only support loading settings from a single file.
Ideally, we support loading from multiple files. This is supported by
the ConfigSettings API. However, that API currently doesn't track where
individual values come from, so if we load from multiple sources then
save, we effectively do a full copy. We don't want this. Until
ConfigSettings does the right thing, we shouldn't expose multi-file
loading.
We look for a settings file in the following locations. The first one
found wins:
1) Command line argument
2) Environment variable
3) Default path
"""
# Settings are disabled until integration with command providers is
# worked out.
self.settings = None
return False
for provider in Registrar.settings_providers:
provider.register_settings()
self.settings.register_provider(provider)
p = os.path.join(self.cwd, 'mach.ini')
if args.settings_file:
p = args.settings_file
elif 'MACH_SETTINGS_FILE' in os.environ:
p = os.environ['MACH_SETTINGS_FILE']
self.settings.load_file(p)
return os.path.exists(p)
def get_argument_parser(self, context):
"""Returns an argument parser for the command-line interface."""
parser = ArgumentParser(add_help=False,
usage='%(prog)s [global arguments] command [command arguments]')
# Order is important here as it dictates the order the auto-generated
# help messages are printed.
global_group = parser.add_argument_group('Global Arguments')
#global_group.add_argument('--settings', dest='settings_file',
# metavar='FILENAME', help='Path to settings file.')
global_group.add_argument('-v', '--verbose', dest='verbose',
action='store_true', default=False,
help='Print verbose output.')
global_group.add_argument('-l', '--log-file', dest='logfile',
metavar='FILENAME', type=argparse.FileType('ab'),
help='Filename to write log data to.')
global_group.add_argument('--log-interval', dest='log_interval',
action='store_true', default=False,
help='Prefix log line with interval from last message rather '
'than relative time. Note that this is NOT execution time '
'if there are parallel operations.')
global_group.add_argument('--log-no-times', dest='log_no_times',
action='store_true', default=False,
help='Do not prefix log lines with times. By default, mach will '
'prefix each output line with the time since command start.')
global_group.add_argument('-h', '--help', dest='help',
action='store_true', default=False,
help='Show this help message.')
for args, kwargs in self.global_arguments:
global_group.add_argument(*args, **kwargs)
# We need to be last because CommandAction swallows all remaining
# arguments and argparse parses arguments in the order they were added.
parser.add_argument('command', action=CommandAction,
registrar=Registrar, context=context)
return parser

0
python/mach/mach/mixin/__init__.py Normal file
View file

55
python/mach/mach/mixin/logging.py Normal file
View file

@ -0,0 +1,55 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
import logging
class LoggingMixin(object):
"""Provides functionality to control logging."""
def populate_logger(self, name=None):
"""Ensure this class instance has a logger associated with it.
Users of this mixin that call log() will need to ensure self._logger is
a logging.Logger instance before they call log(). This function ensures
self._logger is defined by populating it if it isn't.
"""
if hasattr(self, '_logger'):
return
if name is None:
name = '.'.join([self.__module__, self.__class__.__name__])
self._logger = logging.getLogger(name)
def log(self, level, action, params, format_str):
"""Log a structured log event.
A structured log event consists of a logging level, a string action, a
dictionary of attributes, and a formatting string.
The logging level is one of the logging.* constants, such as
logging.INFO.
The action string is essentially the enumeration of the event. Each
different type of logged event should have a different action.
The params dict is the metadata constituting the logged event.
The formatting string is used to convert the structured message back to
human-readable format. Conversion back to human-readable form is
performed by calling format() on this string, feeding into it the dict
of attributes constituting the event.
Example Usage
-------------
self.log(logging.DEBUG, 'login', {'username': 'johndoe'},
'User login: {username}')
"""
self._logger.log(level, format_str,
extra={'action': action, 'params': params})

175
python/mach/mach/mixin/process.py Normal file
View file

@ -0,0 +1,175 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This module provides mixins to perform process execution.
from __future__ import absolute_import, unicode_literals
import logging
import os
import subprocess
import sys
from mozprocess.processhandler import ProcessHandlerMixin
from .logging import LoggingMixin
# Perform detection of operating system environment. This is used by command
# execution. We only do this once to save redundancy. Yes, this can fail module
# loading. That is arguably OK.
if 'SHELL' in os.environ:
_current_shell = os.environ['SHELL']
elif 'MOZILLABUILD' in os.environ:
_current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
elif 'COMSPEC' in os.environ:
_current_shell = os.environ['COMSPEC']
else:
raise Exception('Could not detect environment shell!')
_in_msys = False
if os.environ.get('MSYSTEM', None) == 'MINGW32':
_in_msys = True
if not _current_shell.lower().endswith('.exe'):
_current_shell += '.exe'
class ProcessExecutionMixin(LoggingMixin):
"""Mix-in that provides process execution functionality."""
def run_process(self, args=None, cwd=None, append_env=None,
explicit_env=None, log_name=None, log_level=logging.INFO,
line_handler=None, require_unix_environment=False,
ensure_exit_code=0, ignore_children=False, pass_thru=False):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
ensure_exit_code is used to ensure the exit code of a process matches
what is expected. If it is an integer, we raise an Exception if the
exit code does not match this value. If it is True, we ensure the exit
code is 0. If it is False, we don't perform any exit code validation.
pass_thru is a special execution mode where the child process inherits
this process's standard file handles (stdin, stdout, stderr) as well as
additional file descriptors. It should be used for interactive processes
where buffering from mozprocess could be an issue. pass_thru does not
use mozprocess. Therefore, arguments like log_name, line_handler,
and ignore_children have no effect.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args))
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.rstrip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
# There is a bug in subprocess where it doesn't like unicode types in
# environment variables. Here, ensure all unicode are converted to
# binary. utf-8 is our globally assumed default. If the caller doesn't
# want UTF-8, they shouldn't pass in a unicode instance.
normalized_env = {}
for k, v in use_env.items():
if isinstance(k, unicode):
k = k.encode('utf-8', 'strict')
if isinstance(v, unicode):
v = v.encode('utf-8', 'strict')
normalized_env[k] = v
use_env = normalized_env
if pass_thru:
proc = subprocess.Popen(args, cwd=cwd, env=use_env)
status = None
# Leave it to the subprocess to handle Ctrl+C. If it terminates as
# a result of Ctrl+C, proc.wait() will return a status code, and,
# we get out of the loop. If it doesn't, like e.g. gdb, we continue
# waiting.
while status is None:
try:
status = proc.wait()
except KeyboardInterrupt:
pass
else:
p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
processOutputLine=[handleLine], universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = p.wait()
if ensure_exit_code is False:
return status
if ensure_exit_code is True:
ensure_exit_code = 0
if status != ensure_exit_code:
raise Exception('Process executed with non-0 exit code: %s' % args)
return status
def _normalize_command(self, args, require_unix_environment):
"""Adjust command arguments to run in the necessary environment.
This exists mainly to facilitate execution of programs requiring a *NIX
shell when running on Windows. The caller specifies whether a shell
environment is required. If it is and we are running on Windows but
aren't running in the UNIX-like msys environment, then we rewrite the
command to execute via a shell.
"""
assert isinstance(args, list) and len(args)
if not require_unix_environment or not _in_msys:
return args
# Always munge Windows-style into Unix style for the command.
prog = args[0].replace('\\', '/')
# PyMake removes the C: prefix. But, things seem to work here
# without it. Not sure what that's about.
# We run everything through the msys shell. We need to use
# '-c' and pass all the arguments as one argument because that is
# how sh works.
cline = subprocess.list2cmdline([prog] + args[1:])
return [_current_shell, '-c', cline]

65
python/mach/mach/registrar.py Normal file
View file

@ -0,0 +1,65 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
from .base import MachError
class MachRegistrar(object):
"""Container for mach command and config providers."""
def __init__(self):
self.command_handlers = {}
self.commands_by_category = {}
self.settings_providers = set()
self.categories = {}
self.require_conditions = False
def register_command_handler(self, handler):
name = handler.name
if not handler.category:
raise MachError('Cannot register a mach command without a '
'category: %s' % name)
if handler.category not in self.categories:
raise MachError('Cannot register a command to an undefined '
'category: %s -> %s' % (name, handler.category))
self.command_handlers[name] = handler
self.commands_by_category[handler.category].add(name)
def register_settings_provider(self, cls):
self.settings_providers.add(cls)
def register_category(self, name, title, description, priority=50):
self.categories[name] = (title, description, priority)
self.commands_by_category[name] = set()
def dispatch(self, name, context=None, **args):
"""Dispatch/run a command.
Commands can use this to call other commands.
"""
# TODO The logic in this function overlaps with code in
# mach.main.Main._run() and should be consolidated.
handler = self.command_handlers[name]
cls = handler.cls
if handler.pass_context and not context:
raise Exception('mach command class requires context.')
if handler.pass_context:
instance = cls(context)
else:
instance = cls()
fn = getattr(instance, handler.method)
return fn(**args) or 0
Registrar = MachRegistrar()

75
python/mach/mach/terminal.py Normal file
View file

@ -0,0 +1,75 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""This file contains code for interacting with terminals.
All the terminal interaction code is consolidated so the complexity can be in
one place, away from code that is commonly looked at.
"""
from __future__ import print_function, unicode_literals
import logging
import sys
class LoggingHandler(logging.Handler):
"""Custom logging handler that works with terminal window dressing.
This is alternative terminal logging handler which contains smarts for
emitting terminal control characters properly. Currently, it has generic
support for "footer" elements at the bottom of the screen. Functionality
can be added when needed.
"""
def __init__(self):
logging.Handler.__init__(self)
self.fh = sys.stdout
self.footer = None
def flush(self):
self.acquire()
try:
self.fh.flush()
finally:
self.release()
def emit(self, record):
msg = self.format(record)
if self.footer:
self.footer.clear()
self.fh.write(msg)
self.fh.write('\n')
if self.footer:
self.footer.draw()
# If we don't flush, the footer may not get drawn.
self.flush()
class TerminalFooter(object):
"""Represents something drawn on the bottom of a terminal."""
def __init__(self, terminal):
self.t = terminal
self.fh = sys.stdout
def _clear_lines(self, n):
for i in xrange(n):
self.fh.write(self.t.move_x(0))
self.fh.write(self.t.clear_eol())
self.fh.write(self.t.move_up())
self.fh.write(self.t.move_down())
self.fh.write(self.t.move_x(0))
def clear(self):
raise Exception('clear() must be implemented.')
def draw(self):
raise Exception('draw() must be implemented.')

0
python/mach/mach/test/__init__.py Normal file
View file

40
python/mach/mach/test/common.py Normal file
View file

@ -0,0 +1,40 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
from StringIO import StringIO
import os
import unittest
from mach.main import Mach
from mach.base import CommandContext
here = os.path.abspath(os.path.dirname(__file__))
class TestBase(unittest.TestCase):
provider_dir = os.path.join(here, 'providers')
def _run_mach(self, args, provider_file=None, entry_point=None, context_handler=None):
m = Mach(os.getcwd())
m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10)
m.populate_context_handler = context_handler
if provider_file:
m.load_commands_from_file(os.path.join(self.provider_dir, provider_file))
if entry_point:
m.load_commands_from_entry_point(entry_point)
stdout = StringIO()
stderr = StringIO()
stdout.encoding = 'UTF-8'
stderr.encoding = 'UTF-8'
try:
result = m.run(args, stdout=stdout, stderr=stderr)
except SystemExit:
result = None
return (result, stdout.getvalue(), stderr.getvalue())

0
python/mach/mach/test/providers/__init__.py Normal file
View file

15
python/mach/mach/test/providers/basic.py Normal file
View file

@ -0,0 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
from mach.decorators import (
CommandProvider,
Command,
)
@CommandProvider
class ConditionsProvider(object):
@Command('cmd_foo', category='testing')
def run_foo(self):
pass

53
python/mach/mach/test/providers/conditions.py Normal file
View file

@ -0,0 +1,53 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
from mach.decorators import (
CommandProvider,
Command,
)
def is_foo(cls):
"""Foo must be true"""
return cls.foo
def is_bar(cls):
"""Bar must be true"""
return cls.bar
@CommandProvider
class ConditionsProvider(object):
foo = True
bar = False
@Command('cmd_foo', category='testing', conditions=[is_foo])
def run_foo(self):
pass
@Command('cmd_bar', category='testing', conditions=[is_bar])
def run_bar(self):
pass
@Command('cmd_foobar', category='testing', conditions=[is_foo, is_bar])
def run_foobar(self):
pass
@CommandProvider
class ConditionsContextProvider(object):
def __init__(self, context):
self.foo = context.foo
self.bar = context.bar
@Command('cmd_foo_ctx', category='testing', conditions=[is_foo])
def run_foo(self):
pass
@Command('cmd_bar_ctx', category='testing', conditions=[is_bar])
def run_bar(self):
pass
@Command('cmd_foobar_ctx', category='testing', conditions=[is_foo, is_bar])
def run_foobar(self):
pass

16
python/mach/mach/test/providers/conditions_invalid.py Normal file
View file

@ -0,0 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
from mach.decorators import (
CommandProvider,
Command,
)
@CommandProvider
class ConditionsProvider(object):
@Command('cmd_foo', category='testing', conditions=["invalid"])
def run_foo(self):
pass

29
python/mach/mach/test/providers/throw.py Normal file
View file

@ -0,0 +1,29 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import time
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from mach.test.providers import throw2
@CommandProvider
class TestCommandProvider(object):
@Command('throw', category='testing')
@CommandArgument('--message', '-m', default='General Error')
def throw(self, message):
raise Exception(message)
@Command('throw_deep', category='testing')
@CommandArgument('--message', '-m', default='General Error')
def throw_deep(self, message):
throw2.throw_deep(message)

13
python/mach/mach/test/providers/throw2.py Normal file
View file

@ -0,0 +1,13 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This file exists to trigger the differences in mach error reporting between
# exceptions that occur in mach command modules themselves and in the things
# they call.
def throw_deep(message):
return throw_real(message)
def throw_real(message):
raise Exception(message)

82
python/mach/mach/test/test_conditions.py Normal file
View file

@ -0,0 +1,82 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import os
from mach.base import MachError
from mach.main import Mach
from mach.test.common import TestBase
from mozunit import main
def _populate_context(context, key=None):
if key is None:
return
if key == 'foo':
return True
if key == 'bar':
return False
raise AttributeError(key)
class TestConditions(TestBase):
"""Tests for conditionally filtering commands."""
def _run_mach(self, args, context_handler=None):
return TestBase._run_mach(self, args, 'conditions.py',
context_handler=context_handler)
def test_conditions_pass(self):
"""Test that a command which passes its conditions is runnable."""
self.assertEquals((0, '', ''), self._run_mach(['cmd_foo']))
self.assertEquals((0, '', ''), self._run_mach(['cmd_foo_ctx'], _populate_context))
def test_invalid_context_message(self):
"""Test that commands which do not pass all their conditions
print the proper failure message."""
def is_bar():
"""Bar must be true"""
fail_conditions = [is_bar]
for name in ('cmd_bar', 'cmd_foobar'):
result, stdout, stderr = self._run_mach([name])
self.assertEquals(1, result)
fail_msg = Mach._condition_failed_message(name, fail_conditions)
self.assertEquals(fail_msg.rstrip(), stdout.rstrip())
for name in ('cmd_bar_ctx', 'cmd_foobar_ctx'):
result, stdout, stderr = self._run_mach([name], _populate_context)
self.assertEquals(1, result)
fail_msg = Mach._condition_failed_message(name, fail_conditions)
self.assertEquals(fail_msg.rstrip(), stdout.rstrip())
def test_invalid_type(self):
"""Test that a condition which is not callable raises an exception."""
m = Mach(os.getcwd())
m.define_category('testing', 'Mach unittest', 'Testing for mach core', 10)
self.assertRaises(MachError, m.load_commands_from_file,
os.path.join(self.provider_dir, 'conditions_invalid.py'))
def test_help_message(self):
"""Test that commands that are not runnable do not show up in help."""
result, stdout, stderr = self._run_mach(['help'], _populate_context)
self.assertIn('cmd_foo', stdout)
self.assertNotIn('cmd_bar', stdout)
self.assertNotIn('cmd_foobar', stdout)
self.assertIn('cmd_foo_ctx', stdout)
self.assertNotIn('cmd_bar_ctx', stdout)
self.assertNotIn('cmd_foobar_ctx', stdout)
if __name__ == '__main__':
main()

264
python/mach/mach/test/test_config.py Normal file
View file

@ -0,0 +1,264 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import sys
import unittest
from mozfile.mozfile import NamedTemporaryFile
from mach.config import (
AbsolutePathType,
BooleanType,
ConfigProvider,
ConfigSettings,
IntegerType,
PathType,
PositiveIntegerType,
RelativePathType,
StringType,
)
from mozunit import main
if sys.version_info[0] == 3:
str_type = str
else:
str_type = basestring
CONFIG1 = r"""
[foo]
bar = bar_value
baz = /baz/foo.c
"""
CONFIG2 = r"""
[foo]
bar = value2
"""
class Provider1(ConfigProvider):
@classmethod
def _register_settings(cls):
cls.register_setting('foo', 'bar', StringType)
cls.register_setting('foo', 'baz', AbsolutePathType)
Provider1.register_settings()
class ProviderDuplicate(ConfigProvider):
@classmethod
def _register_settings(cls):
cls.register_setting('dupesect', 'foo', StringType)
cls.register_setting('dupesect', 'foo', StringType)
class TestConfigProvider(unittest.TestCase):
def test_construct(self):
s = Provider1.config_settings
self.assertEqual(len(s), 1)
self.assertIn('foo', s)
self.assertEqual(len(s['foo']), 2)
self.assertIn('bar', s['foo'])
self.assertIn('baz', s['foo'])
def test_duplicate_option(self):
with self.assertRaises(Exception):
ProviderDuplicate.register_settings()
class Provider2(ConfigProvider):
@classmethod
def _register_settings(cls):
cls.register_setting('a', 'string', StringType)
cls.register_setting('a', 'boolean', BooleanType)
cls.register_setting('a', 'pos_int', PositiveIntegerType)
cls.register_setting('a', 'int', IntegerType)
cls.register_setting('a', 'abs_path', AbsolutePathType)
cls.register_setting('a', 'rel_path', RelativePathType)
cls.register_setting('a', 'path', PathType)
Provider2.register_settings()
class TestConfigSettings(unittest.TestCase):
def test_empty(self):
s = ConfigSettings()
self.assertEqual(len(s), 0)
self.assertNotIn('foo', s)
def test_simple(self):
s = ConfigSettings()
s.register_provider(Provider1)
self.assertEqual(len(s), 1)
self.assertIn('foo', s)
foo = s['foo']
foo = s.foo
self.assertEqual(len(foo), 2)
self.assertIn('bar', foo)
self.assertIn('baz', foo)
foo['bar'] = 'value1'
self.assertEqual(foo['bar'], 'value1')
self.assertEqual(foo['bar'], 'value1')
def test_assignment_validation(self):
s = ConfigSettings()
s.register_provider(Provider2)
a = s.a
# Assigning an undeclared setting raises.
with self.assertRaises(KeyError):
a.undefined = True
with self.assertRaises(KeyError):
a['undefined'] = True
# Basic type validation.
a.string = 'foo'
a.string = 'foo'
with self.assertRaises(TypeError):
a.string = False
a.boolean = True
a.boolean = False
with self.assertRaises(TypeError):
a.boolean = 'foo'
a.pos_int = 5
a.pos_int = 0
with self.assertRaises(ValueError):
a.pos_int = -1
with self.assertRaises(TypeError):
a.pos_int = 'foo'
a.int = 5
a.int = 0
a.int = -5
with self.assertRaises(TypeError):
a.int = 1.24
with self.assertRaises(TypeError):
a.int = 'foo'
a.abs_path = '/home/gps'
with self.assertRaises(ValueError):
a.abs_path = 'home/gps'
a.rel_path = 'home/gps'
a.rel_path = './foo/bar'
a.rel_path = 'foo.c'
with self.assertRaises(ValueError):
a.rel_path = '/foo/bar'
a.path = '/home/gps'
a.path = 'foo.c'
a.path = 'foo/bar'
a.path = './foo'
def test_retrieval_type(self):
s = ConfigSettings()
s.register_provider(Provider2)
a = s.a
a.string = 'foo'
a.boolean = True
a.pos_int = 12
a.int = -4
a.abs_path = '/home/gps'
a.rel_path = 'foo.c'
a.path = './foo/bar'
self.assertIsInstance(a.string, str_type)
self.assertIsInstance(a.boolean, bool)
self.assertIsInstance(a.pos_int, int)
self.assertIsInstance(a.int, int)
self.assertIsInstance(a.abs_path, str_type)
self.assertIsInstance(a.rel_path, str_type)
self.assertIsInstance(a.path, str_type)
def test_file_reading_single(self):
temp = NamedTemporaryFile(mode='wt')
temp.write(CONFIG1)
temp.flush()
s = ConfigSettings()
s.register_provider(Provider1)
s.load_file(temp.name)
self.assertEqual(s.foo.bar, 'bar_value')
def test_file_reading_multiple(self):
"""Loading multiple files has proper overwrite behavior."""
temp1 = NamedTemporaryFile(mode='wt')
temp1.write(CONFIG1)
temp1.flush()
temp2 = NamedTemporaryFile(mode='wt')
temp2.write(CONFIG2)
temp2.flush()
s = ConfigSettings()
s.register_provider(Provider1)
s.load_files([temp1.name, temp2.name])
self.assertEqual(s.foo.bar, 'value2')
def test_file_reading_missing(self):
"""Missing files should silently be ignored."""
s = ConfigSettings()
s.load_file('/tmp/foo.ini')
def test_file_writing(self):
s = ConfigSettings()
s.register_provider(Provider2)
s.a.string = 'foo'
s.a.boolean = False
temp = NamedTemporaryFile('wt')
s.write(temp)
temp.flush()
s2 = ConfigSettings()
s2.register_provider(Provider2)
s2.load_file(temp.name)
self.assertEqual(s.a.string, s2.a.string)
self.assertEqual(s.a.boolean, s2.a.boolean)
def test_write_pot(self):
s = ConfigSettings()
s.register_provider(Provider1)
s.register_provider(Provider2)
# Just a basic sanity test.
temp = NamedTemporaryFile('wt')
s.write_pot(temp)
temp.flush()
if __name__ == '__main__':
main()

60
python/mach/mach/test/test_entry_point.py Normal file
View file

@ -0,0 +1,60 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
import imp
import os
import sys
from mach.base import MachError
from mach.test.common import TestBase
from mock import patch
from mozunit import main
here = os.path.abspath(os.path.dirname(__file__))
class Entry():
"""Stub replacement for pkg_resources.EntryPoint"""
def __init__(self, providers):
self.providers = providers
def load(self):
def _providers():
return self.providers
return _providers
class TestEntryPoints(TestBase):
"""Test integrating with setuptools entry points"""
provider_dir = os.path.join(here, 'providers')
def _run_mach(self):
return TestBase._run_mach(self, ['help'], entry_point='mach.providers')
@patch('pkg_resources.iter_entry_points')
def test_load_entry_point_from_directory(self, mock):
# Ensure parent module is present otherwise we'll (likely) get
# an error due to unknown parent.
if b'mach.commands' not in sys.modules:
mod = imp.new_module(b'mach.commands')
sys.modules[b'mach.commands'] = mod
mock.return_value = [Entry(['providers'])]
# Mach error raised due to conditions_invalid.py
with self.assertRaises(MachError):
self._run_mach()
@patch('pkg_resources.iter_entry_points')
def test_load_entry_point_from_file(self, mock):
mock.return_value = [Entry([os.path.join('providers', 'basic.py')])]
result, stdout, stderr = self._run_mach()
self.assertIsNone(result)
self.assertIn('cmd_foo', stdout)
# Not enabled in automation because tests are failing.
#if __name__ == '__main__':
# main()

39
python/mach/mach/test/test_error_output.py Normal file
View file

@ -0,0 +1,39 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import unicode_literals
from mach.main import (
COMMAND_ERROR,
MODULE_ERROR
)
from mach.test.common import TestBase
from mozunit import main
class TestErrorOutput(TestBase):
def _run_mach(self, args):
return TestBase._run_mach(self, args, 'throw.py')
def test_command_error(self):
result, stdout, stderr = self._run_mach(['throw', '--message',
'Command Error'])
self.assertEqual(result, 1)
self.assertIn(COMMAND_ERROR, stdout)
def test_invoked_error(self):
result, stdout, stderr = self._run_mach(['throw_deep', '--message',
'Deep stack'])
self.assertEqual(result, 1)
self.assertIn(MODULE_ERROR, stdout)
if __name__ == '__main__':
main()

47
python/mach/mach/test/test_logger.py Normal file
View file

@ -0,0 +1,47 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
import logging
import time
import unittest
from mach.logging import StructuredHumanFormatter
from mozunit import main
class DummyLogger(logging.Logger):
def __init__(self, cb):
logging.Logger.__init__(self, 'test')
self._cb = cb
def handle(self, record):
self._cb(record)
class TestStructuredHumanFormatter(unittest.TestCase):
def test_non_ascii_logging(self):
# Ensures the formatter doesn't choke when non-ASCII characters are
# present in printed parameters.
formatter = StructuredHumanFormatter(time.time())
def on_record(record):
result = formatter.format(record)
relevant = result[9:]
self.assertEqual(relevant, 'Test: s\xe9curit\xe9')
logger = DummyLogger(on_record)
value = 's\xe9curit\xe9'
logger.log(logging.INFO, 'Test: {utf}',
extra={'action': 'action', 'params': {'utf': value}})
if __name__ == '__main__':
main()

38
python/mach/setup.py Normal file
View file

@ -0,0 +1,38 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
try:
from setuptools import setup
except:
from distutils.core import setup
VERSION = '0.3'
README = open('README.rst').read()
setup(
name='mach',
description='Generic command line command dispatching framework.',
long_description=README,
license='MPL 2.0',
author='Gregory Szorc',
author_email='gregory.szorc@gmail.com',
url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach',
packages=['mach'],
version=VERSION,
classifiers=[
'Environment :: Console',
'Development Status :: 3 - Alpha',
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
'Natural Language :: English',
],
install_requires=[
'blessings',
'mozfile',
'mozprocess',
],
tests_require=['mock'],
)

100
python/mach_bootstrap.py Normal file
View file

@ -0,0 +1,100 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
import os
import platform
import sys
SEARCH_PATHS = [
"python/mach",
"python/toml",
]
# Individual files providing mach commands.
MACH_MODULES = [
'python/servo/bootstrap_commands.py',
'python/servo/build_commands.py',
'python/servo/testing_commands.py',
'python/servo/post_build_commands.py',
'python/servo/devenv_commands.py',
]
CATEGORIES = {
'bootstrap': {
'short': 'Bootstrap Commands',
'long': 'Bootstrap the build system',
'priority': 90,
},
'build': {
'short': 'Build Commands',
'long': 'Interact with the build system',
'priority': 80,
},
'post-build': {
'short': 'Post-build Commands',
'long': 'Common actions performed after completing a build.',
'priority': 70,
},
'testing': {
'short': 'Testing',
'long': 'Run tests.',
'priority': 60,
},
'devenv': {
'short': 'Development Environment',
'long': 'Set up and configure your development environment.',
'priority': 50,
},
'build-dev': {
'short': 'Low-level Build System Interaction',
'long': 'Interact with specific parts of the build system.',
'priority': 20,
},
'misc': {
'short': 'Potpourri',
'long': 'Potent potables and assorted snacks.',
'priority': 10,
},
'disabled': {
'short': 'Disabled',
'long': 'The disabled commands are hidden by default. Use -v to display them. These commands are unavailable for your current context, run "mach <command>" to see why.',
'priority': 0,
}
}
def bootstrap(topdir):
topdir = os.path.abspath(topdir)
# Ensure we are running Python 2.7+. We put this check here so we generate a
# user-friendly error message rather than a cryptic stack trace on module
# import.
if sys.version_info[0] != 2 or sys.version_info[1] < 7:
print('Python 2.7 or above (but not Python 3) is required to run mach.')
print('You are running Python', platform.python_version())
sys.exit(1)
def populate_context(context, key=None):
if key is None:
return
if key == 'topdir':
return topdir
raise AttributeError(key)
sys.path[0:0] = [os.path.join(topdir, path) for path in SEARCH_PATHS]
import mach.main
mach = mach.main.Mach(os.getcwd())
mach.populate_context_handler = populate_context
for category, meta in CATEGORIES.items():
mach.define_category(category, meta['short'], meta['long'],
meta['priority'])
for path in MACH_MODULES:
mach.load_commands_from_file(os.path.join(topdir, path))
return mach

0
python/servo/__init__.py Normal file
View file

153
python/servo/bootstrap_commands.py Normal file
View file

@ -0,0 +1,153 @@
from __future__ import print_function, unicode_literals
import os
import os.path as path
import shutil
import subprocess
import sys
import tarfile
import urllib
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from servo.command_base import CommandBase, cd
def host_triple():
os_type = subprocess.check_output(["uname", "-s"]).strip().lower()
if os_type == "linux":
os_type = "unknown-linux-gnu"
elif os_type == "darwin":
os_type = "apple-darwin"
elif os_type == "android":
os_type == "linux-androideabi"
else:
os_type == "unknown"
cpu_type = subprocess.check_output(["uname", "-m"]).strip().lower()
if cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
cpu_type = "i686"
elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
cpu_type = "x86_64"
elif cpu_type == "arm":
cpu_type = "arm"
else:
cpu_type = "unknown"
return "%s-%s" % (cpu_type, os_type)
def download(desc, src, dst):
recved = [0]
def report(count, bsize, fsize):
recved[0] += bsize
pct = recved[0] * 100.0 / fsize
print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
sys.stdout.flush()
urllib.urlretrieve(src, dst, report)
print()
def extract(src, dst, movedir=None):
tarfile.open(src).extractall(dst)
if movedir:
for f in os.listdir(movedir):
frm = path.join(movedir, f)
to = path.join(dst, f)
os.rename(frm, to)
os.rmdir(movedir)
os.remove(src)
@CommandProvider
class MachCommands(CommandBase):
@Command('env',
description='Print environment setup commands',
category='bootstrap')
def env(self):
env = self.build_env()
print("export PATH=%s" % env["PATH"])
if sys.platform == "darwin":
print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
else:
print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])
@Command('bootstrap-rust',
description='Download the Rust compiler snapshot',
category='bootstrap')
@CommandArgument('--force', '-f',
action='store_true',
help='Force download even if a snapshot already exists')
def bootstrap_rustc(self, force=False):
rust_dir = path.join(self.context.topdir, "rust")
if not force and path.exists(path.join(rust_dir, "bin", "rustc")):
print("Snapshot Rust compiler already downloaded.", end=" ")
print("Use |bootstrap_rust --force| to download again.")
return
if path.isdir(rust_dir):
shutil.rmtree(rust_dir)
os.mkdir(rust_dir)
snapshot_hash = open(path.join(self.context.topdir, "rust-snapshot-hash")).read().strip()
snapshot_path = "%s-%s.tar.gz" % (snapshot_hash, host_triple())
snapshot_url = "https://servo-rust.s3.amazonaws.com/%s" % snapshot_path
tgz_file = path.join(rust_dir, path.basename(snapshot_path))
download("Rust snapshot", snapshot_url, tgz_file)
print("Extracting Rust snapshot...")
snap_dir = path.join(rust_dir,
path.basename(tgz_file).replace(".tar.gz", ""))
extract(tgz_file, rust_dir, movedir=snap_dir)
print("Snapshot Rust ready.")
@Command('bootstrap-cargo',
description='Download the Cargo build tool',
category='bootstrap')
@CommandArgument('--force', '-f',
action='store_true',
help='Force download even if cargo already exists')
def bootstrap_cargo(self, force=False):
cargo_dir = path.join(self.context.topdir, "cargo")
if not force and path.exists(path.join(cargo_dir, "bin", "cargo")):
print("Cargo already downloaded.", end=" ")
print("Use |bootstrap_cargo --force| to download again.")
return
if path.isdir(cargo_dir):
shutil.rmtree(cargo_dir)
os.mkdir(cargo_dir)
tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
nightly_url = "http://static.rust-lang.org/cargo-dist/%s" % tgz_file
download("Cargo nightly", nightly_url, tgz_file)
print("Extracting Cargo nightly...")
nightly_dir = path.join(cargo_dir,
path.basename(tgz_file).replace(".tar.gz", ""))
extract(tgz_file, cargo_dir, movedir=nightly_dir)
print("Cargo ready.")
@Command('update-submodules',
description='Update submodules',
category='bootstrap')
def update_submodules(self):
submodules = subprocess.check_output(["git", "submodule", "status"])
for line in submodules.split('\n'):
components = line.strip().split(' ')
if len(components) > 1:
module_path = components[1]
if path.exists(module_path):
with cd(module_path):
output = subprocess.check_output(["git", "status", "--porcelain"])
if len(output) != 0:
print("error: submodule %s is not clean" % module_path)
print("\nClean the submodule and try again.")
return 1
subprocess.check_call(["git", "submodule", "--quiet", "sync", "--recursive"])
subprocess.check_call(["git", "submodule", "update", "--init", "--recursive"])

85
python/servo/build_commands.py Normal file
View file

@ -0,0 +1,85 @@
from __future__ import print_function, unicode_literals
import json
import os
import os.path as path
import shutil
import subprocess
import sys
import tarfile
from time import time
import urllib
from mach.registrar import Registrar
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from servo.command_base import CommandBase, cd
@CommandProvider
class MachCommands(CommandBase):
@Command('build',
description='Build Servo',
category='build')
@CommandArgument('--target', '-t',
default=None,
help='Cross compile for given target platform')
@CommandArgument('--release', '-r',
action='store_true',
help='Build in release mode')
@CommandArgument('--jobs', '-j',
default=None,
help='Number of jobs to run in parallel')
def build(self, target, release=False, jobs=None):
self.ensure_bootstrapped()
opts = []
if release:
opts += ["--release"]
if jobs is not None:
opts += ["-j", jobs]
build_start = time()
subprocess.check_call(["cargo", "build"] + opts, env=self.build_env())
elapsed = time() - build_start
print("Build completed in %0.2fs" % elapsed)
@Command('build-cef',
description='Build the Chromium Embedding Framework library',
category='build')
@CommandArgument('--jobs', '-j',
default=None,
help='Number of jobs to run in parallel')
def build_cef(self, jobs=None):
self.ensure_bootstrapped()
ret = None
opts = []
if jobs is not None:
opts += ["-j", jobs]
build_start = time()
with cd(path.join("ports", "cef")):
ret = subprocess.call(["cargo", "build"], env=self.build_env())
elapsed = time() - build_start
print("CEF build completed in %0.2fs" % elapsed)
return ret
@Command('build-tests',
description='Build the Servo test suites',
category='build')
@CommandArgument('--jobs', '-j',
default=None,
help='Number of jobs to run in parallel')
def build_tests(self, jobs=None):
self.ensure_bootstrapped()
opts = []
if jobs is not None:
opts += ["-j", jobs]
subprocess.check_call(["cargo", "test", "--no-run"], env=self.build_env())

94
python/servo/command_base.py Normal file
View file

@ -0,0 +1,94 @@
import os
from os import path
import subprocess
import sys
import toml
from mach.registrar import Registrar
class cd:
"""Context manager for changing the current working directory"""
def __init__(self, newPath):
self.newPath = newPath
def __enter__(self):
self.savedPath = os.getcwd()
os.chdir(self.newPath)
def __exit__(self, etype, value, traceback):
os.chdir(self.savedPath)
class CommandBase(object):
"""Base class for mach command providers.
This mostly handles configuration management, such as .servobuild."""
def __init__(self, context):
self.context = context
if not hasattr(self.context, "bootstrapped"):
self.context.bootstrapped = False
config_path = path.join(context.topdir, ".servobuild")
if path.exists(config_path):
self.config = toml.loads(open(config_path).read())
else:
self.config = {}
# Handle missing/default items
self.config.setdefault("tools", {})
self.config["tools"].setdefault("system-rust", False)
self.config["tools"].setdefault("system-cargo", False)
self.config["tools"].setdefault("rust-root", "")
self.config["tools"].setdefault("cargo-root", "")
if not self.config["tools"]["system-rust"]:
self.config["tools"]["rust-root"] = path.join(context.topdir, "rust")
if not self.config["tools"]["system-cargo"]:
self.config["tools"]["cargo-root"] = path.join(context.topdir, "cargo")
def build_env(self):
"""Return an extended environment dictionary."""
env = os.environ.copy()
extra_path = []
extra_lib = []
if not self.config["tools"]["system-rust"] or self.config["tools"]["rust-root"]:
extra_path += [path.join(self.config["tools"]["rust-root"], "bin")]
extra_lib += [path.join(self.config["tools"]["rust-root"], "lib")]
if not self.config["tools"]["system-cargo"] or self.config["tools"]["cargo-root"]:
extra_path += [path.join(self.config["tools"]["cargo-root"], "bin")]
if extra_path:
env["PATH"] = "%s%s%s" % (os.pathsep.join(extra_path), os.pathsep, env["PATH"])
if extra_lib:
if sys.platform == "darwin":
env["DYLD_LIBRARY_PATH"] = "%s%s%s" % \
(os.pathsep.join(extra_lib),
os.pathsep,
env.get("DYLD_LIBRARY_PATH", ""))
else:
env["LD_LIBRARY_PATH"] = "%s%s%s" % \
(os.pathsep.join(extra_lib),
os.pathsep,
env.get("LD_LIBRARY_PATH", ""))
return env
def ensure_bootstrapped(self):
if self.context.bootstrapped: return
submodules = subprocess.check_output(["git", "submodule", "status"])
for line in submodules.split('\n'):
components = line.strip().split(' ')
if len(components) > 1 and components[0].startswith('-'):
module_path = components[1]
subprocess.check_call(["git", "submodule", "update",
"--init", "--recursive", "--", module_path])
if not self.config["tools"]["system-rust"] and \
not path.exists(path.join(self.context.topdir, "rust", "bin", "rustc")):
Registrar.dispatch("bootstrap-rust", context=self.context)
if not self.config["tools"]["system-cargo"] and \
not path.exists(path.join(self.context.topdir, "cargo", "bin", "cargo")):
Registrar.dispatch("bootstrap-cargo", context=self.context)
self.context.bootstrapped = True

32
python/servo/devenv_commands.py Normal file
View file

@ -0,0 +1,32 @@
from __future__ import print_function, unicode_literals
import json
import os
import os.path as path
import shutil
import subprocess
import sys
import tarfile
from time import time
import urllib
from mach.registrar import Registrar
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from servo.command_base import CommandBase
@CommandProvider
class MachCommands(CommandBase):
@Command('cargo',
description='Run Cargo',
category='devenv',
allow_all_args=True)
@CommandArgument('params', default=None, nargs='...',
help="Command-line arguments to be passed through to Cervo")
def run(self, params):
return subprocess.call(["cargo"] + params,
env=self.build_env())

44
python/servo/post_build_commands.py Normal file
View file

@ -0,0 +1,44 @@
from __future__ import print_function, unicode_literals
import json
import os
import os.path as path
import shutil
import subprocess
import sys
import tarfile
from time import time
import urllib
from mach.registrar import Registrar
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from servo.command_base import CommandBase
@CommandProvider
class MachCommands(CommandBase):
@Command('run',
description='Run Servo',
category='post-build',
allow_all_args=True)
@CommandArgument('params', default=None, nargs='...',
help="Command-line arguments to be passed through to Servo")
def run(self, params):
subprocess.check_call([path.join("target", "servo")] + params,
env=self.build_env())
@Command('doc',
description='Generate documentation',
category='post-build',
allow_all_args=True)
@CommandArgument('params', default=None, nargs='...',
help="Command-line arguments to be passed through to cargo doc")
def doc(self, params):
self.ensure_bootstrapped()
return subprocess.call(["cargo", "doc"] + params,
env=self.build_env())

122
python/servo/testing_commands.py Normal file
View file

@ -0,0 +1,122 @@
from __future__ import print_function, unicode_literals
import json
import os
import os.path as path
import shutil
import subprocess
import sys
import tarfile
from time import time
import urllib
from mach.registrar import Registrar
from mach.decorators import (
CommandArgument,
CommandProvider,
Command,
)
from servo.command_base import CommandBase
import tidy
@CommandProvider
class MachCommands(CommandBase):
def __init__(self, context):
CommandBase.__init__(self, context)
if not hasattr(self.context, "built_tests"):
self.context.built_tests = False
def ensure_built_tests(self):
if self.context.built_tests: return
Registrar.dispatch('build-tests', context=self.context)
self.context.built_tests = True
def find_test(self, prefix):
candidates = [f for f in os.listdir(path.join(self.context.topdir, "target"))
if f.startswith(prefix + "-")]
if candidates:
return path.join(self.context.topdir, "target", candidates[0])
return None
def run_test(self, prefix, args=[]):
t = self.find_test(prefix)
if t:
return subprocess.call([t] + args, env=self.build_env())
@Command('test',
description='Run all Servo tests',
category='testing')
def test(self):
test_start = time()
for t in ["tidy", "unit", "ref", "content", "wpt"]:
Registrar.dispatch("test-%s" % t, context=self.context)
elapsed = time() - test_start
print("Tests completed in %0.2fs" % elapsed)
@Command('test-unit',
description='Run libservo unit tests',
category='testing')
def test_unit(self):
self.ensure_bootstrapped()
self.ensure_built_tests()
return self.run_test("servo")
@Command('test-ref',
description='Run the reference tests',
category='testing')
@CommandArgument('--kind', '-k', default=None)
def test_ref(self, kind=None):
self.ensure_bootstrapped()
self.ensure_built_tests()
kinds = ["cpu", "gpu"] if kind is None else [kind]
test_path = path.join(self.context.topdir, "tests", "ref")
error = False
test_start = time()
for k in kinds:
print("Running %s reftests..." % k)
ret = self.run_test("reftest", [k, test_path])
error = error or ret != 0
elapsed = time() - test_start
print("Reference tests completed in %0.2fs" % elapsed)
if error: return 1
@Command('test-content',
description='Run the content tests',
category='testing')
def test_content(self):
self.ensure_bootstrapped()
self.ensure_built_tests()
test_path = path.join(self.context.topdir, "tests", "content")
test_start = time()
ret = self.run_test("contenttest", ["--source-dir=%s" % test_path])
elapsed = time() - test_start
print("Content tests completed in %0.2fs" % elapsed)
return ret
@Command('test-tidy',
description='Run the source code tidiness check',
category='testing')
def test_tidy(self):
errors = 0
for p in ["src", "components"]:
ret = tidy.scan(path.join(self.context.topdir, p))
if ret != 0: errors = 1
return errors
@Command('test-wpt',
description='Run the web platform tests',
category='testing',
allow_all_args=True)
@CommandArgument('params', default=None, nargs='...',
help="Command-line arguments to be passed through to wpt/run.sh")
def test_wpt(self, params):
return subprocess.call(["bash", path.join("tests", "wpt", "run.sh")] + params,
env=self.build_env())

91
python/tidy.py Normal file
View file

@ -0,0 +1,91 @@
# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
#!/usr/bin/env python
import os
from licenseck import check_license
# FIXME(#3242): Don't use globals
err = 0
def report_error_name_no(name, no, s):
global err
print("%s:%d: %s" % (name, no, s))
err = 1
def do_license_check(name, contents):
if not check_license(name, contents):
report_error_name_no(name, 1, "incorrect license")
def do_whitespace_check(name, contents):
for idx, line in enumerate(contents):
if line[-1] == "\n":
line = line[:-1]
else:
report_error_name_no(name, idx + 1, "No newline at EOF")
if line.endswith(' '):
report_error_name_no(name, idx + 1, "trailing whitespace")
if '\t' in line:
report_error_name_no(name, idx + 1, "tab on line")
if '\r' in line:
report_error_name_no(name, idx + 1, "CR on line")
exceptions = [
# Upstream
"support",
"tests/wpt/web-platform-tests",
# Generated and upstream code combined with our own. Could use cleanup
"components/script/dom/bindings/codegen",
"components/style/properties/mod.rs",
]
def should_check(name):
if ".#" in name:
return False
if not (name.endswith(".rs")
or name.endswith(".rc")
or name.endswith(".cpp")
or name.endswith(".c")
or name.endswith(".h")
or name.endswith(".py")):
return False
for exception in exceptions:
if exception in name:
return False
return True
def scan(start_path):
global err
err = 0
file_names = []
for root, dirs, files in os.walk(start_path):
for myfile in files:
file_name = root + "/" + myfile
if should_check(file_name):
file_names.append(file_name)
for path in file_names:
with open(path, "r") as fp:
lines = fp.readlines()
do_license_check(path, "".join(lines))
do_whitespace_check(path, lines)
return err

21
python/toml/LICENSE Normal file
View file

@ -0,0 +1,21 @@
The MIT License
Copyright 2013 Uiri Noyb
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

52
python/toml/PKG-INFO Normal file
View file

@ -0,0 +1,52 @@
Metadata-Version: 1.0
Name: toml
Version: 0.8.2
Summary: Python Library for Tom's Obvious, Minimal Language
Home-page: https://github.com/uiri/toml
Author: Uiri Noyb
Author-email: uiri@xqz.ca
License: License :: OSI Approved :: MIT License
Description: TOML
====
Original repository: https://github.com/uiri/toml
See also https://github.com/mojombo/toml
Python module which parses and emits TOML.
Released under the MIT license.
Passes https://github.com/BurntSushi/toml-test
See http://j.xqz.ca/toml-status for up to date test results.
Current Version of the Specification
------------------------------------
https://github.com/mojombo/toml/blob/v0.2.0/README.md
QUICK GUIDE
-----------
``pip install toml``
toml.loads --- takes a string to be parsed as toml and returns the corresponding dictionary
toml.dumps --- takes a dictionary and returns a string which is the contents of the corresponding toml file.
There are other functions which I use to dump and load various fragments of toml but dumps and loads will cover most usage.
Example usage:
.. code:: python
import toml
with open("conf.toml") as conffile:
config = toml.loads(conffile.read())
# do stuff with config here
. . .
Platform: UNKNOWN

42
python/toml/README.rst Normal file
View file

@ -0,0 +1,42 @@
TOML
====
Original repository: https://github.com/uiri/toml
See also https://github.com/mojombo/toml
Python module which parses and emits TOML.
Released under the MIT license.
Passes https://github.com/BurntSushi/toml-test
See http://j.xqz.ca/toml-status for up to date test results.
Current Version of the Specification
------------------------------------
https://github.com/mojombo/toml/blob/v0.2.0/README.md
QUICK GUIDE
-----------
``pip install toml``
toml.loads --- takes a string to be parsed as toml and returns the corresponding dictionary
toml.dumps --- takes a dictionary and returns a string which is the contents of the corresponding toml file.
There are other functions which I use to dump and load various fragments of toml but dumps and loads will cover most usage.
Example usage:
.. code:: python
import toml
with open("conf.toml") as conffile:
config = toml.loads(conffile.read())
# do stuff with config here
. . .

14
python/toml/setup.py Normal file
View file

@ -0,0 +1,14 @@
from distutils.core import setup
with open("README.rst") as readmefile:
readme = readmefile.read()
setup(name='toml',
version='0.8.2',
description="Python Library for Tom's Obvious, Minimal Language",
author="Uiri Noyb",
author_email="uiri@xqz.ca",
url="https://github.com/uiri/toml",
py_modules=['toml'],
license="License :: OSI Approved :: MIT License",
long_description=readme,
)

443
python/toml/toml.py Normal file
View file

@ -0,0 +1,443 @@
import datetime, decimal
try:
_range = xrange
except NameError:
unicode = str
_range = range
basestring = str
unichr = chr
def load(f):
"""Returns a dictionary containing the named file parsed as toml."""
if isinstance(f, basestring):
with open(f) as ffile:
return loads(ffile.read())
elif isinstance(f, list):
for l in f:
if not isinstance(l, basestring):
raise Exception("Load expects a list to contain filenames only")
d = []
for l in f:
d.append(load(l))
r = {}
for l in d:
toml_merge_dict(r, l)
return r
elif f.read:
return loads(f.read())
else:
raise Exception("You can only load a file descriptor, filename or list")
def loads(s):
"""Returns a dictionary containing s, a string, parsed as toml."""
implicitgroups = []
retval = {}
currentlevel = retval
if isinstance(s, basestring):
try:
s.decode('utf8')
except AttributeError:
pass
sl = list(s)
openarr = 0
openstring = False
arrayoftables = True
beginline = True
keygroup = False
delnum = 1
for i in range(len(sl)):
if sl[i] == '"':
oddbackslash = False
try:
k = 1
j = sl[i-k]
oddbackslash = False
while j == '\\':
oddbackslash = not oddbackslash
k += 1
j = sl[i-k]
except IndexError:
pass
if not oddbackslash:
openstring = not openstring
if keygroup and (sl[i] == ' ' or sl[i] == '\t'):
keygroup = False
if arrayoftables and (sl[i] == ' ' or sl[i] == '\t'):
arrayoftables = False
if sl[i] == '#' and not openstring and not keygroup and not arrayoftables:
j = i
while sl[j] != '\n':
sl.insert(j, ' ')
sl.pop(j+1)
j += 1
if sl[i] == '[' and not openstring and not keygroup and not arrayoftables:
if beginline:
if sl[i+1] == '[':
arrayoftables = True
else:
keygroup = True
else:
openarr += 1
if sl[i] == ']' and not openstring and not keygroup and not arrayoftables:
if keygroup:
keygroup = False
elif arrayoftables:
if sl[i-1] == ']':
arrayoftables = False
else:
openarr -= 1
if sl[i] == '\n':
if openstring:
raise Exception("Unbalanced quotes")
if openarr:
sl.insert(i, ' ')
sl.pop(i+1)
else:
beginline = True
elif beginline and sl[i] != ' ' and sl[i] != '\t':
beginline = False
keygroup = True
s = ''.join(sl)
s = s.split('\n')
else:
raise Exception("What exactly are you trying to pull?")
for line in s:
line = line.strip()
if line == "":
continue
if line[0] == '[':
arrayoftables = False
if line[1] == '[':
arrayoftables = True
line = line[2:].split(']]', 1)
else:
line = line[1:].split(']', 1)
if line[1].strip() != "":
raise Exception("Key group not on a line by itself.")
line = line[0]
if '[' in line:
raise Exception("Key group name cannot contain '['")
if ']' in line:
raise Exception("Key group name cannot contain']'")
groups = line.split('.')
currentlevel = retval
for i in range(len(groups)):
group = groups[i]
if group == "":
raise Exception("Can't have a keygroup with an empty name")
try:
currentlevel[group]
if i == len(groups) - 1:
if group in implicitgroups:
implicitgroups.remove(group)
if arrayoftables:
raise Exception("An implicitly defined table can't be an array")
elif arrayoftables:
currentlevel[group].append({})
else:
raise Exception("What? "+group+" already exists?"+str(currentlevel))
except TypeError:
if i != len(groups) - 1:
implicitgroups.append(group)
currentlevel = currentlevel[0]
if arrayoftables:
currentlevel[group] = [{}]
else:
currentlevel[group] = {}
except KeyError:
if i != len(groups) - 1:
implicitgroups.append(group)
currentlevel[group] = {}
if i == len(groups) - 1 and arrayoftables:
currentlevel[group] = [{}]
currentlevel = currentlevel[group]
if arrayoftables:
try:
currentlevel = currentlevel[-1]
except KeyError:
pass
elif "=" in line:
i = 1
pair = line.split('=', i)
l = len(line)
while pair[-1][0] != ' ' and pair[-1][0] != '\t' and pair[-1][0] != '"' and pair[-1][0] != '[' and pair[-1] != 'true' and pair[-1] != 'false':
try:
float(pair[-1])
break
except ValueError:
try:
datetime.datetime.strptime(pair[-1], "%Y-%m-%dT%H:%M:%SZ")
break
except ValueError:
i += 1
pair = line.split('=', i)
newpair = []
newpair.append('='.join(pair[:-1]))
newpair.append(pair[-1])
pair = newpair
pair[0] = pair[0].strip()
pair[1] = pair[1].strip()
value, vtype = load_value(pair[1])
try:
currentlevel[pair[0]]
raise Exception("Duplicate keys!")
except KeyError:
currentlevel[pair[0]] = value
return retval
def load_value(v):
if v == 'true':
return (True, "bool")
elif v == 'false':
return (False, "bool")
elif v[0] == '"':
testv = v[1:].split('"')
closed = False
for tv in testv:
if tv == '':
closed = True
else:
oddbackslash = False
try:
i = -1
j = tv[i]
while j == '\\':
oddbackslash = not oddbackslash
i -= 1
j = tv[i]
except IndexError:
pass
if not oddbackslash:
if closed:
raise Exception("Stuff after closed string. WTF?")
else:
closed = True
escapes = ['0', 'b', 'f', '/', 'n', 'r', 't', '"', '\\']
escapedchars = ['\0', '\b', '\f', '/', '\n', '\r', '\t', '\"', '\\']
escapeseqs = v.split('\\')[1:]
backslash = False
for i in escapeseqs:
if i == '':
backslash = not backslash
else:
if i[0] not in escapes and i[0] != 'u' and not backslash:
raise Exception("Reserved escape sequence used")
if backslash:
backslash = False
if "\\u" in v:
hexchars = ['0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', 'a', 'b', 'c', 'd', 'e', 'f']
hexbytes = v.split('\\u')
newv = hexbytes[0]
hexbytes = hexbytes[1:]
for hx in hexbytes:
hxb = ""
try:
if hx[0].lower() in hexchars:
hxb += hx[0].lower()
if hx[1].lower() in hexchars:
hxb += hx[1].lower()
if hx[2].lower() in hexchars:
hxb += hx[2].lower()
if hx[3].lower() in hexchars:
hxb += hx[3].lower()
except IndexError:
if len(hxb) != 2:
raise Exception("Invalid escape sequence")
if len(hxb) != 4 and len(hxb) != 2:
raise Exception("Invalid escape sequence")
newv += unichr(int(hxb, 16))
newv += unicode(hx[len(hxb):])
v = newv
for i in range(len(escapes)):
v = v.replace("\\"+escapes[i], escapedchars[i])
# (where (n) signifies a member of escapes:
# undo (\\)(\\)(n) -> (\\)(\n)
v = v.replace("\\"+escapedchars[i], "\\\\"+escapes[i])
return (v[1:-1], "str")
elif v[0] == '[':
return (load_array(v), "array")
elif len(v) == 20 and v[-1] == 'Z':
if v[10] == 'T':
return (datetime.datetime.strptime(v, "%Y-%m-%dT%H:%M:%SZ"), "date")
else:
raise Exception("Wait, what?")
else:
itype = "int"
digits = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
neg = False
if v[0] == '-':
neg = True
v = v[1:]
if '.' in v:
if v.split('.', 1)[1] == '':
raise Exception("This float is missing digits after the point")
if v[0] not in digits:
raise Exception("This float doesn't have a leading digit")
v = float(v)
itype = "float"
else:
v = int(v)
if neg:
return (0 - v, itype)
return (v, itype)
def load_array(a):
atype = None
retval = []
a = a.strip()
if '[' not in a[1:-1]:
strarray = False
tmpa = a[1:-1].strip()
if tmpa != '' and tmpa[0] == '"':
strarray = True
a = a[1:-1].split(',')
b = 0
if strarray:
while b < len(a) - 1:
while a[b].strip()[-1] != '"' and a[b+1].strip()[0] != '"':
a[b] = a[b] + ',' + a[b+1]
if b < len(a) - 2:
a = a[:b+1] + a[b+2:]
else:
a = a[:b+1]
b += 1
else:
al = list(a[1:-1])
a = []
openarr = 0
j = 0
for i in range(len(al)):
if al[i] == '[':
openarr += 1
elif al[i] == ']':
openarr -= 1
elif al[i] == ',' and not openarr:
a.append(''.join(al[j:i]))
j = i+1
a.append(''.join(al[j:]))
for i in range(len(a)):
a[i] = a[i].strip()
if a[i] != '':
nval, ntype = load_value(a[i])
if atype:
if ntype != atype:
raise Exception("Not a homogeneous array")
else:
atype = ntype
retval.append(nval)
return retval
def dump(o, f):
"""Writes out to f the toml corresponding to o. Returns said toml."""
if f.write:
d = dumps(o)
f.write(d)
return d
else:
raise Exception("You can only dump an object to a file descriptor")
def dumps(o):
"""Returns a string containing the toml corresponding to o, a dictionary"""
retval = ""
addtoretval, sections = dump_sections(o, "")
retval += addtoretval
while sections != {}:
newsections = {}
for section in sections:
addtoretval, addtosections = dump_sections(sections[section], section)
if addtoretval:
retval += "["+section+"]\n"
retval += addtoretval
for s in addtosections:
newsections[section+"."+s] = addtosections[s]
sections = newsections
return retval
def dump_sections(o, sup):
retstr = ""
if sup != "" and sup[-1] != ".":
sup += '.'
retdict = {}
arraystr = ""
for section in o:
if not isinstance(o[section], dict):
arrayoftables = False
if isinstance(o[section], list):
for a in o[section]:
if isinstance(a, dict):
arrayoftables = True
if arrayoftables:
for a in o[section]:
arraytabstr = ""
arraystr += "[["+sup+section+"]]\n"
s, d = dump_sections(a, sup+section)
if s:
if s[0] == "[":
arraytabstr += s
else:
arraystr += s
while d != {}:
newd = {}
for dsec in d:
s1, d1 = dump_sections(d[dsec], sup+section+dsec)
if s1:
arraytabstr += "["+sup+section+"."+dsec+"]\n"
arraytabstr += s1
for s1 in d1:
newd[dsec+"."+s1] = d1[s1]
d = newd
arraystr += arraytabstr
else:
retstr += section + " = " + str(dump_value(o[section])) + '\n'
else:
retdict[section] = o[section]
retstr += arraystr
return (retstr, retdict)
def dump_value(v):
if isinstance(v, list):
t = []
retval = "["
for u in v:
t.append(dump_value(u))
while t != []:
s = []
for u in t:
if isinstance(u, list):
for r in u:
s.append(r)
else:
retval += " " + str(u) + ","
t = s
retval += "]"
return retval
if isinstance(v, (str, unicode)):
escapes = ['\\', '0', 'b', 'f', '/', 'n', 'r', 't', '"']
escapedchars = ['\\', '\0', '\b', '\f', '/', '\n', '\r', '\t', '\"']
for i in range(len(escapes)):
v = v.replace(escapedchars[i], "\\"+escapes[i])
return str('"'+v+'"')
if isinstance(v, bool):
return str(v).lower()
if isinstance(v, datetime.datetime):
return v.isoformat()[:19]+'Z'
if isinstance(v, float):
return '{0:f}'.format(decimal.Decimal(str(v)))
return v
def toml_merge_dict(a, b):
for k in a:
if isinstance(a[k], dict):
try:
b[k]
except KeyError:
continue
if isinstance(b[k], dict):
b[k] = toml_merge_dict(a[k], b[k])
else:
raise Exception("Can't merge dict and nondict in toml object")
a.update(b)
return a