diff --git a/.gitignore b/.gitignore index 860c9e5073a..a21c4e30a17 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ /ports/android/libs /ports/android/local.properties /ports/android/obj -/python/_virtualenv +/tests/wpt/_virtualenv *~ *# *.o diff --git a/python/dependencies/flake8-2.4.1-py2.py3-none-any.whl b/python/dependencies/flake8-2.4.1-py2.py3-none-any.whl new file mode 100644 index 00000000000..2aead94cbdf Binary files /dev/null and b/python/dependencies/flake8-2.4.1-py2.py3-none-any.whl differ diff --git a/python/dependencies/pep8-1.5.7-py2.py3-none-any.whl b/python/dependencies/pep8-1.5.7-py2.py3-none-any.whl new file mode 100644 index 00000000000..7ba4e74bcc2 Binary files /dev/null and b/python/dependencies/pep8-1.5.7-py2.py3-none-any.whl differ diff --git a/python/dependencies/pyflakes-0.9.0-py2.py3-none-any.whl b/python/dependencies/pyflakes-0.9.0-py2.py3-none-any.whl new file mode 100644 index 00000000000..90edba90e44 Binary files /dev/null and b/python/dependencies/pyflakes-0.9.0-py2.py3-none-any.whl differ diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index e04d1781bdd..84de5b22b25 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -6,9 +6,7 @@ from __future__ import print_function, unicode_literals import os import platform -import subprocess import sys -from distutils.spawn import find_executable SEARCH_PATHS = [ "python/mach", @@ -75,39 +73,6 @@ CATEGORIES = { } -def _get_exec(name, default=None): - path = find_executable(name) - if not path: - return default - return path - - -def _activate_virtualenv(topdir): - virtualenv_path = os.path.join(topdir, "python", "_virtualenv") - python = _get_exec("python2", "python") - - if not os.path.exists(virtualenv_path): - virtualenv = _get_exec("virtualenv2", "virtualenv") - subprocess.check_call([virtualenv, "-p", python, virtualenv_path]) - - activate_path = os.path.join(virtualenv_path, "bin", "activate_this.py") - execfile(activate_path, dict(__file__=activate_path)) - - # TODO: Right now, we iteratively install all the requirements by invoking - # `pip install` each time. If it were the case that there were conflicting - # requirements, we wouldn't know about them. Once - # https://github.com/pypa/pip/issues/988 is addressed, then we can just - # chain each of the requirements files into the same `pip install` call - # and it will check for conflicts. - requirements_paths = [ - os.path.join(topdir, "python", "requirements.txt"), - os.path.join(topdir, "tests", "wpt", "harness", "requirements.txt"), - os.path.join(topdir, "tests", "wpt", "harness", "requirements_servo.txt"), - ] - for path in requirements_paths: - subprocess.check_call(["pip", "install", "-q", "-r", path]) - - def bootstrap(topdir): topdir = os.path.abspath(topdir) @@ -119,8 +84,6 @@ def bootstrap(topdir): print('You are running Python', platform.python_version()) sys.exit(1) - _activate_virtualenv(topdir) - def populate_context(context, key=None): if key is None: return diff --git a/python/mozdebug/__init__.py b/python/mozdebug/__init__.py new file mode 100644 index 00000000000..54d5b4d5de8 --- /dev/null +++ b/python/mozdebug/__init__.py @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +This module contains a set of function to gather information about the +debugging capabilities of the platform. It allows to look for a specific +debugger or to query the system for a compatible/default debugger. + +The following simple example looks for the default debugger on the +current platform and launches a debugger process with the correct +debugger-specific arguments: + +:: + + import mozdebug + + debugger = mozdebug.get_default_debugger_name() + debuggerInfo = mozdebug.get_debugger_info(debugger) + + debuggeePath = "toDebug" + + processArgs = [self.debuggerInfo.path] + self.debuggerInfo.args + processArgs.append(debuggeePath) + + run_process(args, ...) + +""" + +from mozdebug import * diff --git a/python/mozdebug/mozdebug.py b/python/mozdebug/mozdebug.py new file mode 100644 index 00000000000..3f3a54d6be4 --- /dev/null +++ b/python/mozdebug/mozdebug.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import mozinfo +from collections import namedtuple +from distutils.spawn import find_executable + +__all__ = ['get_debugger_info', + 'get_default_debugger_name', + 'DebuggerSearch'] + +''' +Map of debugging programs to information about them, like default arguments +and whether or not they are interactive. + +To add support for a new debugger, simply add the relative entry in +_DEBUGGER_INFO and optionally update the _DEBUGGER_PRIORITIES. +''' +_DEBUGGER_INFO = { + # gdb requires that you supply the '--args' flag in order to pass arguments + # after the executable name to the executable. + 'gdb': { + 'interactive': True, + 'args': ['-q', '--args'] + }, + + 'cgdb': { + 'interactive': True, + 'args': ['-q', '--args'] + }, + + 'lldb': { + 'interactive': True, + 'args': ['--'], + 'requiresEscapedArgs': True + }, + + # Visual Studio Debugger Support. + 'devenv.exe': { + 'interactive': True, + 'args': ['-debugexe'] + }, + + # Visual C++ Express Debugger Support. + 'wdexpress.exe': { + 'interactive': True, + 'args': ['-debugexe'] + }, + + # valgrind doesn't explain much about leaks unless you set the + # '--leak-check=full' flag. But there are a lot of objects that are + # semi-deliberately leaked, so we set '--show-possibly-lost=no' to avoid + # uninteresting output from those objects. We set '--smc-check==all-non-file' + # and '--vex-iropt-register-updates=allregs-at-mem-access' so that valgrind + # deals properly with JIT'd JavaScript code. + 'valgrind': { + 'interactive': False, + 'args': ['--leak-check=full', + '--show-possibly-lost=no', + '--smc-check=all-non-file', + '--vex-iropt-register-updates=allregs-at-mem-access'] + } +} + +# Maps each OS platform to the preferred debugger programs found in _DEBUGGER_INFO. +_DEBUGGER_PRIORITIES = { + 'win': ['devenv.exe', 'wdexpress.exe'], + 'linux': ['gdb', 'cgdb', 'lldb'], + 'mac': ['lldb', 'gdb'], + 'unknown': ['gdb'] +} + +def get_debugger_info(debugger, debuggerArgs = None, debuggerInteractive = False): + ''' + Get the information about the requested debugger. + + Returns a dictionary containing the |path| of the debugger executable, + if it will run in |interactive| mode, its arguments and whether it needs + to escape arguments it passes to the debugged program (|requiresEscapedArgs|). + If the debugger cannot be found in the system, returns |None|. + + :param debugger: The name of the debugger. + :param debuggerArgs: If specified, it's the arguments to pass to the debugger, + as a string. Any debugger-specific separator arguments are appended after these + arguments. + :param debuggerInteractive: If specified, forces the debugger to be interactive. + ''' + + debuggerPath = None + + if debugger: + # Append '.exe' to the debugger on Windows if it's not present, + # so things like '--debugger=devenv' work. + if (os.name == 'nt' + and not debugger.lower().endswith('.exe')): + debugger += '.exe' + + debuggerPath = find_executable(debugger) + + if not debuggerPath: + print 'Error: Could not find debugger %s.' % debugger + return None + + debuggerName = os.path.basename(debuggerPath).lower() + + def get_debugger_info(type, default): + if debuggerName in _DEBUGGER_INFO and type in _DEBUGGER_INFO[debuggerName]: + return _DEBUGGER_INFO[debuggerName][type] + return default + + # Define a namedtuple to access the debugger information from the outside world. + DebuggerInfo = namedtuple( + 'DebuggerInfo', + ['path', 'interactive', 'args', 'requiresEscapedArgs'] + ) + + debugger_arguments = [] + + if debuggerArgs: + # Append the provided debugger arguments at the end of the arguments list. + debugger_arguments += debuggerArgs.split() + + debugger_arguments += get_debugger_info('args', []) + + # Override the default debugger interactive mode if needed. + debugger_interactive = get_debugger_info('interactive', False) + if debuggerInteractive: + debugger_interactive = debuggerInteractive + + d = DebuggerInfo( + debuggerPath, + debugger_interactive, + debugger_arguments, + get_debugger_info('requiresEscapedArgs', False) + ) + + return d + +# Defines the search policies to use in get_default_debugger_name. +class DebuggerSearch: + OnlyFirst = 1 + KeepLooking = 2 + +def get_default_debugger_name(search=DebuggerSearch.OnlyFirst): + ''' + Get the debugger name for the default debugger on current platform. + + :param search: If specified, stops looking for the debugger if the + default one is not found (|DebuggerSearch.OnlyFirst|) or keeps + looking for other compatible debuggers (|DebuggerSearch.KeepLooking|). + ''' + + # Find out which debuggers are preferred for use on this platform. + debuggerPriorities = _DEBUGGER_PRIORITIES[mozinfo.os if mozinfo.os in _DEBUGGER_PRIORITIES else 'unknown'] + + # Finally get the debugger information. + for debuggerName in debuggerPriorities: + debuggerPath = find_executable(debuggerName) + if debuggerPath: + return debuggerName + elif not search == DebuggerSearch.KeepLooking: + return None + + return None diff --git a/python/mozinfo/__init__.py b/python/mozinfo/__init__.py new file mode 100644 index 00000000000..904dfef71a7 --- /dev/null +++ b/python/mozinfo/__init__.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +interface to transform introspected system information to a format palatable to +Mozilla + +Module variables: + +.. attribute:: bits + + 32 or 64 + +.. attribute:: isBsd + + Returns ``True`` if the operating system is BSD + +.. attribute:: isLinux + + Returns ``True`` if the operating system is Linux + +.. attribute:: isMac + + Returns ``True`` if the operating system is Mac + +.. attribute:: isWin + + Returns ``True`` if the operating system is Windows + +.. attribute:: os + + Operating system [``'win'``, ``'mac'``, ``'linux'``, ...] + +.. attribute:: processor + + Processor architecture [``'x86'``, ``'x86_64'``, ``'ppc'``, ...] + +.. attribute:: version + + Operating system version string. For windows, the service pack information is also included + +.. attribute:: info + + Returns information identifying the current system. + + * :attr:`bits` + * :attr:`os` + * :attr:`processor` + * :attr:`version` + +""" + +import mozinfo +from mozinfo import * +__all__ = mozinfo.__all__ diff --git a/python/mozinfo/mozinfo.py b/python/mozinfo/mozinfo.py new file mode 100755 index 00000000000..96847fea01f --- /dev/null +++ b/python/mozinfo/mozinfo.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# TODO: it might be a good idea of adding a system name (e.g. 'Ubuntu' for +# linux) to the information; I certainly wouldn't want anyone parsing this +# information and having behaviour depend on it + +import os +import platform +import re +import sys + +# keep a copy of the os module since updating globals overrides this +_os = os + +class unknown(object): + """marker class for unknown information""" + def __nonzero__(self): + return False + def __str__(self): + return 'UNKNOWN' +unknown = unknown() # singleton + +# get system information +info = {'os': unknown, + 'processor': unknown, + 'version': unknown, + 'os_version': unknown, + 'bits': unknown, + 'has_sandbox': unknown } +(system, node, release, version, machine, processor) = platform.uname() +(bits, linkage) = platform.architecture() + +# get os information and related data +if system in ["Microsoft", "Windows"]: + info['os'] = 'win' + # There is a Python bug on Windows to determine platform values + # http://bugs.python.org/issue7860 + if "PROCESSOR_ARCHITEW6432" in os.environ: + processor = os.environ.get("PROCESSOR_ARCHITEW6432", processor) + else: + processor = os.environ.get('PROCESSOR_ARCHITECTURE', processor) + system = os.environ.get("OS", system).replace('_', ' ') + (major, minor, _, _, service_pack) = os.sys.getwindowsversion() + info['service_pack'] = service_pack + os_version = "%d.%d" % (major, minor) +elif system == "Linux": + if hasattr(platform, "linux_distribution"): + (distro, os_version, codename) = platform.linux_distribution() + else: + (distro, os_version, codename) = platform.dist() + if not processor: + processor = machine + version = "%s %s" % (distro, os_version) + info['os'] = 'linux' + info['linux_distro'] = distro +elif system in ['DragonFly', 'FreeBSD', 'NetBSD', 'OpenBSD']: + info['os'] = 'bsd' + version = os_version = sys.platform +elif system == "Darwin": + (release, versioninfo, machine) = platform.mac_ver() + version = "OS X %s" % release + versionNums = release.split('.')[:2] + os_version = "%s.%s" % (versionNums[0], versionNums[1]) + info['os'] = 'mac' +elif sys.platform in ('solaris', 'sunos5'): + info['os'] = 'unix' + os_version = version = sys.platform +else: + os_version = version = unknown + +info['version'] = version +info['os_version'] = os_version + +# processor type and bits +if processor in ["i386", "i686"]: + if bits == "32bit": + processor = "x86" + elif bits == "64bit": + processor = "x86_64" +elif processor.upper() == "AMD64": + bits = "64bit" + processor = "x86_64" +elif processor == "Power Macintosh": + processor = "ppc" +bits = re.search('(\d+)bit', bits).group(1) +info.update({'processor': processor, + 'bits': int(bits), + }) + +if info['os'] == 'linux': + import ctypes + import errno + PR_SET_SECCOMP = 22 + SECCOMP_MODE_FILTER = 2 + ctypes.CDLL("libc.so.6", use_errno=True).prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0) + info['has_sandbox'] = ctypes.get_errno() == errno.EFAULT +else: + info['has_sandbox'] = True + +# standard value of choices, for easy inspection +choices = {'os': ['linux', 'bsd', 'win', 'mac', 'unix'], + 'bits': [32, 64], + 'processor': ['x86', 'x86_64', 'ppc']} + + +def sanitize(info): + """Do some sanitization of input values, primarily + to handle universal Mac builds.""" + if "processor" in info and info["processor"] == "universal-x86-x86_64": + # If we're running on OS X 10.6 or newer, assume 64-bit + if release[:4] >= "10.6": # Note this is a string comparison + info["processor"] = "x86_64" + info["bits"] = 64 + else: + info["processor"] = "x86" + info["bits"] = 32 + +# method for updating information +def update(new_info): + """ + Update the info. + + :param new_info: Either a dict containing the new info or a path/url + to a json file containing the new info. + """ + + if isinstance(new_info, basestring): + # lazy import + import mozfile + import json + f = mozfile.load(new_info) + new_info = json.loads(f.read()) + f.close() + + info.update(new_info) + sanitize(info) + globals().update(info) + + # convenience data for os access + for os_name in choices['os']: + globals()['is' + os_name.title()] = info['os'] == os_name + # unix is special + if isLinux or isBsd: + globals()['isUnix'] = True + +def find_and_update_from_json(*dirs): + """ + Find a mozinfo.json file, load it, and update the info with the + contents. + + :param dirs: Directories in which to look for the file. They will be + searched after first looking in the root of the objdir + if the current script is being run from a Mozilla objdir. + + Returns the full path to mozinfo.json if it was found, or None otherwise. + """ + # First, see if we're in an objdir + try: + from mozbuild.base import MozbuildObject, BuildEnvironmentNotFoundException + build = MozbuildObject.from_environment() + json_path = _os.path.join(build.topobjdir, "mozinfo.json") + if _os.path.isfile(json_path): + update(json_path) + return json_path + except ImportError: + pass + except BuildEnvironmentNotFoundException: + pass + + for d in dirs: + d = _os.path.abspath(d) + json_path = _os.path.join(d, "mozinfo.json") + if _os.path.isfile(json_path): + update(json_path) + return json_path + + return None + +update({}) + +# exports +__all__ = info.keys() +__all__ += ['is' + os_name.title() for os_name in choices['os']] +__all__ += [ + 'info', + 'unknown', + 'main', + 'choices', + 'update', + 'find_and_update_from_json', + ] + +def main(args=None): + + # parse the command line + from optparse import OptionParser + parser = OptionParser(description=__doc__) + for key in choices: + parser.add_option('--%s' % key, dest=key, + action='store_true', default=False, + help="display choices for %s" % key) + options, args = parser.parse_args() + + # args are JSON blobs to override info + if args: + # lazy import + import json + for arg in args: + if _os.path.exists(arg): + string = file(arg).read() + else: + string = arg + update(json.loads(string)) + + # print out choices if requested + flag = False + for key, value in options.__dict__.items(): + if value is True: + print '%s choices: %s' % (key, ' '.join([str(choice) + for choice in choices[key]])) + flag = True + if flag: return + + # otherwise, print out all info + for key, value in info.items(): + print '%s: %s' % (key, value) + +if __name__ == '__main__': + main() diff --git a/python/mozlog/mozlog/__init__.py b/python/mozlog/mozlog/__init__.py new file mode 100644 index 00000000000..bfa23eae645 --- /dev/null +++ b/python/mozlog/mozlog/__init__.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Mozlog aims to standardize log formatting within Mozilla. + +It simply wraps Python's logging_ module and adds a few convenience methods +for logging test results and events. + +The structured submodule takes a different approach and implements a +JSON-based logging protocol designed for recording test results.""" + +from logger import * +from loglistener import LogMessageServer +from loggingmixin import LoggingMixin + +try: + import structured +except ImportError: + # Structured logging doesn't work on python 2.6 which is still used on some + # legacy test machines; https://bugzilla.mozilla.org/show_bug.cgi?id=864866 + # Once we move away from Python 2.6, please cleanup devicemanager.py's + # exception block + pass + diff --git a/python/mozlog/mozlog/logger.py b/python/mozlog/mozlog/logger.py new file mode 100644 index 00000000000..60bd4912fb3 --- /dev/null +++ b/python/mozlog/mozlog/logger.py @@ -0,0 +1,180 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from logging import getLogger as getSysLogger +from logging import * +# Some of the build slave environments don't see the following when doing +# 'from logging import *' +# see https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c35 +from logging import getLoggerClass, addLevelName, setLoggerClass, shutdown, debug, info, basicConfig +import json + +_default_level = INFO +_LoggerClass = getLoggerClass() + +# Define mozlog specific log levels +START = _default_level + 1 +END = _default_level + 2 +PASS = _default_level + 3 +KNOWN_FAIL = _default_level + 4 +FAIL = _default_level + 5 +CRASH = _default_level + 6 +# Define associated text of log levels +addLevelName(START, 'TEST-START') +addLevelName(END, 'TEST-END') +addLevelName(PASS, 'TEST-PASS') +addLevelName(KNOWN_FAIL, 'TEST-KNOWN-FAIL') +addLevelName(FAIL, 'TEST-UNEXPECTED-FAIL') +addLevelName(CRASH, 'PROCESS-CRASH') + +class MozLogger(_LoggerClass): + """ + MozLogger class which adds some convenience log levels + related to automated testing in Mozilla and ability to + output structured log messages. + """ + def testStart(self, message, *args, **kwargs): + """Logs a test start message""" + self.log(START, message, *args, **kwargs) + + def testEnd(self, message, *args, **kwargs): + """Logs a test end message""" + self.log(END, message, *args, **kwargs) + + def testPass(self, message, *args, **kwargs): + """Logs a test pass message""" + self.log(PASS, message, *args, **kwargs) + + def testFail(self, message, *args, **kwargs): + """Logs a test fail message""" + self.log(FAIL, message, *args, **kwargs) + + def testKnownFail(self, message, *args, **kwargs): + """Logs a test known fail message""" + self.log(KNOWN_FAIL, message, *args, **kwargs) + + def processCrash(self, message, *args, **kwargs): + """Logs a process crash message""" + self.log(CRASH, message, *args, **kwargs) + + def log_structured(self, action, params=None): + """Logs a structured message object.""" + if params is None: + params = {} + + level = params.get('_level', _default_level) + if isinstance(level, int): + params['_level'] = getLevelName(level) + else: + params['_level'] = level + level = getLevelName(level.upper()) + + # If the logger is fed a level number unknown to the logging + # module, getLevelName will return a string. Unfortunately, + # the logging module will raise a type error elsewhere if + # the level is not an integer. + if not isinstance(level, int): + level = _default_level + + params['action'] = action + + # The can message be None. This is expected, and shouldn't cause + # unstructured formatters to fail. + message = params.get('_message') + + self.log(level, message, extra={'params': params}) + +class JSONFormatter(Formatter): + """Log formatter for emitting structured JSON entries.""" + + def format(self, record): + # Default values determined by logger metadata + output = { + '_time': int(round(record.created * 1000, 0)), + '_namespace': record.name, + '_level': getLevelName(record.levelno), + } + + # If this message was created by a call to log_structured, + # anything specified by the caller's params should act as + # an override. + output.update(getattr(record, 'params', {})) + + if record.msg and output.get('_message') is None: + # For compatibility with callers using the printf like + # API exposed by python logging, call the default formatter. + output['_message'] = Formatter.format(self, record) + + return json.dumps(output, indent=output.get('indent')) + +class MozFormatter(Formatter): + """ + MozFormatter class used to standardize formatting + If a different format is desired, this can be explicitly + overriden with the log handler's setFormatter() method + """ + level_length = 0 + max_level_length = len('TEST-START') + + def __init__(self, include_timestamp=False): + """ + Formatter.__init__ has fmt and datefmt parameters that won't have + any affect on a MozFormatter instance. + + :param include_timestamp: if True, include formatted time at the + beginning of the message + """ + self.include_timestamp = include_timestamp + Formatter.__init__(self) + + def format(self, record): + # Handles padding so record levels align nicely + if len(record.levelname) > self.level_length: + pad = 0 + if len(record.levelname) <= self.max_level_length: + self.level_length = len(record.levelname) + else: + pad = self.level_length - len(record.levelname) + 1 + sep = '|'.rjust(pad) + fmt = '%(name)s %(levelname)s ' + sep + ' %(message)s' + if self.include_timestamp: + fmt = '%(asctime)s ' + fmt + # this protected member is used to define the format + # used by the base Formatter's method + self._fmt = fmt + return Formatter.format(self, record) + +def getLogger(name, handler=None): + """ + Returns the logger with the specified name. + If the logger doesn't exist, it is created. + If handler is specified, adds it to the logger. Otherwise a default handler + that logs to standard output will be used. + + :param name: The name of the logger to retrieve + :param handler: A handler to add to the logger. If the logger already exists, + and a handler is specified, an exception will be raised. To + add a handler to an existing logger, call that logger's + addHandler method. + """ + setLoggerClass(MozLogger) + + if name in Logger.manager.loggerDict: + if handler: + raise ValueError('The handler parameter requires ' + \ + 'that a logger by this name does ' + \ + 'not already exist') + return Logger.manager.loggerDict[name] + + logger = getSysLogger(name) + logger.setLevel(_default_level) + + if handler is None: + handler = StreamHandler() + handler.setFormatter(MozFormatter()) + + logger.addHandler(handler) + logger.propagate = False + return logger + diff --git a/python/mozlog/mozlog/loggingmixin.py b/python/mozlog/mozlog/loggingmixin.py new file mode 100644 index 00000000000..8e958edac82 --- /dev/null +++ b/python/mozlog/mozlog/loggingmixin.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozlog + +class LoggingMixin(object): + """Expose a subset of logging functions to an inheriting class.""" + + def set_logger(self, logger_instance=None, name=None): + """Method for setting the underlying logger instance to be used.""" + + if logger_instance and not isinstance(logger_instance, mozlog.Logger): + raise ValueError("logger_instance must be an instance of" + + "mozlog.Logger") + + if name is None: + name = ".".join([self.__module__, self.__class__.__name__]) + + self._logger = logger_instance or mozlog.getLogger(name) + + def _log_msg(self, cmd, *args, **kwargs): + if not hasattr(self, "_logger"): + self._logger = mozlog.getLogger(".".join([self.__module__, + self.__class__.__name__])) + getattr(self._logger, cmd)(*args, **kwargs) + + def log(self, *args, **kwargs): + self._log_msg("log", *args, **kwargs) + + def info(self, *args, **kwargs): + self._log_msg("info", *args, **kwargs) + + def error(self, *args, **kwargs): + self._log_msg("error", *args, **kwargs) + + def warn(self, *args, **kwargs): + self._log_msg("warn", *args, **kwargs) + + def log_structured(self, *args, **kwargs): + self._log_msg("log_structured", *args, **kwargs) diff --git a/python/mozlog/mozlog/loglistener.py b/python/mozlog/mozlog/loglistener.py new file mode 100644 index 00000000000..e4b54c3988b --- /dev/null +++ b/python/mozlog/mozlog/loglistener.py @@ -0,0 +1,47 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import SocketServer +import socket +import json + +class LogMessageServer(SocketServer.TCPServer): + def __init__(self, server_address, logger, message_callback=None, timeout=3): + SocketServer.TCPServer.__init__(self, server_address, LogMessageHandler) + self._logger = logger + self._message_callback = message_callback + self.timeout = timeout + +class LogMessageHandler(SocketServer.BaseRequestHandler): + """Processes output from a connected log source, logging to an + existing logger upon receipt of a well-formed log messsage.""" + + def handle(self): + """Continually listens for log messages.""" + self._partial_message = '' + self.request.settimeout(self.server.timeout) + + while True: + try: + data = self.request.recv(1024) + if not data: + return + self.process_message(data) + except socket.timeout: + return + + def process_message(self, data): + """Processes data from a connected log source. Messages are assumed + to be newline delimited, and generally well-formed JSON.""" + for part in data.split('\n'): + msg_string = self._partial_message + part + try: + msg = json.loads(msg_string) + self._partial_message = '' + self.server._logger.log_structured(msg.get('action', 'UNKNOWN'), msg) + if self.server._message_callback: + self.server._message_callback() + + except ValueError: + self._partial_message = msg_string diff --git a/python/mozlog/mozlog/structured/__init__.py b/python/mozlog/mozlog/structured/__init__.py new file mode 100644 index 00000000000..31a779108b4 --- /dev/null +++ b/python/mozlog/mozlog/structured/__init__.py @@ -0,0 +1,7 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import commandline +import structuredlog +from structuredlog import get_default_logger, set_default_logger diff --git a/python/mozlog/mozlog/structured/commandline.py b/python/mozlog/mozlog/structured/commandline.py new file mode 100644 index 00000000000..d4a993febe5 --- /dev/null +++ b/python/mozlog/mozlog/structured/commandline.py @@ -0,0 +1,225 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import os +import optparse + +from collections import defaultdict +from structuredlog import StructuredLogger, set_default_logger +import handlers +import formatters + +log_formatters = { + 'raw': (formatters.JSONFormatter, "Raw structured log messages"), + 'unittest': (formatters.UnittestFormatter, "Unittest style output"), + 'xunit': (formatters.XUnitFormatter, "xUnit compatible XML"), + 'html': (formatters.HTMLFormatter, "HTML report"), + 'mach': (formatters.MachFormatter, "Human-readable output"), + 'tbpl': (formatters.TbplFormatter, "TBPL style log format"), +} + +TEXT_FORMATTERS = ('raw', 'mach') +"""a subset of formatters for non test harnesses related applications""" + +def level_filter_wrapper(formatter, level): + return handlers.LogLevelFilter(formatter, level) + +def verbose_wrapper(formatter, verbose): + formatter.verbose = verbose + return formatter + +def buffer_handler_wrapper(handler, buffer_limit): + if buffer_limit == "UNLIMITED": + buffer_limit = None + else: + buffer_limit = int(buffer_limit) + return handlers.BufferingLogFilter(handler, buffer_limit) + +formatter_option_defaults = { + 'verbose': False, + 'level': 'info', +} + +fmt_options = { + #