From 84ed8717975f1fe3014c8f6e73ab5738fa14d8c6 Mon Sep 17 00:00:00 2001 From: Angus Salkeld Date: Wed, 4 Jun 2014 10:39:02 +1000 Subject: [PATCH] Update openstack/common code This is just before removing the log module. Change-Id: I9aa34530a571232311bef3db3166295924f525a3 --- .../openstack/common/apiclient/auth.py | 4 +- .../openstack/common/apiclient/base.py | 4 +- .../openstack/common/apiclient/client.py | 5 +- mistralclient/openstack/common/cliutils.py | 7 +- .../openstack/common/gettextutils.py | 179 +++++++++++------- mistralclient/openstack/common/importutils.py | 4 +- mistralclient/openstack/common/jsonutils.py | 22 ++- mistralclient/openstack/common/log.py | 72 +++---- mistralclient/openstack/common/strutils.py | 2 +- 9 files changed, 184 insertions(+), 115 deletions(-) diff --git a/mistralclient/openstack/common/apiclient/auth.py b/mistralclient/openstack/common/apiclient/auth.py index 3e65c7a2..12c49892 100644 --- a/mistralclient/openstack/common/apiclient/auth.py +++ b/mistralclient/openstack/common/apiclient/auth.py @@ -213,8 +213,8 @@ class BaseAuthPlugin(object): :type service_type: string :param endpoint_type: Type of endpoint. Possible values: public or publicURL, - internal or internalURL, - admin or adminURL + internal or internalURL, + admin or adminURL :type endpoint_type: string :returns: tuple of token and endpoint strings :raises: EndpointException diff --git a/mistralclient/openstack/common/apiclient/base.py b/mistralclient/openstack/common/apiclient/base.py index 57917a88..eb230c51 100644 --- a/mistralclient/openstack/common/apiclient/base.py +++ b/mistralclient/openstack/common/apiclient/base.py @@ -75,8 +75,8 @@ class HookableMixin(object): :param cls: class that registers hooks :param hook_type: hook type, e.g., '__pre_parse_args__' - :param **args: args to be passed to every hook function - :param **kwargs: kwargs to be passed to every hook function + :param args: args to be passed to every hook function + :param kwargs: kwargs to be passed to every hook function """ hook_funcs = cls._hooks_map.get(hook_type) or [] for hook_func in hook_funcs: diff --git a/mistralclient/openstack/common/apiclient/client.py b/mistralclient/openstack/common/apiclient/client.py index 3c9903e3..cac3bb10 100644 --- a/mistralclient/openstack/common/apiclient/client.py +++ b/mistralclient/openstack/common/apiclient/client.py @@ -47,6 +47,7 @@ class HTTPClient(object): """This client handles sending HTTP requests to OpenStack servers. Features: + - share authentication information between several clients to different services (e.g., for compute and image clients); - reissue authentication request for expired tokens; @@ -152,7 +153,7 @@ class HTTPClient(object): :param method: method of HTTP request :param url: URL of HTTP request :param kwargs: any other parameter that can be passed to -' requests.Session.request (such as `headers`) or `json` + requests.Session.request (such as `headers`) or `json` that will be encoded as JSON and used as `data` argument """ kwargs.setdefault("headers", kwargs.get("headers", {})) @@ -207,7 +208,7 @@ class HTTPClient(object): :param method: method of HTTP request :param url: URL of HTTP request :param kwargs: any other parameter that can be passed to -' `HTTPClient.request` + `HTTPClient.request` """ filter_args = { diff --git a/mistralclient/openstack/common/cliutils.py b/mistralclient/openstack/common/cliutils.py index cde7ad76..dc5e73f7 100644 --- a/mistralclient/openstack/common/cliutils.py +++ b/mistralclient/openstack/common/cliutils.py @@ -56,7 +56,7 @@ def validate_args(fn, *args, **kwargs): required_args = argspec.args[:len(argspec.args) - num_defaults] def isbound(method): - return getattr(method, 'im_self', None) is not None + return getattr(method, '__self__', None) is not None if isbound(fn): required_args.pop(0) @@ -235,7 +235,10 @@ def find_resource(manager, name_or_id, **find_args): # now try to get entity as uuid try: - tmp_id = strutils.safe_encode(name_or_id) + if six.PY2: + tmp_id = strutils.safe_encode(name_or_id) + else: + tmp_id = strutils.safe_decode(name_or_id) if uuidutils.is_uuid_like(tmp_id): return manager.get(tmp_id) diff --git a/mistralclient/openstack/common/gettextutils.py b/mistralclient/openstack/common/gettextutils.py index 8b42e585..0c57b8d9 100644 --- a/mistralclient/openstack/common/gettextutils.py +++ b/mistralclient/openstack/common/gettextutils.py @@ -32,24 +32,113 @@ import os from babel import localedata import six -_localedir = os.environ.get('mistralclient'.upper() + '_LOCALEDIR') -_t = gettext.translation('mistralclient', localedir=_localedir, fallback=True) - -# We use separate translation catalogs for each log level, so set up a -# mapping between the log level name and the translator. The domain -# for the log level is project_name + "-log-" + log_level so messages -# for each level end up in their own catalog. -_t_log_levels = dict( - (level, gettext.translation('mistralclient' + '-log-' + level, - localedir=_localedir, - fallback=True)) - for level in ['info', 'warning', 'error', 'critical'] -) - _AVAILABLE_LANGUAGES = {} + +# FIXME(dhellmann): Remove this when moving to oslo.i18n. USE_LAZY = False +class TranslatorFactory(object): + """Create translator functions + """ + + def __init__(self, domain, lazy=False, localedir=None): + """Establish a set of translation functions for the domain. + + :param domain: Name of translation domain, + specifying a message catalog. + :type domain: str + :param lazy: Delays translation until a message is emitted. + Defaults to False. + :type lazy: Boolean + :param localedir: Directory with translation catalogs. + :type localedir: str + """ + self.domain = domain + self.lazy = lazy + if localedir is None: + localedir = os.environ.get(domain.upper() + '_LOCALEDIR') + self.localedir = localedir + + def _make_translation_func(self, domain=None): + """Return a new translation function ready for use. + + Takes into account whether or not lazy translation is being + done. + + The domain can be specified to override the default from the + factory, but the localedir from the factory is always used + because we assume the log-level translation catalogs are + installed in the same directory as the main application + catalog. + + """ + if domain is None: + domain = self.domain + if self.lazy: + return functools.partial(Message, domain=domain) + t = gettext.translation( + domain, + localedir=self.localedir, + fallback=True, + ) + if six.PY3: + return t.gettext + return t.ugettext + + @property + def primary(self): + "The default translation function." + return self._make_translation_func() + + def _make_log_translation_func(self, level): + return self._make_translation_func(self.domain + '-log-' + level) + + @property + def log_info(self): + "Translate info-level log messages." + return self._make_log_translation_func('info') + + @property + def log_warning(self): + "Translate warning-level log messages." + return self._make_log_translation_func('warning') + + @property + def log_error(self): + "Translate error-level log messages." + return self._make_log_translation_func('error') + + @property + def log_critical(self): + "Translate critical-level log messages." + return self._make_log_translation_func('critical') + + +# NOTE(dhellmann): When this module moves out of the incubator into +# oslo.i18n, these global variables can be moved to an integration +# module within each application. + +# Create the global translation functions. +_translators = TranslatorFactory('mistralclient') + +# The primary translation function using the well-known name "_" +_ = _translators.primary + +# Translators for log levels. +# +# The abbreviated names are meant to reflect the usual use of a short +# name like '_'. The "L" is for "log" and the other letter comes from +# the level. +_LI = _translators.log_info +_LW = _translators.log_warning +_LE = _translators.log_error +_LC = _translators.log_critical + +# NOTE(dhellmann): End of globals that will move to the application's +# integration module. + + def enable_lazy(): """Convenience function for configuring _() to use lazy gettext @@ -58,41 +147,18 @@ def enable_lazy(): your project is importing _ directly instead of using the gettextutils.install() way of importing the _ function. """ - global USE_LAZY + # FIXME(dhellmann): This function will be removed in oslo.i18n, + # because the TranslatorFactory makes it superfluous. + global _, _LI, _LW, _LE, _LC, USE_LAZY + tf = TranslatorFactory('mistralclient', lazy=True) + _ = tf.primary + _LI = tf.log_info + _LW = tf.log_warning + _LE = tf.log_error + _LC = tf.log_critical USE_LAZY = True -def _(msg): - if USE_LAZY: - return Message(msg, domain='mistralclient') - else: - if six.PY3: - return _t.gettext(msg) - return _t.ugettext(msg) - - -def _log_translation(msg, level): - """Build a single translation of a log message - """ - if USE_LAZY: - return Message(msg, domain='mistralclient' + '-log-' + level) - else: - translator = _t_log_levels[level] - if six.PY3: - return translator.gettext(msg) - return translator.ugettext(msg) - -# Translators for log levels. -# -# The abbreviated names are meant to reflect the usual use of a short -# name like '_'. The "L" is for "log" and the other letter comes from -# the level. -_LI = functools.partial(_log_translation, level='info') -_LW = functools.partial(_log_translation, level='warning') -_LE = functools.partial(_log_translation, level='error') -_LC = functools.partial(_log_translation, level='critical') - - def install(domain, lazy=False): """Install a _() function using the given translation domain. @@ -112,26 +178,9 @@ def install(domain, lazy=False): any available locale. """ if lazy: - # NOTE(mrodden): Lazy gettext functionality. - # - # The following introduces a deferred way to do translations on - # messages in OpenStack. We override the standard _() function - # and % (format string) operation to build Message objects that can - # later be translated when we have more information. - def _lazy_gettext(msg): - """Create and return a Message object. - - Lazy gettext function for a given domain, it is a factory method - for a project/module to get a lazy gettext function for its own - translation domain (i.e. nova, glance, cinder, etc.) - - Message encapsulates a string so that we can translate - it later when needed. - """ - return Message(msg, domain=domain) - from six import moves - moves.builtins.__dict__['_'] = _lazy_gettext + tf = TranslatorFactory(domain, lazy=True) + moves.builtins.__dict__['_'] = tf.primary else: localedir = '%s_LOCALEDIR' % domain.upper() if six.PY3: diff --git a/mistralclient/openstack/common/importutils.py b/mistralclient/openstack/common/importutils.py index 4f901ae9..58a3c2ee 100644 --- a/mistralclient/openstack/common/importutils.py +++ b/mistralclient/openstack/common/importutils.py @@ -24,10 +24,10 @@ import traceback def import_class(import_str): """Returns a class from a string including module and class.""" mod_str, _sep, class_str = import_str.rpartition('.') + __import__(mod_str) try: - __import__(mod_str) return getattr(sys.modules[mod_str], class_str) - except (ValueError, AttributeError): + except AttributeError: raise ImportError('Class %s cannot be found (%s)' % (class_str, traceback.format_exception(*sys.exc_info()))) diff --git a/mistralclient/openstack/common/jsonutils.py b/mistralclient/openstack/common/jsonutils.py index 2c825e0a..730a0a95 100644 --- a/mistralclient/openstack/common/jsonutils.py +++ b/mistralclient/openstack/common/jsonutils.py @@ -31,17 +31,29 @@ This module provides a few things: ''' +import codecs import datetime import functools import inspect import itertools -import json +import sys + +if sys.version_info < (2, 7): + # On Python <= 2.6, json module is not C boosted, so try to use + # simplejson module if available + try: + import simplejson as json + except ImportError: + import json +else: + import json import six import six.moves.xmlrpc_client as xmlrpclib from mistralclient.openstack.common import gettextutils from mistralclient.openstack.common import importutils +from mistralclient.openstack.common import strutils from mistralclient.openstack.common import timeutils netaddr = importutils.try_import("netaddr") @@ -156,12 +168,12 @@ def dumps(value, default=to_primitive, **kwargs): return json.dumps(value, default=default, **kwargs) -def loads(s): - return json.loads(s) +def loads(s, encoding='utf-8'): + return json.loads(strutils.safe_decode(s, encoding)) -def load(s): - return json.load(s) +def load(fp, encoding='utf-8'): + return json.load(codecs.getreader(encoding)(fp)) try: diff --git a/mistralclient/openstack/common/log.py b/mistralclient/openstack/common/log.py index c6b0cd64..22d997d6 100644 --- a/mistralclient/openstack/common/log.py +++ b/mistralclient/openstack/common/log.py @@ -59,7 +59,10 @@ _SANITIZE_PATTERNS = [] _FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', r'(<%(key)s>).*?()', r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])'] + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', + r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' + '.*?([\'"])', + r'(%(key)s\s*--?[A-z]+\s*).*?([\s])'] for key in _SANITIZE_KEYS: for pattern in _FORMAT_PATTERNS: @@ -84,14 +87,11 @@ logging_cli_opts = [ cfg.StrOpt('log-config-append', metavar='PATH', deprecated_name='log-config', - help='The name of logging configuration file. It does not ' - 'disable existing loggers, but just appends specified ' - 'logging configuration to any other existing logging ' - 'options. Please see the Python logging module ' - 'documentation for details on logging configuration ' - 'files.'), + help='The name of a logging configuration file. This file ' + 'is appended to any existing logging configuration ' + 'files. For details about logging configuration files, ' + 'see the Python logging module documentation.'), cfg.StrOpt('log-format', - default=None, metavar='FORMAT', help='DEPRECATED. ' 'A logging.Formatter log message format string which may ' @@ -103,7 +103,7 @@ logging_cli_opts = [ default=_DEFAULT_LOG_DATE_FORMAT, metavar='DATE_FORMAT', help='Format string for %%(asctime)s in log records. ' - 'Default: %(default)s'), + 'Default: %(default)s .'), cfg.StrOpt('log-file', metavar='PATH', deprecated_name='logfile', @@ -112,30 +112,30 @@ logging_cli_opts = [ cfg.StrOpt('log-dir', deprecated_name='logdir', help='(Optional) The base directory used for relative ' - '--log-file paths'), + '--log-file paths.'), cfg.BoolOpt('use-syslog', default=False, help='Use syslog for logging. ' 'Existing syslog format is DEPRECATED during I, ' - 'and then will be changed in J to honor RFC5424'), + 'and will change in J to honor RFC5424.'), cfg.BoolOpt('use-syslog-rfc-format', # TODO(bogdando) remove or use True after existing # syslog format deprecation in J default=False, - help='(Optional) Use syslog rfc5424 format for logging. ' - 'If enabled, will add APP-NAME (RFC5424) before the ' - 'MSG part of the syslog message. The old format ' - 'without APP-NAME is deprecated in I, ' + help='(Optional) Enables or disables syslog rfc5424 format ' + 'for logging. If enabled, prefixes the MSG part of the ' + 'syslog message with APP-NAME (RFC5424). The ' + 'format without the APP-NAME is deprecated in I, ' 'and will be removed in J.'), cfg.StrOpt('syslog-log-facility', default='LOG_USER', - help='Syslog facility to receive log lines') + help='Syslog facility to receive log lines.') ] generic_log_opts = [ cfg.BoolOpt('use_stderr', default=True, - help='Log output to standard error') + help='Log output to standard error.') ] log_opts = [ @@ -143,18 +143,18 @@ log_opts = [ default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [%(request_id)s %(user_identity)s] ' '%(instance)s%(message)s', - help='Format string to use for log messages with context'), + help='Format string to use for log messages with context.'), cfg.StrOpt('logging_default_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' '%(name)s [-] %(instance)s%(message)s', - help='Format string to use for log messages without context'), + help='Format string to use for log messages without context.'), cfg.StrOpt('logging_debug_format_suffix', default='%(funcName)s %(pathname)s:%(lineno)d', - help='Data to append to log format when level is DEBUG'), + help='Data to append to log format when level is DEBUG.'), cfg.StrOpt('logging_exception_prefix', default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' '%(instance)s', - help='Prefix each line of exception output with this format'), + help='Prefix each line of exception output with this format.'), cfg.ListOpt('default_log_levels', default=[ 'amqp=WARN', @@ -167,25 +167,25 @@ log_opts = [ 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN' ], - help='List of logger=LEVEL pairs'), + help='List of logger=LEVEL pairs.'), cfg.BoolOpt('publish_errors', default=False, - help='Publish error events'), + help='Enables or disables publication of error events.'), cfg.BoolOpt('fatal_deprecations', default=False, - help='Make deprecations fatal'), + help='Enables or disables fatal status of deprecations.'), # NOTE(mikal): there are two options here because sometimes we are handed # a full instance (and could include more information), and other times we # are just handed a UUID for the instance. cfg.StrOpt('instance_format', default='[instance: %(uuid)s] ', - help='If an instance is passed with the log message, format ' - 'it like this'), + help='The format for an instance that is passed with the log ' + 'message. '), cfg.StrOpt('instance_uuid_format', default='[instance: %(uuid)s] ', - help='If an instance UUID is passed with the log message, ' - 'format it like this'), + help='The format for an instance UUID that is passed with the ' + 'log message. '), ] CONF = cfg.CONF @@ -424,9 +424,7 @@ class JSONFormatter(logging.Formatter): def _create_logging_excepthook(product_name): def logging_excepthook(exc_type, value, tb): - extra = {} - if CONF.verbose or CONF.debug: - extra['exc_info'] = (exc_type, value, tb) + extra = {'exc_info': (exc_type, value, tb)} getLogger(product_name).critical( "".join(traceback.format_exception_only(exc_type, value)), **extra) @@ -451,7 +449,7 @@ def _load_log_config(log_config_append): logging.config.fileConfig(log_config_append, disable_existing_loggers=False) except moves.configparser.Error as exc: - raise LogConfigError(log_config_append, str(exc)) + raise LogConfigError(log_config_append, six.text_type(exc)) def setup(product_name, version='unknown'): @@ -571,9 +569,15 @@ def _setup_logging_from_conf(project, version): for pair in CONF.default_log_levels: mod, _sep, level_name = pair.partition('=') - level = logging.getLevelName(level_name) logger = logging.getLogger(mod) - logger.setLevel(level) + # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name + # to integer code. + if sys.version_info < (2, 7): + level = logging.getLevelName(level_name) + logger.setLevel(level) + else: + logger.setLevel(level_name) + _loggers = {} diff --git a/mistralclient/openstack/common/strutils.py b/mistralclient/openstack/common/strutils.py index 936a9289..f496d0ac 100644 --- a/mistralclient/openstack/common/strutils.py +++ b/mistralclient/openstack/common/strutils.py @@ -78,7 +78,7 @@ def bool_from_string(subject, strict=False, default=False): Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. """ if not isinstance(subject, six.string_types): - subject = str(subject) + subject = six.text_type(subject) lowered = subject.strip().lower()