From 74d06db19fe0037435c12e52c3c88f980e619420 Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Tue, 15 Jul 2014 14:58:32 +0200 Subject: [PATCH] Partial oslo-incubator sync Generated with: python update.py --base nova --dest-dir ~/Develop/tmp/nova/ --modules timeutils,sslutils,importutils,memorycache,strutils,context,fileutils,versionutils,systemd context: 9b73877 Add a RequestContext.from_dict method 85d1ce6 Python 3: enable tests/unit/middleware/test_request_id.py fileutils: 2b966f9 Fix deletion of cached file for policy enforcer gettextutils: 3d90045 Backport code for i18n to check lazy at runtime jsonutils: ef37e03 Added missing jsonutils.dump() function log: 5cac11d Merge "Add default log level for websocket" 433fa0b Make logging_context_format_string optional in log.set_defaults ac92c06 Add default log level for websocket 0aa2bd4 Merge "Ability to customize default_log_levels for each project" 5fd77eb Ability to customize default_log_levels for each project i4d9328c Python 3: enable tests/unit/test_log.py 722f418 Merge "update new requests logger to default WARN" cb5a804 Move `mask_password` to strutils memorycache: 90ae24b Remove redundant default=None for config options 297d772 Raise exception when importing memcache error strutils: cb5a804 Move `mask_password` to strutils systemd: 17c4e21 Fix docstring indentation in systemd 667d1ba Fixed spelling error - occured to occurred versionutils: a2ad3a2 Allow deprecated decorator to specify no plan for removal 05ae498 Add JUNO as a target to versionutils module Change-Id: I9e8e8e9b15075d99cf394170c6f5300e7bd0d4cc --- nova/openstack/common/context.py | 17 ++++- nova/openstack/common/fileutils.py | 15 +++- nova/openstack/common/gettextutils.py | 63 ++++++---------- nova/openstack/common/jsonutils.py | 4 ++ nova/openstack/common/log.py | 100 ++++++++------------------ nova/openstack/common/memorycache.py | 8 +-- nova/openstack/common/sslutils.py | 3 - nova/openstack/common/strutils.py | 56 +++++++++++++++ nova/openstack/common/systemd.py | 6 +- nova/openstack/common/versionutils.py | 37 +++++++++- 10 files changed, 182 insertions(+), 127 deletions(-) diff --git a/nova/openstack/common/context.py b/nova/openstack/common/context.py index 09019ee3843d..b612db71407c 100644 --- a/nova/openstack/common/context.py +++ b/nova/openstack/common/context.py @@ -25,7 +25,7 @@ import uuid def generate_request_id(): - return 'req-%s' % str(uuid.uuid4()) + return b'req-' + str(uuid.uuid4()).encode('ascii') class RequestContext(object): @@ -77,6 +77,21 @@ class RequestContext(object): 'instance_uuid': self.instance_uuid, 'user_identity': user_idt} + @classmethod + def from_dict(cls, ctx): + return cls( + auth_token=ctx.get("auth_token"), + user=ctx.get("user"), + tenant=ctx.get("tenant"), + domain=ctx.get("domain"), + user_domain=ctx.get("user_domain"), + project_domain=ctx.get("project_domain"), + is_admin=ctx.get("is_admin", False), + read_only=ctx.get("read_only", False), + show_deleted=ctx.get("show_deleted", False), + request_id=ctx.get("request_id"), + instance_uuid=ctx.get("instance_uuid")) + def get_admin_context(show_deleted=False): context = RequestContext(None, diff --git a/nova/openstack/common/fileutils.py b/nova/openstack/common/fileutils.py index 16050a138f29..12ae19830319 100644 --- a/nova/openstack/common/fileutils.py +++ b/nova/openstack/common/fileutils.py @@ -50,8 +50,8 @@ def read_cached_file(filename, force_reload=False): """ global _FILE_CACHE - if force_reload and filename in _FILE_CACHE: - del _FILE_CACHE[filename] + if force_reload: + delete_cached_file(filename) reloaded = False mtime = os.path.getmtime(filename) @@ -66,6 +66,17 @@ def read_cached_file(filename, force_reload=False): return (reloaded, cache_info['data']) +def delete_cached_file(filename): + """Delete cached file if present. + + :param filename: filename to delete + """ + global _FILE_CACHE + + if filename in _FILE_CACHE: + del _FILE_CACHE[filename] + + def delete_if_exists(path, remove=os.unlink): """Delete a file, but ignore file not found error. diff --git a/nova/openstack/common/gettextutils.py b/nova/openstack/common/gettextutils.py index dcb53582a1be..4dd2ac6594da 100644 --- a/nova/openstack/common/gettextutils.py +++ b/nova/openstack/common/gettextutils.py @@ -23,7 +23,6 @@ Usual usage in an openstack.common module: """ import copy -import functools import gettext import locale from logging import handlers @@ -42,7 +41,7 @@ class TranslatorFactory(object): """Create translator functions """ - def __init__(self, domain, lazy=False, localedir=None): + def __init__(self, domain, localedir=None): """Establish a set of translation functions for the domain. :param domain: Name of translation domain, @@ -55,7 +54,6 @@ class TranslatorFactory(object): :type localedir: str """ self.domain = domain - self.lazy = lazy if localedir is None: localedir = os.environ.get(domain.upper() + '_LOCALEDIR') self.localedir = localedir @@ -75,16 +73,19 @@ class TranslatorFactory(object): """ if domain is None: domain = self.domain - if self.lazy: - return functools.partial(Message, domain=domain) - t = gettext.translation( - domain, - localedir=self.localedir, - fallback=True, - ) - if six.PY3: - return t.gettext - return t.ugettext + t = gettext.translation(domain, + localedir=self.localedir, + fallback=True) + # Use the appropriate method of the translation object based + # on the python version. + m = t.gettext if six.PY3 else t.ugettext + + def f(msg): + """oslo.i18n.gettextutils translation function.""" + if USE_LAZY: + return Message(msg, domain=domain) + return m(msg) + return f @property def primary(self): @@ -147,19 +148,11 @@ def enable_lazy(): your project is importing _ directly instead of using the gettextutils.install() way of importing the _ function. """ - # FIXME(dhellmann): This function will be removed in oslo.i18n, - # because the TranslatorFactory makes it superfluous. - global _, _LI, _LW, _LE, _LC, USE_LAZY - tf = TranslatorFactory('nova', lazy=True) - _ = tf.primary - _LI = tf.log_info - _LW = tf.log_warning - _LE = tf.log_error - _LC = tf.log_critical + global USE_LAZY USE_LAZY = True -def install(domain, lazy=False): +def install(domain): """Install a _() function using the given translation domain. Given a translation domain, install a _() function using gettext's @@ -170,26 +163,14 @@ def install(domain, lazy=False): a translation-domain-specific environment variable (e.g. NOVA_LOCALEDIR). + Note that to enable lazy translation, enable_lazy must be + called. + :param domain: the translation domain - :param lazy: indicates whether or not to install the lazy _() function. - The lazy _() introduces a way to do deferred translation - of messages by installing a _ that builds Message objects, - instead of strings, which can then be lazily translated into - any available locale. """ - if lazy: - from six import moves - tf = TranslatorFactory(domain, lazy=True) - moves.builtins.__dict__['_'] = tf.primary - else: - localedir = '%s_LOCALEDIR' % domain.upper() - if six.PY3: - gettext.install(domain, - localedir=os.environ.get(localedir)) - else: - gettext.install(domain, - localedir=os.environ.get(localedir), - unicode=True) + from six import moves + tf = TranslatorFactory(domain) + moves.builtins.__dict__['_'] = tf.primary class Message(six.text_type): diff --git a/nova/openstack/common/jsonutils.py b/nova/openstack/common/jsonutils.py index 650c98328176..a201b8c8177f 100644 --- a/nova/openstack/common/jsonutils.py +++ b/nova/openstack/common/jsonutils.py @@ -168,6 +168,10 @@ def dumps(value, default=to_primitive, **kwargs): return json.dumps(value, default=default, **kwargs) +def dump(obj, fp, *args, **kwargs): + return json.dump(obj, fp, *args, **kwargs) + + def loads(s, encoding='utf-8', **kwargs): return json.loads(strutils.safe_decode(s, encoding), **kwargs) diff --git a/nova/openstack/common/log.py b/nova/openstack/common/log.py index 354af01e259c..bc9cbfa41150 100644 --- a/nova/openstack/common/log.py +++ b/nova/openstack/common/log.py @@ -33,7 +33,6 @@ import logging import logging.config import logging.handlers import os -import re import sys import traceback @@ -45,30 +44,13 @@ from nova.openstack.common.gettextutils import _ from nova.openstack.common import importutils from nova.openstack.common import jsonutils from nova.openstack.common import local +# NOTE(flaper87): Pls, remove when graduating this module +# from the incubator. +from nova.openstack.common.strutils import mask_password # noqa _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" -_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] - -# NOTE(ldbragst): Let's build a list of regex objects using the list of -# _SANITIZE_KEYS we already have. This way, we only have to add the new key -# to the list of _SANITIZE_KEYS and we can generate regular expressions -# for XML and JSON automatically. -_SANITIZE_PATTERNS = [] -_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', - r'(<%(key)s>).*?()', - r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', - r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', - r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' - '.*?([\'"])', - r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] - -for key in _SANITIZE_KEYS: - for pattern in _FORMAT_PATTERNS: - reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) - _SANITIZE_PATTERNS.append(reg_ex) - common_cli_opts = [ cfg.BoolOpt('debug', @@ -138,6 +120,12 @@ generic_log_opts = [ help='Log output to standard error.') ] +DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', + 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', + 'oslo.messaging=INFO', 'iso8601=WARN', + 'requests.packages.urllib3.connectionpool=WARN', + 'urllib3.connectionpool=WARN', 'websocket=WARN'] + log_opts = [ cfg.StrOpt('logging_context_format_string', default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' @@ -156,17 +144,7 @@ log_opts = [ '%(instance)s', help='Prefix each line of exception output with this format.'), cfg.ListOpt('default_log_levels', - default=[ - 'amqp=WARN', - 'amqplib=WARN', - 'boto=WARN', - 'qpid=WARN', - 'sqlalchemy=WARN', - 'suds=INFO', - 'oslo.messaging=INFO', - 'iso8601=WARN', - 'requests.packages.urllib3.connectionpool=WARN' - ], + default=DEFAULT_LOG_LEVELS, help='List of logger=LEVEL pairs.'), cfg.BoolOpt('publish_errors', default=False, @@ -244,40 +222,6 @@ def _get_log_file_path(binary=None): return None -def mask_password(message, secret="***"): - """Replace password with 'secret' in message. - - :param message: The string which includes security information. - :param secret: value with which to replace passwords. - :returns: The unicode value of message with the password fields masked. - - For example: - - >>> mask_password("'adminPass' : 'aaaaa'") - "'adminPass' : '***'" - >>> mask_password("'admin_pass' : 'aaaaa'") - "'admin_pass' : '***'" - >>> mask_password('"password" : "aaaaa"') - '"password" : "***"' - >>> mask_password("'original_password' : 'aaaaa'") - "'original_password' : '***'" - >>> mask_password("u'original_password' : u'aaaaa'") - "u'original_password' : u'***'" - """ - message = six.text_type(message) - - # NOTE(ldbragst): Check to see if anything in message contains any key - # specified in _SANITIZE_KEYS, if not then just return the message since - # we don't have to mask any passwords. - if not any(key in message for key in _SANITIZE_KEYS): - return message - - secret = r'\g<1>' + secret + r'\g<2>' - for pattern in _SANITIZE_PATTERNS: - message = re.sub(pattern, secret, message) - return message - - class BaseLoggerAdapter(logging.LoggerAdapter): def audit(self, msg, *args, **kwargs): @@ -295,6 +239,11 @@ class LazyAdapter(BaseLoggerAdapter): def logger(self): if not self._logger: self._logger = getLogger(self.name, self.version) + if six.PY3: + # In Python 3, the code fails because the 'manager' attribute + # cannot be found when using a LoggerAdapter as the + # underlying logger. Work around this issue. + self._logger.manager = self._logger.logger.manager return self._logger @@ -448,7 +397,7 @@ def _load_log_config(log_config_append): try: logging.config.fileConfig(log_config_append, disable_existing_loggers=False) - except moves.configparser.Error as exc: + except (moves.configparser.Error, KeyError) as exc: raise LogConfigError(log_config_append, six.text_type(exc)) @@ -461,9 +410,20 @@ def setup(product_name, version='unknown'): sys.excepthook = _create_logging_excepthook(product_name) -def set_defaults(logging_context_format_string): - cfg.set_defaults( - log_opts, logging_context_format_string=logging_context_format_string) +def set_defaults(logging_context_format_string=None, + default_log_levels=None): + # Just in case the caller is not setting the + # default_log_level. This is insurance because + # we introduced the default_log_level parameter + # later in a backwards in-compatible change + if default_log_levels is not None: + cfg.set_defaults( + log_opts, + default_log_levels=default_log_levels) + if logging_context_format_string is not None: + cfg.set_defaults( + log_opts, + logging_context_format_string=logging_context_format_string) def _find_facility_from_conf(): diff --git a/nova/openstack/common/memorycache.py b/nova/openstack/common/memorycache.py index 313a8c14fb58..5e16363eaaf0 100644 --- a/nova/openstack/common/memorycache.py +++ b/nova/openstack/common/memorycache.py @@ -22,7 +22,6 @@ from nova.openstack.common import timeutils memcache_opts = [ cfg.ListOpt('memcached_servers', - default=None, help='Memcached servers or None for in process cache.'), ] @@ -36,11 +35,8 @@ def get_client(memcached_servers=None): if not memcached_servers: memcached_servers = CONF.memcached_servers if memcached_servers: - try: - import memcache - client_cls = memcache.Client - except ImportError: - pass + import memcache + client_cls = memcache.Client return client_cls(memcached_servers, debug=0) diff --git a/nova/openstack/common/sslutils.py b/nova/openstack/common/sslutils.py index a18e7fd051f0..00e6173d07e2 100644 --- a/nova/openstack/common/sslutils.py +++ b/nova/openstack/common/sslutils.py @@ -22,15 +22,12 @@ from nova.openstack.common.gettextutils import _ ssl_opts = [ cfg.StrOpt('ca_file', - default=None, help="CA certificate file to use to verify " "connecting clients."), cfg.StrOpt('cert_file', - default=None, help="Certificate file to use when starting " "the server securely."), cfg.StrOpt('key_file', - default=None, help="Private key file to use when starting " "the server securely."), ] diff --git a/nova/openstack/common/strutils.py b/nova/openstack/common/strutils.py index 3d98260b1d2f..b75eb85354eb 100644 --- a/nova/openstack/common/strutils.py +++ b/nova/openstack/common/strutils.py @@ -50,6 +50,28 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") +# NOTE(flaper87): The following 3 globals are used by `mask_password` +_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] + +# NOTE(ldbragst): Let's build a list of regex objects using the list of +# _SANITIZE_KEYS we already have. This way, we only have to add the new key +# to the list of _SANITIZE_KEYS and we can generate regular expressions +# for XML and JSON automatically. +_SANITIZE_PATTERNS = [] +_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', + r'(<%(key)s>).*?()', + r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', + r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' + '.*?([\'"])', + r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] + +for key in _SANITIZE_KEYS: + for pattern in _FORMAT_PATTERNS: + reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) + _SANITIZE_PATTERNS.append(reg_ex) + + def int_from_bool_as_string(subject): """Interpret a string as a boolean and return either 1 or 0. @@ -237,3 +259,37 @@ def to_slug(value, incoming=None, errors="strict"): "ascii", "ignore").decode("ascii") value = SLUGIFY_STRIP_RE.sub("", value).strip().lower() return SLUGIFY_HYPHENATE_RE.sub("-", value) + + +def mask_password(message, secret="***"): + """Replace password with 'secret' in message. + + :param message: The string which includes security information. + :param secret: value with which to replace passwords. + :returns: The unicode value of message with the password fields masked. + + For example: + + >>> mask_password("'adminPass' : 'aaaaa'") + "'adminPass' : '***'" + >>> mask_password("'admin_pass' : 'aaaaa'") + "'admin_pass' : '***'" + >>> mask_password('"password" : "aaaaa"') + '"password" : "***"' + >>> mask_password("'original_password' : 'aaaaa'") + "'original_password' : '***'" + >>> mask_password("u'original_password' : u'aaaaa'") + "u'original_password' : u'***'" + """ + message = six.text_type(message) + + # NOTE(ldbragst): Check to see if anything in message contains any key + # specified in _SANITIZE_KEYS, if not then just return the message since + # we don't have to mask any passwords. + if not any(key in message for key in _SANITIZE_KEYS): + return message + + secret = r'\g<1>' + secret + r'\g<2>' + for pattern in _SANITIZE_PATTERNS: + message = re.sub(pattern, secret, message) + return message diff --git a/nova/openstack/common/systemd.py b/nova/openstack/common/systemd.py index 4fa0c6279046..5628d54f004f 100644 --- a/nova/openstack/common/systemd.py +++ b/nova/openstack/common/systemd.py @@ -50,14 +50,16 @@ def _sd_notify(unset_env, msg): def notify(): """Send notification to Systemd that service is ready. + For details see - http://www.freedesktop.org/software/systemd/man/sd_notify.html + http://www.freedesktop.org/software/systemd/man/sd_notify.html """ _sd_notify(False, 'READY=1') def notify_once(): """Send notification once to Systemd that service is ready. + Systemd sets NOTIFY_SOCKET environment variable with the name of the socket listening for notifications from services. This method removes the NOTIFY_SOCKET environment variable to ensure @@ -75,7 +77,7 @@ def onready(notify_socket, timeout): :type timeout: float :returns: 0 service ready 1 service not ready - 2 timeout occured + 2 timeout occurred """ sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) sock.settimeout(timeout) diff --git a/nova/openstack/common/versionutils.py b/nova/openstack/common/versionutils.py index 86e196140d27..1facce7726da 100644 --- a/nova/openstack/common/versionutils.py +++ b/nova/openstack/common/versionutils.py @@ -18,6 +18,7 @@ Helpers for comparing version strings. """ import functools + import pkg_resources from nova.openstack.common.gettextutils import _ @@ -52,18 +53,34 @@ class deprecated(object): >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1) ... def c(): pass + 4. Specifying the deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0) + ... def d(): pass + + 5. Specifying a replacement, deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0) + ... def e(): pass + """ + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + BEXAR = 'B' FOLSOM = 'F' GRIZZLY = 'G' HAVANA = 'H' ICEHOUSE = 'I' + JUNO = 'J' _RELEASES = { + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + 'B': 'Bexar', 'F': 'Folsom', 'G': 'Grizzly', 'H': 'Havana', 'I': 'Icehouse', + 'J': 'Juno', } _deprecated_msg_with_alternative = _( @@ -74,6 +91,12 @@ class deprecated(object): '%(what)s is deprecated as of %(as_of)s and may be ' 'removed in %(remove_in)s. It will not be superseded.') + _deprecated_msg_with_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.') + + _deprecated_msg_with_no_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s. It will not be superseded.') + def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None): """Initialize decorator @@ -119,9 +142,19 @@ class deprecated(object): if self.in_favor_of: details['in_favor_of'] = self.in_favor_of - msg = self._deprecated_msg_with_alternative + if self.remove_in > 0: + msg = self._deprecated_msg_with_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_alternative_no_removal else: - msg = self._deprecated_msg_no_alternative + if self.remove_in > 0: + msg = self._deprecated_msg_no_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_no_alternative_no_removal return msg, details