Partial oslo-incubator sync

Generated with:
python update.py --base nova --dest-dir ~/Develop/tmp/nova/ --modules
timeutils,sslutils,importutils,memorycache,strutils,context,fileutils,versionutils,systemd

context:
 9b73877 Add a RequestContext.from_dict method
 85d1ce6 Python 3: enable tests/unit/middleware/test_request_id.py
fileutils:
  2b966f9 Fix deletion of cached file for policy enforcer
gettextutils:
 3d90045 Backport code for i18n to check lazy at runtime
jsonutils:
 ef37e03 Added missing jsonutils.dump() function
log:
 5cac11d Merge "Add default log level for websocket"
 433fa0b Make logging_context_format_string optional in log.set_defaults
 ac92c06 Add default log level for websocket
 0aa2bd4 Merge "Ability to customize default_log_levels for each project"
 5fd77eb Ability to customize default_log_levels for each project
 i4d9328c Python 3: enable tests/unit/test_log.py
 722f418 Merge "update new requests logger to default WARN"
 cb5a804 Move `mask_password` to strutils
memorycache:
 90ae24b Remove redundant default=None for config options
 297d772 Raise exception when importing memcache error
strutils:
 cb5a804 Move `mask_password` to strutils
systemd:
 17c4e21 Fix docstring indentation in systemd
 667d1ba Fixed spelling error - occured to occurred
versionutils:
 a2ad3a2 Allow deprecated decorator to specify no plan for removal
 05ae498 Add JUNO as a target to versionutils module

Change-Id: I9e8e8e9b15075d99cf394170c6f5300e7bd0d4cc

nova/openstack/common/context.py

@@ -25,7 +25,7 @@ import uuid
 
 
 def generate_request_id():
-    return 'req-%s' % str(uuid.uuid4())
+    return b'req-' + str(uuid.uuid4()).encode('ascii')
 
 
 class RequestContext(object):
@@ -77,6 +77,21 @@ class RequestContext(object):
                 'instance_uuid': self.instance_uuid,
                 'user_identity': user_idt}
 
+    @classmethod
+    def from_dict(cls, ctx):
+        return cls(
+            auth_token=ctx.get("auth_token"),
+            user=ctx.get("user"),
+            tenant=ctx.get("tenant"),
+            domain=ctx.get("domain"),
+            user_domain=ctx.get("user_domain"),
+            project_domain=ctx.get("project_domain"),
+            is_admin=ctx.get("is_admin", False),
+            read_only=ctx.get("read_only", False),
+            show_deleted=ctx.get("show_deleted", False),
+            request_id=ctx.get("request_id"),
+            instance_uuid=ctx.get("instance_uuid"))
+
 
 def get_admin_context(show_deleted=False):
     context = RequestContext(None,

nova/openstack/common/fileutils.py

@@ -50,8 +50,8 @@ def read_cached_file(filename, force_reload=False):
     """
     global _FILE_CACHE
 
-    if force_reload and filename in _FILE_CACHE:
+    if force_reload:
-        del _FILE_CACHE[filename]
+        delete_cached_file(filename)
 
     reloaded = False
     mtime = os.path.getmtime(filename)
@@ -66,6 +66,17 @@ def read_cached_file(filename, force_reload=False):
     return (reloaded, cache_info['data'])
 
 
+def delete_cached_file(filename):
+    """Delete cached file if present.
+
+    :param filename: filename to delete
+    """
+    global _FILE_CACHE
+
+    if filename in _FILE_CACHE:
+        del _FILE_CACHE[filename]
+
+
 def delete_if_exists(path, remove=os.unlink):
     """Delete a file, but ignore file not found error.
 

nova/openstack/common/gettextutils.py

@@ -23,7 +23,6 @@ Usual usage in an openstack.common module:
 """
 
 import copy
-import functools
 import gettext
 import locale
 from logging import handlers
@@ -42,7 +41,7 @@ class TranslatorFactory(object):
     """Create translator functions
     """
 
-    def __init__(self, domain, lazy=False, localedir=None):
+    def __init__(self, domain, localedir=None):
         """Establish a set of translation functions for the domain.
 
         :param domain: Name of translation domain,
@@ -55,7 +54,6 @@ class TranslatorFactory(object):
         :type localedir: str
         """
         self.domain = domain
-        self.lazy = lazy
         if localedir is None:
             localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
         self.localedir = localedir
@@ -75,16 +73,19 @@ class TranslatorFactory(object):
         """
         if domain is None:
             domain = self.domain
-        if self.lazy:
+        t = gettext.translation(domain,
-            return functools.partial(Message, domain=domain)
+                                localedir=self.localedir,
-        t = gettext.translation(
+                                fallback=True)
-            domain,
+        # Use the appropriate method of the translation object based
-            localedir=self.localedir,
+        # on the python version.
-            fallback=True,
+        m = t.gettext if six.PY3 else t.ugettext
-        )
+
-        if six.PY3:
+        def f(msg):
-            return t.gettext
+            """oslo.i18n.gettextutils translation function."""
-        return t.ugettext
+            if USE_LAZY:
+                return Message(msg, domain=domain)
+            return m(msg)
+        return f
 
     @property
     def primary(self):
@@ -147,19 +148,11 @@ def enable_lazy():
     your project is importing _ directly instead of using the
     gettextutils.install() way of importing the _ function.
     """
-    # FIXME(dhellmann): This function will be removed in oslo.i18n,
+    global USE_LAZY
-    # because the TranslatorFactory makes it superfluous.
-    global _, _LI, _LW, _LE, _LC, USE_LAZY
-    tf = TranslatorFactory('nova', lazy=True)
-    _ = tf.primary
-    _LI = tf.log_info
-    _LW = tf.log_warning
-    _LE = tf.log_error
-    _LC = tf.log_critical
     USE_LAZY = True
 
 
-def install(domain, lazy=False):
+def install(domain):
     """Install a _() function using the given translation domain.
 
     Given a translation domain, install a _() function using gettext's
@@ -170,26 +163,14 @@ def install(domain, lazy=False):
     a translation-domain-specific environment variable (e.g.
     NOVA_LOCALEDIR).
 
+    Note that to enable lazy translation, enable_lazy must be
+    called.
+
     :param domain: the translation domain
-    :param lazy: indicates whether or not to install the lazy _() function.
-                 The lazy _() introduces a way to do deferred translation
-                 of messages by installing a _ that builds Message objects,
-                 instead of strings, which can then be lazily translated into
-                 any available locale.
     """
-    if lazy:
+    from six import moves
-        from six import moves
+    tf = TranslatorFactory(domain)
-        tf = TranslatorFactory(domain, lazy=True)
+    moves.builtins.__dict__['_'] = tf.primary
-        moves.builtins.__dict__['_'] = tf.primary
-    else:
-        localedir = '%s_LOCALEDIR' % domain.upper()
-        if six.PY3:
-            gettext.install(domain,
-                            localedir=os.environ.get(localedir))
-        else:
-            gettext.install(domain,
-                            localedir=os.environ.get(localedir),
-                            unicode=True)
 
 
 class Message(six.text_type):

nova/openstack/common/jsonutils.py

@@ -168,6 +168,10 @@ def dumps(value, default=to_primitive, **kwargs):
     return json.dumps(value, default=default, **kwargs)
 
 
+def dump(obj, fp, *args, **kwargs):
+    return json.dump(obj, fp, *args, **kwargs)
+
+
 def loads(s, encoding='utf-8', **kwargs):
     return json.loads(strutils.safe_decode(s, encoding), **kwargs)
 

nova/openstack/common/log.py

@@ -33,7 +33,6 @@ import logging
 import logging.config
 import logging.handlers
 import os
-import re
 import sys
 import traceback
 
@@ -45,30 +44,13 @@ from nova.openstack.common.gettextutils import _
 from nova.openstack.common import importutils
 from nova.openstack.common import jsonutils
 from nova.openstack.common import local
+# NOTE(flaper87): Pls, remove when graduating this module
+# from the incubator.
+from nova.openstack.common.strutils import mask_password  # noqa
 
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
-_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
-
-# NOTE(ldbragst): Let's build a list of regex objects using the list of
-# _SANITIZE_KEYS we already have. This way, we only have to add the new key
-# to the list of _SANITIZE_KEYS and we can generate regular expressions
-# for XML and JSON automatically.
-_SANITIZE_PATTERNS = []
-_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
-                    r'(<%(key)s>).*?(</%(key)s>)',
-                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
-                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
-                    r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
-                    '.*?([\'"])',
-                    r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
-
-for key in _SANITIZE_KEYS:
-    for pattern in _FORMAT_PATTERNS:
-        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
-        _SANITIZE_PATTERNS.append(reg_ex)
-
 
 common_cli_opts = [
     cfg.BoolOpt('debug',
@@ -138,6 +120,12 @@ generic_log_opts = [
                 help='Log output to standard error.')
 ]
 
+DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
+                      'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
+                      'oslo.messaging=INFO', 'iso8601=WARN',
+                      'requests.packages.urllib3.connectionpool=WARN',
+                      'urllib3.connectionpool=WARN', 'websocket=WARN']
+
 log_opts = [
     cfg.StrOpt('logging_context_format_string',
                default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
@@ -156,17 +144,7 @@ log_opts = [
                '%(instance)s',
                help='Prefix each line of exception output with this format.'),
     cfg.ListOpt('default_log_levels',
-                default=[
+                default=DEFAULT_LOG_LEVELS,
-                    'amqp=WARN',
-                    'amqplib=WARN',
-                    'boto=WARN',
-                    'qpid=WARN',
-                    'sqlalchemy=WARN',
-                    'suds=INFO',
-                    'oslo.messaging=INFO',
-                    'iso8601=WARN',
-                    'requests.packages.urllib3.connectionpool=WARN'
-                ],
                 help='List of logger=LEVEL pairs.'),
     cfg.BoolOpt('publish_errors',
                 default=False,
@@ -244,40 +222,6 @@ def _get_log_file_path(binary=None):
     return None
 
 
-def mask_password(message, secret="***"):
-    """Replace password with 'secret' in message.
-
-    :param message: The string which includes security information.
-    :param secret: value with which to replace passwords.
-    :returns: The unicode value of message with the password fields masked.
-
-    For example:
-
-    >>> mask_password("'adminPass' : 'aaaaa'")
-    "'adminPass' : '***'"
-    >>> mask_password("'admin_pass' : 'aaaaa'")
-    "'admin_pass' : '***'"
-    >>> mask_password('"password" : "aaaaa"')
-    '"password" : "***"'
-    >>> mask_password("'original_password' : 'aaaaa'")
-    "'original_password' : '***'"
-    >>> mask_password("u'original_password' :   u'aaaaa'")
-    "u'original_password' :   u'***'"
-    """
-    message = six.text_type(message)
-
-    # NOTE(ldbragst): Check to see if anything in message contains any key
-    # specified in _SANITIZE_KEYS, if not then just return the message since
-    # we don't have to mask any passwords.
-    if not any(key in message for key in _SANITIZE_KEYS):
-        return message
-
-    secret = r'\g<1>' + secret + r'\g<2>'
-    for pattern in _SANITIZE_PATTERNS:
-        message = re.sub(pattern, secret, message)
-    return message
-
-
 class BaseLoggerAdapter(logging.LoggerAdapter):
 
     def audit(self, msg, *args, **kwargs):
@@ -295,6 +239,11 @@ class LazyAdapter(BaseLoggerAdapter):
     def logger(self):
         if not self._logger:
             self._logger = getLogger(self.name, self.version)
+            if six.PY3:
+                # In Python 3, the code fails because the 'manager' attribute
+                # cannot be found when using a LoggerAdapter as the
+                # underlying logger. Work around this issue.
+                self._logger.manager = self._logger.logger.manager
         return self._logger
 
 
@@ -448,7 +397,7 @@ def _load_log_config(log_config_append):
     try:
         logging.config.fileConfig(log_config_append,
                                   disable_existing_loggers=False)
-    except moves.configparser.Error as exc:
+    except (moves.configparser.Error, KeyError) as exc:
         raise LogConfigError(log_config_append, six.text_type(exc))
 
 
@@ -461,9 +410,20 @@ def setup(product_name, version='unknown'):
     sys.excepthook = _create_logging_excepthook(product_name)
 
 
-def set_defaults(logging_context_format_string):
+def set_defaults(logging_context_format_string=None,
-    cfg.set_defaults(
+                 default_log_levels=None):
-        log_opts, logging_context_format_string=logging_context_format_string)
+    # Just in case the caller is not setting the
+    # default_log_level. This is insurance because
+    # we introduced the default_log_level parameter
+    # later in a backwards in-compatible change
+    if default_log_levels is not None:
+        cfg.set_defaults(
+                log_opts,
+                default_log_levels=default_log_levels)
+    if logging_context_format_string is not None:
+        cfg.set_defaults(
+                log_opts,
+                logging_context_format_string=logging_context_format_string)
 
 
 def _find_facility_from_conf():

nova/openstack/common/memorycache.py

@@ -22,7 +22,6 @@ from nova.openstack.common import timeutils
 
 memcache_opts = [
     cfg.ListOpt('memcached_servers',
-                default=None,
                 help='Memcached servers or None for in process cache.'),
 ]
 
@@ -36,11 +35,8 @@ def get_client(memcached_servers=None):
     if not memcached_servers:
         memcached_servers = CONF.memcached_servers
     if memcached_servers:
-        try:
+        import memcache
-            import memcache
+        client_cls = memcache.Client
-            client_cls = memcache.Client
-        except ImportError:
-            pass
 
     return client_cls(memcached_servers, debug=0)
 

nova/openstack/common/sslutils.py

@@ -22,15 +22,12 @@ from nova.openstack.common.gettextutils import _
 
 ssl_opts = [
     cfg.StrOpt('ca_file',
-               default=None,
                help="CA certificate file to use to verify "
                     "connecting clients."),
     cfg.StrOpt('cert_file',
-               default=None,
                help="Certificate file to use when starting "
                     "the server securely."),
     cfg.StrOpt('key_file',
-               default=None,
                help="Private key file to use when starting "
                     "the server securely."),
 ]

nova/openstack/common/strutils.py

@@ -50,6 +50,28 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
 SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
 
 
+# NOTE(flaper87): The following 3 globals are used by `mask_password`
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS = []
+_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
+                    r'(<%(key)s>).*?(</%(key)s>)',
+                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
+                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
+                    r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
+                    '.*?([\'"])',
+                    r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
+
+for key in _SANITIZE_KEYS:
+    for pattern in _FORMAT_PATTERNS:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
+        _SANITIZE_PATTERNS.append(reg_ex)
+
+
 def int_from_bool_as_string(subject):
     """Interpret a string as a boolean and return either 1 or 0.
 
@@ -237,3 +259,37 @@ def to_slug(value, incoming=None, errors="strict"):
         "ascii", "ignore").decode("ascii")
     value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
     return SLUGIFY_HYPHENATE_RE.sub("-", value)
+
+
+def mask_password(message, secret="***"):
+    """Replace password with 'secret' in message.
+
+    :param message: The string which includes security information.
+    :param secret: value with which to replace passwords.
+    :returns: The unicode value of message with the password fields masked.
+
+    For example:
+
+    >>> mask_password("'adminPass' : 'aaaaa'")
+    "'adminPass' : '***'"
+    >>> mask_password("'admin_pass' : 'aaaaa'")
+    "'admin_pass' : '***'"
+    >>> mask_password('"password" : "aaaaa"')
+    '"password" : "***"'
+    >>> mask_password("'original_password' : 'aaaaa'")
+    "'original_password' : '***'"
+    >>> mask_password("u'original_password' :   u'aaaaa'")
+    "u'original_password' :   u'***'"
+    """
+    message = six.text_type(message)
+
+    # NOTE(ldbragst): Check to see if anything in message contains any key
+    # specified in _SANITIZE_KEYS, if not then just return the message since
+    # we don't have to mask any passwords.
+    if not any(key in message for key in _SANITIZE_KEYS):
+        return message
+
+    secret = r'\g<1>' + secret + r'\g<2>'
+    for pattern in _SANITIZE_PATTERNS:
+        message = re.sub(pattern, secret, message)
+    return message

nova/openstack/common/systemd.py

@@ -50,14 +50,16 @@ def _sd_notify(unset_env, msg):
 
 def notify():
     """Send notification to Systemd that service is ready.
+
     For details see
-      http://www.freedesktop.org/software/systemd/man/sd_notify.html
+    http://www.freedesktop.org/software/systemd/man/sd_notify.html
     """
     _sd_notify(False, 'READY=1')
 
 
 def notify_once():
     """Send notification once to Systemd that service is ready.
+
     Systemd sets NOTIFY_SOCKET environment variable with the name of the
     socket listening for notifications from services.
     This method removes the NOTIFY_SOCKET environment variable to ensure
@@ -75,7 +77,7 @@ def onready(notify_socket, timeout):
     :type timeout:        float
     :returns:             0 service ready
                           1 service not ready
-                          2 timeout occured
+                          2 timeout occurred
     """
     sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
     sock.settimeout(timeout)

nova/openstack/common/versionutils.py

@@ -18,6 +18,7 @@ Helpers for comparing version strings.
 """
 
 import functools
+
 import pkg_resources
 
 from nova.openstack.common.gettextutils import _
@@ -52,18 +53,34 @@ class deprecated(object):
     >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1)
     ... def c(): pass
 
+    4. Specifying the deprecated functionality will not be removed:
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0)
+    ... def d(): pass
+
+    5. Specifying a replacement, deprecated functionality will not be removed:
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0)
+    ... def e(): pass
+
     """
 
+    # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+    # expected we maintain a gap between Bexar and Folsom in this list.
+    BEXAR = 'B'
     FOLSOM = 'F'
     GRIZZLY = 'G'
     HAVANA = 'H'
     ICEHOUSE = 'I'
+    JUNO = 'J'
 
     _RELEASES = {
+        # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+        # expected we maintain a gap between Bexar and Folsom in this list.
+        'B': 'Bexar',
         'F': 'Folsom',
         'G': 'Grizzly',
         'H': 'Havana',
         'I': 'Icehouse',
+        'J': 'Juno',
     }
 
     _deprecated_msg_with_alternative = _(
@@ -74,6 +91,12 @@ class deprecated(object):
         '%(what)s is deprecated as of %(as_of)s and may be '
         'removed in %(remove_in)s. It will not be superseded.')
 
+    _deprecated_msg_with_alternative_no_removal = _(
+        '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.')
+
+    _deprecated_msg_with_no_alternative_no_removal = _(
+        '%(what)s is deprecated as of %(as_of)s. It will not be superseded.')
+
     def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None):
         """Initialize decorator
 
@@ -119,9 +142,19 @@ class deprecated(object):
 
         if self.in_favor_of:
             details['in_favor_of'] = self.in_favor_of
-            msg = self._deprecated_msg_with_alternative
+            if self.remove_in > 0:
+                msg = self._deprecated_msg_with_alternative
+            else:
+                # There are no plans to remove this function, but it is
+                # now deprecated.
+                msg = self._deprecated_msg_with_alternative_no_removal
         else:
-            msg = self._deprecated_msg_no_alternative
+            if self.remove_in > 0:
+                msg = self._deprecated_msg_no_alternative
+            else:
+                # There are no plans to remove this function, but it is
+                # now deprecated.
+                msg = self._deprecated_msg_with_no_alternative_no_removal
         return msg, details