Update from oslo-incubator

Change-Id: I16b9f94b74de927bb831f0f38483fb3909bbb3f7

designate/openstack/common/context.py

@@ -77,6 +77,21 @@ class RequestContext(object):
                 'instance_uuid': self.instance_uuid,
                 'user_identity': user_idt}
 
+    @classmethod
+    def from_dict(cls, ctx):
+        return cls(
+            auth_token=ctx.get("auth_token"),
+            user=ctx.get("user"),
+            tenant=ctx.get("tenant"),
+            domain=ctx.get("domain"),
+            user_domain=ctx.get("user_domain"),
+            project_domain=ctx.get("project_domain"),
+            is_admin=ctx.get("is_admin", False),
+            read_only=ctx.get("read_only", False),
+            show_deleted=ctx.get("show_deleted", False),
+            request_id=ctx.get("request_id"),
+            instance_uuid=ctx.get("instance_uuid"))
+
 
 def get_admin_context(show_deleted=False):
     context = RequestContext(None,

designate/openstack/common/eventlet_backdoor.py

@@ -41,7 +41,6 @@ help_for_backdoor_port = (
     "chosen port is displayed in the service's log file.")
 eventlet_backdoor_opts = [
     cfg.StrOpt('backdoor_port',
-               default=None,
                help="Enable eventlet backdoor.  %s" % help_for_backdoor_port)
 ]
 

designate/openstack/common/fileutils.py

@@ -50,8 +50,8 @@ def read_cached_file(filename, force_reload=False):
     """
     global _FILE_CACHE
 
-    if force_reload and filename in _FILE_CACHE:
+    if force_reload:
-        del _FILE_CACHE[filename]
+        delete_cached_file(filename)
 
     reloaded = False
     mtime = os.path.getmtime(filename)
@@ -66,6 +66,17 @@ def read_cached_file(filename, force_reload=False):
     return (reloaded, cache_info['data'])
 
 
+def delete_cached_file(filename):
+    """Delete cached file if present.
+
+    :param filename: filename to delete
+    """
+    global _FILE_CACHE
+
+    if filename in _FILE_CACHE:
+        del _FILE_CACHE[filename]
+
+
 def delete_if_exists(path, remove=os.unlink):
     """Delete a file, but ignore file not found error.
 
@@ -99,13 +110,13 @@ def remove_path_on_error(path, remove=delete_if_exists):
 def file_open(*args, **kwargs):
     """Open file
 
-    see built-in file() documentation for more details
+    see built-in open() documentation for more details
 
     Note: The reason this is kept in a separate module is to easily
     be able to provide a stub module that doesn't alter system
     state at all (for unit tests)
     """
-    return file(*args, **kwargs)
+    return open(*args, **kwargs)
 
 
 def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):

designate/openstack/common/fixture/lockutils.py

@@ -1,51 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import fixtures
-
-from designate.openstack.common import lockutils
-
-
-class LockFixture(fixtures.Fixture):
-    """External locking fixture.
-
-    This fixture is basically an alternative to the synchronized decorator with
-    the external flag so that tearDowns and addCleanups will be included in
-    the lock context for locking between tests. The fixture is recommended to
-    be the first line in a test method, like so::
-
-        def test_method(self):
-            self.useFixture(LockFixture)
-                ...
-
-    or the first line in setUp if all the test methods in the class are
-    required to be serialized. Something like::
-
-        class TestCase(testtools.testcase):
-            def setUp(self):
-                self.useFixture(LockFixture)
-                super(TestCase, self).setUp()
-                    ...
-
-    This is because addCleanups are put on a LIFO queue that gets run after the
-    test method exits. (either by completing or raising an exception)
-    """
-    def __init__(self, name, lock_file_prefix=None):
-        self.mgr = lockutils.lock(name, lock_file_prefix, True)
-
-    def setUp(self):
-        super(LockFixture, self).setUp()
-        self.addCleanup(self.mgr.__exit__, None, None, None)
-        self.lock = self.mgr.__enter__()

designate/openstack/common/fixture/logging.py

@@ -1,34 +0,0 @@
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import fixtures
-
-
-def get_logging_handle_error_fixture():
-    """returns a fixture to make logging raise formatting exceptions.
-
-    Usage:
-    self.useFixture(logging.get_logging_handle_error_fixture())
-    """
-    return fixtures.MonkeyPatch('logging.Handler.handleError',
-                                _handleError)
-
-
-def _handleError(self, record):
-    """Monkey patch for logging.Handler.handleError.
-
-    The default handleError just logs the error to stderr but we want
-    the option of actually raising an exception.
-    """
-    raise

designate/openstack/common/fixture/mockpatch.py

@@ -1,62 +0,0 @@
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# Copyright 2013 Hewlett-Packard Development Company, L.P.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-##############################################################################
-##############################################################################
-##
-## DO NOT MODIFY THIS FILE
-##
-## This file is being graduated to the designatetest library. Please make all
-## changes there, and only backport critical fixes here. - dhellmann
-##
-##############################################################################
-##############################################################################
-
-import fixtures
-import mock
-
-
-class PatchObject(fixtures.Fixture):
-    """Deal with code around mock."""
-
-    def __init__(self, obj, attr, new=mock.DEFAULT, **kwargs):
-        self.obj = obj
-        self.attr = attr
-        self.kwargs = kwargs
-        self.new = new
-
-    def setUp(self):
-        super(PatchObject, self).setUp()
-        _p = mock.patch.object(self.obj, self.attr, self.new, **self.kwargs)
-        self.mock = _p.start()
-        self.addCleanup(_p.stop)
-
-
-class Patch(fixtures.Fixture):
-
-    """Deal with code around mock.patch."""
-
-    def __init__(self, obj, new=mock.DEFAULT, **kwargs):
-        self.obj = obj
-        self.kwargs = kwargs
-        self.new = new
-
-    def setUp(self):
-        super(Patch, self).setUp()
-        _p = mock.patch(self.obj, self.new, **self.kwargs)
-        self.mock = _p.start()
-        self.addCleanup(_p.stop)

designate/openstack/common/fixture/moxstubout.py

@@ -1,43 +0,0 @@
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# Copyright 2013 Hewlett-Packard Development Company, L.P.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-##############################################################################
-##############################################################################
-##
-## DO NOT MODIFY THIS FILE
-##
-## This file is being graduated to the designatetest library. Please make all
-## changes there, and only backport critical fixes here. - dhellmann
-##
-##############################################################################
-##############################################################################
-
-import fixtures
-from six.moves import mox
-
-
-class MoxStubout(fixtures.Fixture):
-    """Deal with code around mox and stubout as a fixture."""
-
-    def setUp(self):
-        super(MoxStubout, self).setUp()
-        # emulate some of the mox stuff, we can't use the metaclass
-        # because it screws with our generators
-        self.mox = mox.Mox()
-        self.stubs = self.mox.stubs
-        self.addCleanup(self.mox.UnsetStubs)
-        self.addCleanup(self.mox.VerifyAll)

designate/openstack/common/gettextutils.py

@@ -23,7 +23,6 @@ Usual usage in an openstack.common module:
 """
 
 import copy
-import functools
 import gettext
 import locale
 from logging import handlers
@@ -32,24 +31,115 @@ import os
 from babel import localedata
 import six
 
-_localedir = os.environ.get('designate'.upper() + '_LOCALEDIR')
-_t = gettext.translation('designate', localedir=_localedir, fallback=True)
-
-# We use separate translation catalogs for each log level, so set up a
-# mapping between the log level name and the translator. The domain
-# for the log level is project_name + "-log-" + log_level so messages
-# for each level end up in their own catalog.
-_t_log_levels = dict(
-    (level, gettext.translation('designate' + '-log-' + level,
-                                localedir=_localedir,
-                                fallback=True))
-    for level in ['info', 'warning', 'error', 'critical']
-)
-
 _AVAILABLE_LANGUAGES = {}
+
+# FIXME(dhellmann): Remove this when moving to oslo.i18n.
 USE_LAZY = False
 
 
+class TranslatorFactory(object):
+    """Create translator functions
+    """
+
+    def __init__(self, domain, localedir=None):
+        """Establish a set of translation functions for the domain.
+
+        :param domain: Name of translation domain,
+                       specifying a message catalog.
+        :type domain: str
+        :param lazy: Delays translation until a message is emitted.
+                     Defaults to False.
+        :type lazy: Boolean
+        :param localedir: Directory with translation catalogs.
+        :type localedir: str
+        """
+        self.domain = domain
+        if localedir is None:
+            localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
+        self.localedir = localedir
+
+    def _make_translation_func(self, domain=None):
+        """Return a new translation function ready for use.
+
+        Takes into account whether or not lazy translation is being
+        done.
+
+        The domain can be specified to override the default from the
+        factory, but the localedir from the factory is always used
+        because we assume the log-level translation catalogs are
+        installed in the same directory as the main application
+        catalog.
+
+        """
+        if domain is None:
+            domain = self.domain
+        t = gettext.translation(domain,
+                                localedir=self.localedir,
+                                fallback=True)
+        # Use the appropriate method of the translation object based
+        # on the python version.
+        m = t.gettext if six.PY3 else t.ugettext
+
+        def f(msg):
+            """oslo.i18n.gettextutils translation function."""
+            if USE_LAZY:
+                return Message(msg, domain=domain)
+            return m(msg)
+        return f
+
+    @property
+    def primary(self):
+        "The default translation function."
+        return self._make_translation_func()
+
+    def _make_log_translation_func(self, level):
+        return self._make_translation_func(self.domain + '-log-' + level)
+
+    @property
+    def log_info(self):
+        "Translate info-level log messages."
+        return self._make_log_translation_func('info')
+
+    @property
+    def log_warning(self):
+        "Translate warning-level log messages."
+        return self._make_log_translation_func('warning')
+
+    @property
+    def log_error(self):
+        "Translate error-level log messages."
+        return self._make_log_translation_func('error')
+
+    @property
+    def log_critical(self):
+        "Translate critical-level log messages."
+        return self._make_log_translation_func('critical')
+
+
+# NOTE(dhellmann): When this module moves out of the incubator into
+# oslo.i18n, these global variables can be moved to an integration
+# module within each application.
+
+# Create the global translation functions.
+_translators = TranslatorFactory('designate')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical
+
+# NOTE(dhellmann): End of globals that will move to the application's
+# integration module.
+
+
 def enable_lazy():
     """Convenience function for configuring _() to use lazy gettext
 
@@ -62,38 +152,7 @@ def enable_lazy():
     USE_LAZY = True
 
 
-def _(msg):
+def install(domain):
-    if USE_LAZY:
-        return Message(msg, domain='designate')
-    else:
-        if six.PY3:
-            return _t.gettext(msg)
-        return _t.ugettext(msg)
-
-
-def _log_translation(msg, level):
-    """Build a single translation of a log message
-    """
-    if USE_LAZY:
-        return Message(msg, domain='designate' + '-log-' + level)
-    else:
-        translator = _t_log_levels[level]
-        if six.PY3:
-            return translator.gettext(msg)
-        return translator.ugettext(msg)
-
-# Translators for log levels.
-#
-# The abbreviated names are meant to reflect the usual use of a short
-# name like '_'. The "L" is for "log" and the other letter comes from
-# the level.
-_LI = functools.partial(_log_translation, level='info')
-_LW = functools.partial(_log_translation, level='warning')
-_LE = functools.partial(_log_translation, level='error')
-_LC = functools.partial(_log_translation, level='critical')
-
-
-def install(domain, lazy=False):
     """Install a _() function using the given translation domain.
 
     Given a translation domain, install a _() function using gettext's
@@ -104,43 +163,14 @@ def install(domain, lazy=False):
     a translation-domain-specific environment variable (e.g.
     NOVA_LOCALEDIR).
 
+    Note that to enable lazy translation, enable_lazy must be
+    called.
+
     :param domain: the translation domain
-    :param lazy: indicates whether or not to install the lazy _() function.
-                 The lazy _() introduces a way to do deferred translation
-                 of messages by installing a _ that builds Message objects,
-                 instead of strings, which can then be lazily translated into
-                 any available locale.
     """
-    if lazy:
+    from six import moves
-        # NOTE(mrodden): Lazy gettext functionality.
+    tf = TranslatorFactory(domain)
-        #
+    moves.builtins.__dict__['_'] = tf.primary
-        # The following introduces a deferred way to do translations on
-        # messages in OpenStack. We override the standard _() function
-        # and % (format string) operation to build Message objects that can
-        # later be translated when we have more information.
-        def _lazy_gettext(msg):
-            """Create and return a Message object.
-
-            Lazy gettext function for a given domain, it is a factory method
-            for a project/module to get a lazy gettext function for its own
-            translation domain (i.e. nova, glance, cinder, etc.)
-
-            Message encapsulates a string so that we can translate
-            it later when needed.
-            """
-            return Message(msg, domain=domain)
-
-        from six import moves
-        moves.builtins.__dict__['_'] = _lazy_gettext
-    else:
-        localedir = '%s_LOCALEDIR' % domain.upper()
-        if six.PY3:
-            gettext.install(domain,
-                            localedir=os.environ.get(localedir))
-        else:
-            gettext.install(domain,
-                            localedir=os.environ.get(localedir),
-                            unicode=True)
 
 
 class Message(six.text_type):
@@ -324,8 +354,8 @@ def get_available_languages(domain):
                'zh_Hant_HK': 'zh_HK',
                'zh_Hant': 'zh_TW',
                'fil': 'tl_PH'}
-    for (locale, alias) in six.iteritems(aliases):
+    for (locale_, alias) in six.iteritems(aliases):
-        if locale in language_list and alias not in language_list:
+        if locale_ in language_list and alias not in language_list:
             language_list.append(alias)
 
     _AVAILABLE_LANGUAGES[domain] = language_list

designate/openstack/common/importutils.py

@@ -24,10 +24,10 @@ import traceback
 def import_class(import_str):
     """Returns a class from a string including module and class."""
     mod_str, _sep, class_str = import_str.rpartition('.')
+    __import__(mod_str)
     try:
-        __import__(mod_str)
         return getattr(sys.modules[mod_str], class_str)
-    except (ValueError, AttributeError):
+    except AttributeError:
         raise ImportError('Class %s cannot be found (%s)' %
                           (class_str,
                            traceback.format_exception(*sys.exc_info())))

designate/openstack/common/jsonutils.py

@@ -31,17 +31,29 @@ This module provides a few things:
 '''
 
 
+import codecs
 import datetime
 import functools
 import inspect
 import itertools
-import json
+import sys
+
+if sys.version_info < (2, 7):
+    # On Python <= 2.6, json module is not C boosted, so try to use
+    # simplejson module if available
+    try:
+        import simplejson as json
+    except ImportError:
+        import json
+else:
+    import json
 
 import six
 import six.moves.xmlrpc_client as xmlrpclib
 
 from designate.openstack.common import gettextutils
 from designate.openstack.common import importutils
+from designate.openstack.common import strutils
 from designate.openstack.common import timeutils
 
 netaddr = importutils.try_import("netaddr")
@@ -156,12 +168,16 @@ def dumps(value, default=to_primitive, **kwargs):
     return json.dumps(value, default=default, **kwargs)
 
 
-def loads(s):
+def dump(obj, fp, *args, **kwargs):
-    return json.loads(s)
+    return json.dump(obj, fp, *args, **kwargs)
 
 
-def load(s):
+def loads(s, encoding='utf-8', **kwargs):
-    return json.load(s)
+    return json.loads(strutils.safe_decode(s, encoding), **kwargs)
+
+
+def load(fp, encoding='utf-8', **kwargs):
+    return json.load(codecs.getreader(encoding)(fp), **kwargs)
 
 
 try:

designate/openstack/common/lockutils.py

@@ -1,377 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import contextlib
-import errno
-import fcntl
-import functools
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import threading
-import time
-import weakref
-
-from oslo.config import cfg
-
-from designate.openstack.common import fileutils
-from designate.openstack.common.gettextutils import _, _LE, _LI
-from designate.openstack.common import log as logging
-
-
-LOG = logging.getLogger(__name__)
-
-
-util_opts = [
-    cfg.BoolOpt('disable_process_locking', default=False,
-                help='Whether to disable inter-process locks'),
-    cfg.StrOpt('lock_path',
-               default=os.environ.get("DESIGNATE_LOCK_PATH"),
-               help=('Directory to use for lock files.'))
-]
-
-
-CONF = cfg.CONF
-CONF.register_opts(util_opts)
-
-
-def set_defaults(lock_path):
-    cfg.set_defaults(util_opts, lock_path=lock_path)
-
-
-class _FileLock(object):
-    """Lock implementation which allows multiple locks, working around
-    issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
-    not require any cleanup. Since the lock is always held on a file
-    descriptor rather than outside of the process, the lock gets dropped
-    automatically if the process crashes, even if __exit__ is not executed.
-
-    There are no guarantees regarding usage by multiple green threads in a
-    single process here. This lock works only between processes. Exclusive
-    access between local threads should be achieved using the semaphores
-    in the @synchronized decorator.
-
-    Note these locks are released when the descriptor is closed, so it's not
-    safe to close the file descriptor while another green thread holds the
-    lock. Just opening and closing the lock file can break synchronisation,
-    so lock files must be accessed only using this abstraction.
-    """
-
-    def __init__(self, name):
-        self.lockfile = None
-        self.fname = name
-
-    def acquire(self):
-        basedir = os.path.dirname(self.fname)
-
-        if not os.path.exists(basedir):
-            fileutils.ensure_tree(basedir)
-            LOG.info(_LI('Created lock path: %s'), basedir)
-
-        self.lockfile = open(self.fname, 'w')
-
-        while True:
-            try:
-                # Using non-blocking locks since green threads are not
-                # patched to deal with blocking locking calls.
-                # Also upon reading the MSDN docs for locking(), it seems
-                # to have a laughable 10 attempts "blocking" mechanism.
-                self.trylock()
-                LOG.debug('Got file lock "%s"', self.fname)
-                return True
-            except IOError as e:
-                if e.errno in (errno.EACCES, errno.EAGAIN):
-                    # external locks synchronise things like iptables
-                    # updates - give it some time to prevent busy spinning
-                    time.sleep(0.01)
-                else:
-                    raise threading.ThreadError(_("Unable to acquire lock on"
-                                                  " `%(filename)s` due to"
-                                                  " %(exception)s") %
-                                                {
-                                                    'filename': self.fname,
-                                                    'exception': e,
-                                                })
-
-    def __enter__(self):
-        self.acquire()
-        return self
-
-    def release(self):
-        try:
-            self.unlock()
-            self.lockfile.close()
-            LOG.debug('Released file lock "%s"', self.fname)
-        except IOError:
-            LOG.exception(_LE("Could not release the acquired lock `%s`"),
-                          self.fname)
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.release()
-
-    def exists(self):
-        return os.path.exists(self.fname)
-
-    def trylock(self):
-        raise NotImplementedError()
-
-    def unlock(self):
-        raise NotImplementedError()
-
-
-class _WindowsLock(_FileLock):
-    def trylock(self):
-        msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
-
-    def unlock(self):
-        msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
-
-
-class _FcntlLock(_FileLock):
-    def trylock(self):
-        fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
-
-    def unlock(self):
-        fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
-
-
-class _PosixLock(object):
-    def __init__(self, name):
-        # Hash the name because it's not valid to have POSIX semaphore
-        # names with things like / in them. Then use base64 to encode
-        # the digest() instead taking the hexdigest() because the
-        # result is shorter and most systems can't have shm sempahore
-        # names longer than 31 characters.
-        h = hashlib.sha1()
-        h.update(name.encode('ascii'))
-        self.name = str((b'/' + base64.urlsafe_b64encode(
-            h.digest())).decode('ascii'))
-
-    def acquire(self, timeout=None):
-        self.semaphore = posix_ipc.Semaphore(self.name,
-                                             flags=posix_ipc.O_CREAT,
-                                             initial_value=1)
-        self.semaphore.acquire(timeout)
-        return self
-
-    def __enter__(self):
-        self.acquire()
-        return self
-
-    def release(self):
-        self.semaphore.release()
-        self.semaphore.close()
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.release()
-
-    def exists(self):
-        try:
-            semaphore = posix_ipc.Semaphore(self.name)
-        except posix_ipc.ExistentialError:
-            return False
-        else:
-            semaphore.close()
-        return True
-
-
-if os.name == 'nt':
-    import msvcrt
-    InterProcessLock = _WindowsLock
-    FileLock = _WindowsLock
-else:
-    import base64
-    import hashlib
-    import posix_ipc
-    InterProcessLock = _PosixLock
-    FileLock = _FcntlLock
-
-_semaphores = weakref.WeakValueDictionary()
-_semaphores_lock = threading.Lock()
-
-
-def _get_lock_path(name, lock_file_prefix, lock_path=None):
-    # NOTE(mikal): the lock name cannot contain directory
-    # separators
-    name = name.replace(os.sep, '_')
-    if lock_file_prefix:
-        sep = '' if lock_file_prefix.endswith('-') else '-'
-        name = '%s%s%s' % (lock_file_prefix, sep, name)
-
-    local_lock_path = lock_path or CONF.lock_path
-
-    if not local_lock_path:
-        # NOTE(bnemec): Create a fake lock path for posix locks so we don't
-        # unnecessarily raise the RequiredOptError below.
-        if InterProcessLock is not _PosixLock:
-            raise cfg.RequiredOptError('lock_path')
-        local_lock_path = 'posixlock:/'
-
-    return os.path.join(local_lock_path, name)
-
-
-def external_lock(name, lock_file_prefix=None, lock_path=None):
-    LOG.debug('Attempting to grab external lock "%(lock)s"',
-              {'lock': name})
-
-    lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
-
-    # NOTE(bnemec): If an explicit lock_path was passed to us then it
-    # means the caller is relying on file-based locking behavior, so
-    # we can't use posix locks for those calls.
-    if lock_path:
-        return FileLock(lock_file_path)
-    return InterProcessLock(lock_file_path)
-
-
-def remove_external_lock_file(name, lock_file_prefix=None):
-    """Remove a external lock file when it's not used anymore
-    This will be helpful when we have a lot of lock files
-    """
-    with internal_lock(name):
-        lock_file_path = _get_lock_path(name, lock_file_prefix)
-        try:
-            os.remove(lock_file_path)
-        except OSError:
-            LOG.info(_LI('Failed to remove file %(file)s'),
-                     {'file': lock_file_path})
-
-
-def internal_lock(name):
-    with _semaphores_lock:
-        try:
-            sem = _semaphores[name]
-        except KeyError:
-            sem = threading.Semaphore()
-            _semaphores[name] = sem
-
-    LOG.debug('Got semaphore "%(lock)s"', {'lock': name})
-    return sem
-
-
-@contextlib.contextmanager
-def lock(name, lock_file_prefix=None, external=False, lock_path=None):
-    """Context based lock
-
-    This function yields a `threading.Semaphore` instance (if we don't use
-    eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
-    True, in which case, it'll yield an InterProcessLock instance.
-
-    :param lock_file_prefix: The lock_file_prefix argument is used to provide
-      lock files on disk with a meaningful prefix.
-
-    :param external: The external keyword argument denotes whether this lock
-      should work across multiple processes. This means that if two different
-      workers both run a a method decorated with @synchronized('mylock',
-      external=True), only one of them will execute at a time.
-    """
-    int_lock = internal_lock(name)
-    with int_lock:
-        if external and not CONF.disable_process_locking:
-            ext_lock = external_lock(name, lock_file_prefix, lock_path)
-            with ext_lock:
-                yield ext_lock
-        else:
-            yield int_lock
-
-
-def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
-    """Synchronization decorator.
-
-    Decorating a method like so::
-
-        @synchronized('mylock')
-        def foo(self, *args):
-           ...
-
-    ensures that only one thread will execute the foo method at a time.
-
-    Different methods can share the same lock::
-
-        @synchronized('mylock')
-        def foo(self, *args):
-           ...
-
-        @synchronized('mylock')
-        def bar(self, *args):
-           ...
-
-    This way only one of either foo or bar can be executing at a time.
-    """
-
-    def wrap(f):
-        @functools.wraps(f)
-        def inner(*args, **kwargs):
-            try:
-                with lock(name, lock_file_prefix, external, lock_path):
-                    LOG.debug('Got semaphore / lock "%(function)s"',
-                              {'function': f.__name__})
-                    return f(*args, **kwargs)
-            finally:
-                LOG.debug('Semaphore / lock released "%(function)s"',
-                          {'function': f.__name__})
-        return inner
-    return wrap
-
-
-def synchronized_with_prefix(lock_file_prefix):
-    """Partial object generator for the synchronization decorator.
-
-    Redefine @synchronized in each project like so::
-
-        (in nova/utils.py)
-        from nova.openstack.common import lockutils
-
-        synchronized = lockutils.synchronized_with_prefix('nova-')
-
-
-        (in nova/foo.py)
-        from nova import utils
-
-        @utils.synchronized('mylock')
-        def bar(self, *args):
-           ...
-
-    The lock_file_prefix argument is used to provide lock files on disk with a
-    meaningful prefix.
-    """
-
-    return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
-
-
-def main(argv):
-    """Create a dir for locks and pass it to command from arguments
-
-    If you run this:
-    python -m openstack.common.lockutils python setup.py testr <etc>
-
-    a temporary directory will be created for all your locks and passed to all
-    your tests in an environment variable. The temporary dir will be deleted
-    afterwards and the return value will be preserved.
-    """
-
-    lock_dir = tempfile.mkdtemp()
-    os.environ["DESIGNATE_LOCK_PATH"] = lock_dir
-    try:
-        ret_val = subprocess.call(argv[1:])
-    finally:
-        shutil.rmtree(lock_dir, ignore_errors=True)
-    return ret_val
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv))

designate/openstack/common/log.py

@@ -33,7 +33,6 @@ import logging
 import logging.config
 import logging.handlers
 import os
-import re
 import sys
 import traceback
 
@@ -45,27 +44,13 @@ from designate.openstack.common.gettextutils import _
 from designate.openstack.common import importutils
 from designate.openstack.common import jsonutils
 from designate.openstack.common import local
+# NOTE(flaper87): Pls, remove when graduating this module
+# from the incubator.
+from designate.openstack.common.strutils import mask_password  # noqa
 
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
 
-_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
-
-# NOTE(ldbragst): Let's build a list of regex objects using the list of
-# _SANITIZE_KEYS we already have. This way, we only have to add the new key
-# to the list of _SANITIZE_KEYS and we can generate regular expressions
-# for XML and JSON automatically.
-_SANITIZE_PATTERNS = []
-_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
-                    r'(<%(key)s>).*?(</%(key)s>)',
-                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
-                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
-
-for key in _SANITIZE_KEYS:
-    for pattern in _FORMAT_PATTERNS:
-        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
-        _SANITIZE_PATTERNS.append(reg_ex)
-
 
 common_cli_opts = [
     cfg.BoolOpt('debug',
@@ -84,14 +69,11 @@ logging_cli_opts = [
     cfg.StrOpt('log-config-append',
                metavar='PATH',
                deprecated_name='log-config',
-               help='The name of logging configuration file. It does not '
+               help='The name of a logging configuration file. This file '
-                    'disable existing loggers, but just appends specified '
+                    'is appended to any existing logging configuration '
-                    'logging configuration to any other existing logging '
+                    'files. For details about logging configuration files, '
-                    'options. Please see the Python logging module '
+                    'see the Python logging module documentation.'),
-                    'documentation for details on logging configuration '
-                    'files.'),
     cfg.StrOpt('log-format',
-               default=None,
                metavar='FORMAT',
                help='DEPRECATED. '
                     'A logging.Formatter log message format string which may '
@@ -103,7 +85,7 @@ logging_cli_opts = [
                default=_DEFAULT_LOG_DATE_FORMAT,
                metavar='DATE_FORMAT',
                help='Format string for %%(asctime)s in log records. '
-                    'Default: %(default)s'),
+                    'Default: %(default)s .'),
     cfg.StrOpt('log-file',
                metavar='PATH',
                deprecated_name='logfile',
@@ -112,80 +94,76 @@ logging_cli_opts = [
     cfg.StrOpt('log-dir',
                deprecated_name='logdir',
                help='(Optional) The base directory used for relative '
-                    '--log-file paths'),
+                    '--log-file paths.'),
     cfg.BoolOpt('use-syslog',
                 default=False,
                 help='Use syslog for logging. '
                      'Existing syslog format is DEPRECATED during I, '
-                     'and then will be changed in J to honor RFC5424'),
+                     'and will change in J to honor RFC5424.'),
     cfg.BoolOpt('use-syslog-rfc-format',
                 # TODO(bogdando) remove or use True after existing
                 #    syslog format deprecation in J
                 default=False,
-                help='(Optional) Use syslog rfc5424 format for logging. '
+                help='(Optional) Enables or disables syslog rfc5424 format '
-                     'If enabled, will add APP-NAME (RFC5424) before the '
+                     'for logging. If enabled, prefixes the MSG part of the '
-                     'MSG part of the syslog message.  The old format '
+                     'syslog message with APP-NAME (RFC5424). The '
-                     'without APP-NAME is deprecated in I, '
+                     'format without the APP-NAME is deprecated in I, '
                      'and will be removed in J.'),
     cfg.StrOpt('syslog-log-facility',
                default='LOG_USER',
-               help='Syslog facility to receive log lines')
+               help='Syslog facility to receive log lines.')
 ]
 
 generic_log_opts = [
     cfg.BoolOpt('use_stderr',
                 default=True,
-                help='Log output to standard error')
+                help='Log output to standard error.')
 ]
 
+DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
+                      'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
+                      'oslo.messaging=INFO', 'iso8601=WARN',
+                      'requests.packages.urllib3.connectionpool=WARN',
+                      'urllib3.connectionpool=WARN', 'websocket=WARN']
+
 log_opts = [
     cfg.StrOpt('logging_context_format_string',
                default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
                        '%(name)s [%(request_id)s %(user_identity)s] '
                        '%(instance)s%(message)s',
-               help='Format string to use for log messages with context'),
+               help='Format string to use for log messages with context.'),
     cfg.StrOpt('logging_default_format_string',
                default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
                        '%(name)s [-] %(instance)s%(message)s',
-               help='Format string to use for log messages without context'),
+               help='Format string to use for log messages without context.'),
     cfg.StrOpt('logging_debug_format_suffix',
                default='%(funcName)s %(pathname)s:%(lineno)d',
-               help='Data to append to log format when level is DEBUG'),
+               help='Data to append to log format when level is DEBUG.'),
     cfg.StrOpt('logging_exception_prefix',
                default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
                '%(instance)s',
-               help='Prefix each line of exception output with this format'),
+               help='Prefix each line of exception output with this format.'),
     cfg.ListOpt('default_log_levels',
-                default=[
+                default=DEFAULT_LOG_LEVELS,
-                    'amqp=WARN',
+                help='List of logger=LEVEL pairs.'),
-                    'amqplib=WARN',
-                    'boto=WARN',
-                    'qpid=WARN',
-                    'sqlalchemy=WARN',
-                    'suds=INFO',
-                    'oslo.messaging=INFO',
-                    'iso8601=WARN',
-                    'requests.packages.urllib3.connectionpool=WARN'
-                ],
-                help='List of logger=LEVEL pairs'),
     cfg.BoolOpt('publish_errors',
                 default=False,
-                help='Publish error events'),
+                help='Enables or disables publication of error events.'),
     cfg.BoolOpt('fatal_deprecations',
                 default=False,
-                help='Make deprecations fatal'),
+                help='Enables or disables fatal status of deprecations.'),
 
     # NOTE(mikal): there are two options here because sometimes we are handed
     # a full instance (and could include more information), and other times we
     # are just handed a UUID for the instance.
     cfg.StrOpt('instance_format',
                default='[instance: %(uuid)s] ',
-               help='If an instance is passed with the log message, format '
+               help='The format for an instance that is passed with the log '
-                    'it like this'),
+                    'message.'),
     cfg.StrOpt('instance_uuid_format',
                default='[instance: %(uuid)s] ',
-               help='If an instance UUID is passed with the log message, '
+               help='The format for an instance UUID that is passed with the '
-                    'format it like this'),
+                    'log message.'),
 ]
 
 CONF = cfg.CONF
@@ -244,40 +222,6 @@ def _get_log_file_path(binary=None):
     return None
 
 
-def mask_password(message, secret="***"):
-    """Replace password with 'secret' in message.
-
-    :param message: The string which includes security information.
-    :param secret: value with which to replace passwords.
-    :returns: The unicode value of message with the password fields masked.
-
-    For example:
-
-    >>> mask_password("'adminPass' : 'aaaaa'")
-    "'adminPass' : '***'"
-    >>> mask_password("'admin_pass' : 'aaaaa'")
-    "'admin_pass' : '***'"
-    >>> mask_password('"password" : "aaaaa"')
-    '"password" : "***"'
-    >>> mask_password("'original_password' : 'aaaaa'")
-    "'original_password' : '***'"
-    >>> mask_password("u'original_password' :   u'aaaaa'")
-    "u'original_password' :   u'***'"
-    """
-    message = six.text_type(message)
-
-    # NOTE(ldbragst): Check to see if anything in message contains any key
-    # specified in _SANITIZE_KEYS, if not then just return the message since
-    # we don't have to mask any passwords.
-    if not any(key in message for key in _SANITIZE_KEYS):
-        return message
-
-    secret = r'\g<1>' + secret + r'\g<2>'
-    for pattern in _SANITIZE_PATTERNS:
-        message = re.sub(pattern, secret, message)
-    return message
-
-
 class BaseLoggerAdapter(logging.LoggerAdapter):
 
     def audit(self, msg, *args, **kwargs):
@@ -295,6 +239,11 @@ class LazyAdapter(BaseLoggerAdapter):
     def logger(self):
         if not self._logger:
             self._logger = getLogger(self.name, self.version)
+            if six.PY3:
+                # In Python 3, the code fails because the 'manager' attribute
+                # cannot be found when using a LoggerAdapter as the
+                # underlying logger. Work around this issue.
+                self._logger.manager = self._logger.logger.manager
         return self._logger
 
 
@@ -424,9 +373,7 @@ class JSONFormatter(logging.Formatter):
 
 def _create_logging_excepthook(product_name):
     def logging_excepthook(exc_type, value, tb):
-        extra = {}
+        extra = {'exc_info': (exc_type, value, tb)}
-        if CONF.verbose or CONF.debug:
-            extra['exc_info'] = (exc_type, value, tb)
         getLogger(product_name).critical(
             "".join(traceback.format_exception_only(exc_type, value)),
             **extra)
@@ -450,8 +397,8 @@ def _load_log_config(log_config_append):
     try:
         logging.config.fileConfig(log_config_append,
                                   disable_existing_loggers=False)
-    except moves.configparser.Error as exc:
+    except (moves.configparser.Error, KeyError) as exc:
-        raise LogConfigError(log_config_append, str(exc))
+        raise LogConfigError(log_config_append, six.text_type(exc))
 
 
 def setup(product_name, version='unknown'):
@@ -463,10 +410,20 @@ def setup(product_name, version='unknown'):
     sys.excepthook = _create_logging_excepthook(product_name)
 
 
-def set_defaults(logging_context_format_string):
+def set_defaults(logging_context_format_string=None,
-    cfg.set_defaults(log_opts,
+                 default_log_levels=None):
-                     logging_context_format_string=
+    # Just in case the caller is not setting the
-                     logging_context_format_string)
+    # default_log_level. This is insurance because
+    # we introduced the default_log_level parameter
+    # later in a backwards in-compatible change
+    if default_log_levels is not None:
+        cfg.set_defaults(
+                log_opts,
+                default_log_levels=default_log_levels)
+    if logging_context_format_string is not None:
+        cfg.set_defaults(
+                log_opts,
+                logging_context_format_string=logging_context_format_string)
 
 
 def _find_facility_from_conf():
@@ -496,10 +453,16 @@ def _find_facility_from_conf():
 class RFCSysLogHandler(logging.handlers.SysLogHandler):
     def __init__(self, *args, **kwargs):
         self.binary_name = _get_binary_name()
-        super(RFCSysLogHandler, self).__init__(*args, **kwargs)
+        # Do not use super() unless type(logging.handlers.SysLogHandler)
+        #  is 'type' (Python 2.7).
+        # Use old style calls, if the type is 'classobj' (Python 2.6)
+        logging.handlers.SysLogHandler.__init__(self, *args, **kwargs)
 
     def format(self, record):
-        msg = super(RFCSysLogHandler, self).format(record)
+        # Do not use super() unless type(logging.handlers.SysLogHandler)
+        #  is 'type' (Python 2.7).
+        # Use old style calls, if the type is 'classobj' (Python 2.6)
+        msg = logging.handlers.SysLogHandler.format(self, record)
         msg = self.binary_name + ' ' + msg
         return msg
 
@@ -537,9 +500,14 @@ def _setup_logging_from_conf(project, version):
         log_root.addHandler(streamlog)
 
     if CONF.publish_errors:
-        handler = importutils.import_object(
+        try:
-            "designate.openstack.common.log_handler.PublishErrorsHandler",
+            handler = importutils.import_object(
-            logging.ERROR)
+                "designate.openstack.common.log_handler.PublishErrorsHandler",
+                logging.ERROR)
+        except ImportError:
+            handler = importutils.import_object(
+                "oslo.messaging.notify.log_handler.PublishErrorsHandler",
+                logging.ERROR)
         log_root.addHandler(handler)
 
     datefmt = CONF.log_date_format
@@ -565,9 +533,15 @@ def _setup_logging_from_conf(project, version):
 
     for pair in CONF.default_log_levels:
         mod, _sep, level_name = pair.partition('=')
-        level = logging.getLevelName(level_name)
         logger = logging.getLogger(mod)
-        logger.setLevel(level)
+        # NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name
+        # to integer code.
+        if sys.version_info < (2, 7):
+            level = logging.getLevelName(level_name)
+            logger.setLevel(level)
+        else:
+            logger.setLevel(level_name)
+
 
 _loggers = {}
 
@@ -656,14 +630,19 @@ class ContextFormatter(logging.Formatter):
                 record.__dict__[key] = ''
 
         if record.__dict__.get('request_id'):
-            self._fmt = CONF.logging_context_format_string
+            fmt = CONF.logging_context_format_string
         else:
-            self._fmt = CONF.logging_default_format_string
+            fmt = CONF.logging_default_format_string
 
         if (record.levelno == logging.DEBUG and
                 CONF.logging_debug_format_suffix):
-            self._fmt += " " + CONF.logging_debug_format_suffix
+            fmt += " " + CONF.logging_debug_format_suffix
 
+        if sys.version_info < (3, 2):
+            self._fmt = fmt
+        else:
+            self._style = logging.PercentStyle(fmt)
+            self._fmt = self._style._fmt
         # Cache this on the record, Logger will respect our formatted copy
         if record.exc_info:
             record.exc_text = self.formatException(record.exc_info, record)

designate/openstack/common/loopingcall.py

@@ -16,31 +16,36 @@
 #    under the License.
 
 import sys
+import time
 
 from eventlet import event
 from eventlet import greenthread
 
 from designate.openstack.common.gettextutils import _LE, _LW
 from designate.openstack.common import log as logging
-from designate.openstack.common import timeutils
 
 LOG = logging.getLogger(__name__)
 
+# NOTE(zyluo): This lambda function was declared to avoid mocking collisions
+#              with time.time() called in the standard logging module
+#              during unittests.
+_ts = lambda: time.time()
+
 
 class LoopingCallDone(Exception):
-    """Exception to break out and stop a LoopingCall.
+    """Exception to break out and stop a LoopingCallBase.
 
-    The poll-function passed to LoopingCall can raise this exception to
+    The poll-function passed to LoopingCallBase can raise this exception to
     break out of the loop normally. This is somewhat analogous to
     StopIteration.
 
     An optional return-value can be included as the argument to the exception;
-    this return-value will be returned by LoopingCall.wait()
+    this return-value will be returned by LoopingCallBase.wait()
 
     """
 
     def __init__(self, retvalue=True):
-        """:param retvalue: Value that LoopingCall.wait() should return."""
+        """:param retvalue: Value that LoopingCallBase.wait() should return."""
         self.retvalue = retvalue
 
 
@@ -72,16 +77,17 @@ class FixedIntervalLoopingCall(LoopingCallBase):
 
             try:
                 while self._running:
-                    start = timeutils.utcnow()
+                    start = _ts()
                     self.f(*self.args, **self.kw)
-                    end = timeutils.utcnow()
+                    end = _ts()
                     if not self._running:
                         break
-                    delay = interval - timeutils.delta_seconds(start, end)
+                    delay = end - start - interval
-                    if delay <= 0:
+                    if delay > 0:
-                        LOG.warn(_LW('task run outlasted interval by %s sec') %
+                        LOG.warn(_LW('task %(func_name)s run outlasted '
-                                 -delay)
+                                     'interval by %(delay).2f sec'),
-                    greenthread.sleep(delay if delay > 0 else 0)
+                                 {'func_name': repr(self.f), 'delay': delay})
+                    greenthread.sleep(-delay if delay < 0 else 0)
             except LoopingCallDone as e:
                 self.stop()
                 done.send(e.retvalue)
@@ -98,11 +104,6 @@ class FixedIntervalLoopingCall(LoopingCallBase):
         return self.done
 
 
-# TODO(mikal): this class name is deprecated in Havana and should be removed
-# in the I release
-LoopingCall = FixedIntervalLoopingCall
-
-
 class DynamicLoopingCall(LoopingCallBase):
     """A looping call which sleeps until the next known event.
 
@@ -126,8 +127,9 @@ class DynamicLoopingCall(LoopingCallBase):
 
                     if periodic_interval_max is not None:
                         idle = min(idle, periodic_interval_max)
-                    LOG.debug('Dynamic looping call sleeping for %.02f '
+                    LOG.debug('Dynamic looping call %(func_name)s sleeping '
-                              'seconds', idle)
+                              'for %(idle).02f seconds',
+                              {'func_name': repr(self.f), 'idle': idle})
                     greenthread.sleep(idle)
             except LoopingCallDone as e:
                 self.stop()

designate/openstack/common/network_utils.py

@@ -1,89 +0,0 @@
-# Copyright 2012 OpenStack Foundation.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-Network-related utilities and helper functions.
-"""
-
-# TODO(jd) Use six.moves once
-# https://bitbucket.org/gutworth/six/pull-request/28
-# is merged
-try:
-    import urllib.parse
-    SplitResult = urllib.parse.SplitResult
-except ImportError:
-    import urlparse
-    SplitResult = urlparse.SplitResult
-
-from six.moves.urllib import parse
-
-
-def parse_host_port(address, default_port=None):
-    """Interpret a string as a host:port pair.
-
-    An IPv6 address MUST be escaped if accompanied by a port,
-    because otherwise ambiguity ensues: 2001:db8:85a3::8a2e:370:7334
-    means both [2001:db8:85a3::8a2e:370:7334] and
-    [2001:db8:85a3::8a2e:370]:7334.
-
-    >>> parse_host_port('server01:80')
-    ('server01', 80)
-    >>> parse_host_port('server01')
-    ('server01', None)
-    >>> parse_host_port('server01', default_port=1234)
-    ('server01', 1234)
-    >>> parse_host_port('[::1]:80')
-    ('::1', 80)
-    >>> parse_host_port('[::1]')
-    ('::1', None)
-    >>> parse_host_port('[::1]', default_port=1234)
-    ('::1', 1234)
-    >>> parse_host_port('2001:db8:85a3::8a2e:370:7334', default_port=1234)
-    ('2001:db8:85a3::8a2e:370:7334', 1234)
-
-    """
-    if address[0] == '[':
-        # Escaped ipv6
-        _host, _port = address[1:].split(']')
-        host = _host
-        if ':' in _port:
-            port = _port.split(':')[1]
-        else:
-            port = default_port
-    else:
-        if address.count(':') == 1:
-            host, port = address.split(':')
-        else:
-            # 0 means ipv4, >1 means ipv6.
-            # We prohibit unescaped ipv6 addresses with port.
-            host = address
-            port = default_port
-
-    return (host, None if port is None else int(port))
-
-
-def urlsplit(url, scheme='', allow_fragments=True):
-    """Parse a URL using urlparse.urlsplit(), splitting query and fragments.
-    This function papers over Python issue9374 when needed.
-
-    The parameters are the same as urlparse.urlsplit.
-    """
-    scheme, netloc, path, query, fragment = parse.urlsplit(
-        url, scheme, allow_fragments)
-    if allow_fragments and '#' in path:
-        path, fragment = path.split('#', 1)
-    if '?' in path:
-        path, query = path.split('?', 1)
-    return SplitResult(scheme, netloc, path, query, fragment)

designate/openstack/common/policy.py

@@ -216,6 +216,7 @@ class Enforcer(object):
     def clear(self):
         """Clears Enforcer rules, policy's cache and policy's path."""
         self.set_rules({})
+        fileutils.delete_cached_file(self.policy_path)
         self.default_rule = None
         self.policy_path = None
 
@@ -283,10 +284,6 @@ class Enforcer(object):
                 from the expression.
         """
 
-        # NOTE(flaper87): Not logging target or creds to avoid
-        # potential security issues.
-        LOG.debug("Rule %s will be now enforced" % rule)
-
         self.load_rules()
 
         # Allow the rule to be a Check tree

designate/openstack/common/processutils.py

@@ -18,7 +18,8 @@ System-level utilities and helper functions.
 """
 
 import errno
-import logging as stdlib_logging
+import logging
+import multiprocessing
 import os
 import random
 import shlex
@@ -29,7 +30,7 @@ from eventlet import greenthread
 import six
 
 from designate.openstack.common.gettextutils import _
-from designate.openstack.common import log as logging
+from designate.openstack.common import strutils
 
 
 LOG = logging.getLogger(__name__)
@@ -90,6 +91,9 @@ def execute(*cmd, **kwargs):
     :type cmd:              string
     :param process_input:   Send to opened process.
     :type process_input:    string
+    :param env_variables:   Environment variables and their values that
+                            will be set for the process.
+    :type env_variables:    dict
     :param check_exit_code: Single bool, int, or list of allowed exit
                             codes.  Defaults to [0].  Raise
                             :class:`ProcessExecutionError` unless
@@ -111,8 +115,7 @@ def execute(*cmd, **kwargs):
                             execute this command. Defaults to false.
     :type shell:            boolean
     :param loglevel:        log level for execute commands.
-    :type loglevel:         int.  (Should be stdlib_logging.DEBUG or
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
-                            stdlib_logging.INFO)
     :returns:               (stdout, stderr) from process execution
     :raises:                :class:`UnknownArgumentError` on
                             receiving unknown arguments
@@ -120,6 +123,7 @@ def execute(*cmd, **kwargs):
     """
 
     process_input = kwargs.pop('process_input', None)
+    env_variables = kwargs.pop('env_variables', None)
     check_exit_code = kwargs.pop('check_exit_code', [0])
     ignore_exit_code = False
     delay_on_retry = kwargs.pop('delay_on_retry', True)
@@ -127,7 +131,7 @@ def execute(*cmd, **kwargs):
     run_as_root = kwargs.pop('run_as_root', False)
     root_helper = kwargs.pop('root_helper', '')
     shell = kwargs.pop('shell', False)
-    loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+    loglevel = kwargs.pop('loglevel', logging.DEBUG)
 
     if isinstance(check_exit_code, bool):
         ignore_exit_code = not check_exit_code
@@ -136,8 +140,7 @@ def execute(*cmd, **kwargs):
         check_exit_code = [check_exit_code]
 
     if kwargs:
-        raise UnknownArgumentError(_('Got unknown keyword args '
+        raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
-                                     'to utils.execute: %r') % kwargs)
 
     if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
         if not root_helper:
@@ -152,7 +155,7 @@ def execute(*cmd, **kwargs):
         attempts -= 1
         try:
             LOG.log(loglevel, 'Running cmd (subprocess): %s',
-                    ' '.join(cmd))
+                    strutils.mask_password(' '.join(cmd)))
             _PIPE = subprocess.PIPE  # pylint: disable=E1101
 
             if os.name == 'nt':
@@ -168,7 +171,8 @@ def execute(*cmd, **kwargs):
                                    stderr=_PIPE,
                                    close_fds=close_fds,
                                    preexec_fn=preexec_fn,
-                                   shell=shell)
+                                   shell=shell,
+                                   env=env_variables)
             result = None
             for _i in six.moves.range(20):
                 # NOTE(russellb) 20 is an arbitrary number of retries to
@@ -224,7 +228,7 @@ def trycmd(*args, **kwargs):
         out, err = execute(*args, **kwargs)
         failed = False
     except ProcessExecutionError as exn:
-        out, err = '', str(exn)
+        out, err = '', six.text_type(exn)
         failed = True
 
     if not failed and discard_warnings and err:
@@ -265,3 +269,15 @@ def ssh_execute(ssh, cmd, process_input=None,
                                         cmd=cmd)
 
     return (stdout, stderr)
+
+
+def get_worker_count():
+    """Utility to get the default worker count.
+
+    @return: The number of CPUs if that can be determined, else a default
+             worker count of 1 is returned.
+    """
+    try:
+        return multiprocessing.cpu_count()
+    except NotImplementedError:
+        return 1

designate/openstack/common/py3kcompat/urlutils.py

@@ -1,67 +0,0 @@
-#
-# Copyright 2013 Canonical Ltd.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-#
-
-"""
-Python2/Python3 compatibility layer for OpenStack
-"""
-
-import six
-
-if six.PY3:
-    # python3
-    import urllib.error
-    import urllib.parse
-    import urllib.request
-
-    urlencode = urllib.parse.urlencode
-    urljoin = urllib.parse.urljoin
-    quote = urllib.parse.quote
-    quote_plus = urllib.parse.quote_plus
-    parse_qsl = urllib.parse.parse_qsl
-    unquote = urllib.parse.unquote
-    unquote_plus = urllib.parse.unquote_plus
-    urlparse = urllib.parse.urlparse
-    urlsplit = urllib.parse.urlsplit
-    urlunsplit = urllib.parse.urlunsplit
-    SplitResult = urllib.parse.SplitResult
-
-    urlopen = urllib.request.urlopen
-    URLError = urllib.error.URLError
-    pathname2url = urllib.request.pathname2url
-else:
-    # python2
-    import urllib
-    import urllib2
-    import urlparse
-
-    urlencode = urllib.urlencode
-    quote = urllib.quote
-    quote_plus = urllib.quote_plus
-    unquote = urllib.unquote
-    unquote_plus = urllib.unquote_plus
-
-    parse = urlparse
-    parse_qsl = parse.parse_qsl
-    urljoin = parse.urljoin
-    urlparse = parse.urlparse
-    urlsplit = parse.urlsplit
-    urlunsplit = parse.urlunsplit
-    SplitResult = parse.SplitResult
-
-    urlopen = urllib2.urlopen
-    URLError = urllib2.URLError
-    pathname2url = urllib.pathname2url

designate/openstack/common/service.py

@@ -190,6 +190,7 @@ class ServiceLauncher(Launcher):
         return status, signo
 
     def wait(self, ready_callback=None):
+        systemd.notify_once()
         while True:
             self.handle_signal()
             status, signo = self._wait_for_exit_or_signal(ready_callback)
@@ -267,7 +268,7 @@ class ProcessLauncher(object):
             launcher.wait()
         except SignalExit as exc:
             signame = _signo_to_signame(exc.signo)
-            LOG.info(_LI('Caught %s, exiting'), signame)
+            LOG.info(_LI('Child caught %s, exiting'), signame)
             status = exc.code
             signo = exc.signo
         except SystemExit as exc:
@@ -382,6 +383,7 @@ class ProcessLauncher(object):
     def wait(self):
         """Loop waiting on children to die and respawning as necessary."""
 
+        systemd.notify_once()
         LOG.debug('Full set of CONF:')
         CONF.log_opt_values(LOG, std_logging.DEBUG)
 
@@ -389,9 +391,12 @@ class ProcessLauncher(object):
             while True:
                 self.handle_signal()
                 self._respawn_children()
-                if self.sigcaught:
+                # No signal means that stop was called.  Don't clean up here.
-                    signame = _signo_to_signame(self.sigcaught)
+                if not self.sigcaught:
-                    LOG.info(_LI('Caught %s, stopping children'), signame)
+                    return
+
+                signame = _signo_to_signame(self.sigcaught)
+                LOG.info(_LI('Caught %s, stopping children'), signame)
                 if not _is_sighup_and_daemon(self.sigcaught):
                     break
 
@@ -402,6 +407,11 @@ class ProcessLauncher(object):
         except eventlet.greenlet.GreenletExit:
             LOG.info(_LI("Wait called after thread killed.  Cleaning up."))
 
+        self.stop()
+
+    def stop(self):
+        """Terminate child processes and wait on each."""
+        self.running = False
         for pid in self.children:
             try:
                 os.kill(pid, signal.SIGTERM)
@@ -488,7 +498,6 @@ class Services(object):
 
         """
         service.start()
-        systemd.notify_once()
         done.wait()
 
 

designate/openstack/common/sslutils.py

@@ -22,15 +22,12 @@ from designate.openstack.common.gettextutils import _
 
 ssl_opts = [
     cfg.StrOpt('ca_file',
-               default=None,
                help="CA certificate file to use to verify "
                     "connecting clients."),
     cfg.StrOpt('cert_file',
-               default=None,
                help="Certificate file to use when starting "
                     "the server securely."),
     cfg.StrOpt('key_file',
-               default=None,
                help="Private key file to use when starting "
                     "the server securely."),
 ]

designate/openstack/common/strutils.py

@@ -50,6 +50,28 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
 SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
 
 
+# NOTE(flaper87): The following 3 globals are used by `mask_password`
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS = []
+_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
+                    r'(<%(key)s>).*?(</%(key)s>)',
+                    r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
+                    r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
+                    r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])'
+                    '.*?([\'"])',
+                    r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
+
+for key in _SANITIZE_KEYS:
+    for pattern in _FORMAT_PATTERNS:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
+        _SANITIZE_PATTERNS.append(reg_ex)
+
+
 def int_from_bool_as_string(subject):
     """Interpret a string as a boolean and return either 1 or 0.
 
@@ -78,7 +100,7 @@ def bool_from_string(subject, strict=False, default=False):
     Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
     """
     if not isinstance(subject, six.string_types):
-        subject = str(subject)
+        subject = six.text_type(subject)
 
     lowered = subject.strip().lower()
 
@@ -159,19 +181,13 @@ def safe_encode(text, incoming=None,
                     sys.getdefaultencoding())
 
     if isinstance(text, six.text_type):
-        if six.PY3:
+        return text.encode(encoding, errors)
-            return text.encode(encoding, errors).decode(incoming)
-        else:
-            return text.encode(encoding, errors)
     elif text and encoding != incoming:
         # Decode text before encoding it with `encoding`
         text = safe_decode(text, incoming, errors)
-        if six.PY3:
+        return text.encode(encoding, errors)
-            return text.encode(encoding, errors).decode(incoming)
+    else:
-        else:
+        return text
-            return text.encode(encoding, errors)
-
-    return text
 
 
 def string_to_bytes(text, unit_system='IEC', return_int=False):
@@ -243,3 +259,37 @@ def to_slug(value, incoming=None, errors="strict"):
         "ascii", "ignore").decode("ascii")
     value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
     return SLUGIFY_HYPHENATE_RE.sub("-", value)
+
+
+def mask_password(message, secret="***"):
+    """Replace password with 'secret' in message.
+
+    :param message: The string which includes security information.
+    :param secret: value with which to replace passwords.
+    :returns: The unicode value of message with the password fields masked.
+
+    For example:
+
+    >>> mask_password("'adminPass' : 'aaaaa'")
+    "'adminPass' : '***'"
+    >>> mask_password("'admin_pass' : 'aaaaa'")
+    "'admin_pass' : '***'"
+    >>> mask_password('"password" : "aaaaa"')
+    '"password" : "***"'
+    >>> mask_password("'original_password' : 'aaaaa'")
+    "'original_password' : '***'"
+    >>> mask_password("u'original_password' :   u'aaaaa'")
+    "u'original_password' :   u'***'"
+    """
+    message = six.text_type(message)
+
+    # NOTE(ldbragst): Check to see if anything in message contains any key
+    # specified in _SANITIZE_KEYS, if not then just return the message since
+    # we don't have to mask any passwords.
+    if not any(key in message for key in _SANITIZE_KEYS):
+        return message
+
+    secret = r'\g<1>' + secret + r'\g<2>'
+    for pattern in _SANITIZE_PATTERNS:
+        message = re.sub(pattern, secret, message)
+    return message

designate/openstack/common/systemd.py

@@ -50,14 +50,16 @@ def _sd_notify(unset_env, msg):
 
 def notify():
     """Send notification to Systemd that service is ready.
+
     For details see
-      http://www.freedesktop.org/software/systemd/man/sd_notify.html
+    http://www.freedesktop.org/software/systemd/man/sd_notify.html
     """
     _sd_notify(False, 'READY=1')
 
 
 def notify_once():
     """Send notification once to Systemd that service is ready.
+
     Systemd sets NOTIFY_SOCKET environment variable with the name of the
     socket listening for notifications from services.
     This method removes the NOTIFY_SOCKET environment variable to ensure
@@ -75,7 +77,7 @@ def onready(notify_socket, timeout):
     :type timeout:        float
     :returns:             0 service ready
                           1 service not ready
-                          2 timeout occured
+                          2 timeout occurred
     """
     sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
     sock.settimeout(timeout)

designate/openstack/common/threadgroup.py

@@ -85,7 +85,7 @@ class ThreadGroup(object):
     def thread_done(self, thread):
         self.threads.remove(thread)
 
-    def stop(self):
+    def _stop_threads(self):
         current = threading.current_thread()
 
         # Iterate over a copy of self.threads so thread_done doesn't
@@ -99,6 +99,7 @@ class ThreadGroup(object):
             except Exception as ex:
                 LOG.exception(ex)
 
+    def stop_timers(self):
         for x in self.timers:
             try:
                 x.stop()
@@ -106,6 +107,23 @@ class ThreadGroup(object):
                 LOG.exception(ex)
         self.timers = []
 
+    def stop(self, graceful=False):
+        """stop function has the option of graceful=True/False.
+
+        * In case of graceful=True, wait for all threads to be finished.
+          Never kill threads.
+        * In case of graceful=False, kill threads immediately.
+        """
+        self.stop_timers()
+        if graceful:
+            # In case of graceful=True, wait for all threads to be
+            # finished, never kill threads
+            self.wait()
+        else:
+            # In case of graceful=False(Default), kill threads
+            # immediately
+            self._stop_threads()
+
     def wait(self):
         for x in self.timers:
             try:

designate/openstack/common/timeutils.py

@@ -114,7 +114,7 @@ def utcnow():
 
 
 def iso8601_from_timestamp(timestamp):
-    """Returns a iso8601 formatted date from timestamp."""
+    """Returns an iso8601 formatted date from timestamp."""
     return isotime(datetime.datetime.utcfromtimestamp(timestamp))
 
 
@@ -134,7 +134,7 @@ def set_time_override(override_time=None):
 
 def advance_time_delta(timedelta):
     """Advance overridden time using a datetime.timedelta."""
-    assert(not utcnow.override_time is None)
+    assert utcnow.override_time is not None
     try:
         for dt in utcnow.override_time:
             dt += timedelta

designate/openstack/common/versionutils.py

@@ -1,148 +0,0 @@
-# Copyright (c) 2013 OpenStack Foundation
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-Helpers for comparing version strings.
-"""
-
-import functools
-import pkg_resources
-
-from designate.openstack.common.gettextutils import _
-from designate.openstack.common import log as logging
-
-
-LOG = logging.getLogger(__name__)
-
-
-class deprecated(object):
-    """A decorator to mark callables as deprecated.
-
-    This decorator logs a deprecation message when the callable it decorates is
-    used. The message will include the release where the callable was
-    deprecated, the release where it may be removed and possibly an optional
-    replacement.
-
-    Examples:
-
-    1. Specifying the required deprecated release
-
-    >>> @deprecated(as_of=deprecated.ICEHOUSE)
-    ... def a(): pass
-
-    2. Specifying a replacement:
-
-    >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()')
-    ... def b(): pass
-
-    3. Specifying the release where the functionality may be removed:
-
-    >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1)
-    ... def c(): pass
-
-    """
-
-    FOLSOM = 'F'
-    GRIZZLY = 'G'
-    HAVANA = 'H'
-    ICEHOUSE = 'I'
-
-    _RELEASES = {
-        'F': 'Folsom',
-        'G': 'Grizzly',
-        'H': 'Havana',
-        'I': 'Icehouse',
-    }
-
-    _deprecated_msg_with_alternative = _(
-        '%(what)s is deprecated as of %(as_of)s in favor of '
-        '%(in_favor_of)s and may be removed in %(remove_in)s.')
-
-    _deprecated_msg_no_alternative = _(
-        '%(what)s is deprecated as of %(as_of)s and may be '
-        'removed in %(remove_in)s. It will not be superseded.')
-
-    def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None):
-        """Initialize decorator
-
-        :param as_of: the release deprecating the callable. Constants
-            are define in this class for convenience.
-        :param in_favor_of: the replacement for the callable (optional)
-        :param remove_in: an integer specifying how many releases to wait
-            before removing (default: 2)
-        :param what: name of the thing being deprecated (default: the
-            callable's name)
-
-        """
-        self.as_of = as_of
-        self.in_favor_of = in_favor_of
-        self.remove_in = remove_in
-        self.what = what
-
-    def __call__(self, func):
-        if not self.what:
-            self.what = func.__name__ + '()'
-
-        @functools.wraps(func)
-        def wrapped(*args, **kwargs):
-            msg, details = self._build_message()
-            LOG.deprecated(msg, details)
-            return func(*args, **kwargs)
-        return wrapped
-
-    def _get_safe_to_remove_release(self, release):
-        # TODO(dstanek): this method will have to be reimplemented once
-        #    when we get to the X release because once we get to the Y
-        #    release, what is Y+2?
-        new_release = chr(ord(release) + self.remove_in)
-        if new_release in self._RELEASES:
-            return self._RELEASES[new_release]
-        else:
-            return new_release
-
-    def _build_message(self):
-        details = dict(what=self.what,
-                       as_of=self._RELEASES[self.as_of],
-                       remove_in=self._get_safe_to_remove_release(self.as_of))
-
-        if self.in_favor_of:
-            details['in_favor_of'] = self.in_favor_of
-            msg = self._deprecated_msg_with_alternative
-        else:
-            msg = self._deprecated_msg_no_alternative
-        return msg, details
-
-
-def is_compatible(requested_version, current_version, same_major=True):
-    """Determine whether `requested_version` is satisfied by
-    `current_version`; in other words, `current_version` is >=
-    `requested_version`.
-
-    :param requested_version: version to check for compatibility
-    :param current_version: version to check against
-    :param same_major: if True, the major version must be identical between
-        `requested_version` and `current_version`. This is used when a
-        major-version difference indicates incompatibility between the two
-        versions. Since this is the common-case in practice, the default is
-        True.
-    :returns: True if compatible, False if not
-    """
-    requested_parts = pkg_resources.parse_version(requested_version)
-    current_parts = pkg_resources.parse_version(current_version)
-
-    if same_major and (requested_parts[0] != current_parts[0]):
-        return False
-
-    return current_parts >= requested_parts

designate/tests/__init__.py

@@ -31,7 +31,7 @@ import sqlalchemy
 from testtools import testcase
 
 from designate.openstack.common import log as logging
-from designate.openstack.common.fixture import config
+from designate.openstack.common.fixture import config as cfg_fixture
 from designate.openstack.common import importutils
 from designate import policy
 from designate import utils
@@ -239,7 +239,7 @@ class TestCase(base.BaseTestCase):
         super(TestCase, self).setUp()
 
         self.useFixture(fixtures.FakeLogger('designate', level='DEBUG'))
-        self.CONF = self.useFixture(config.Config(cfg.CONF)).conf
+        self.CONF = self.useFixture(cfg_fixture.Config(cfg.CONF)).conf
 
         self.messaging_conf = self.useFixture(
             messaging_fixture.ConfFixture(cfg.CONF))

openstack-common.conf

@@ -1,16 +1,19 @@
 [DEFAULT]
 
+# The list of scriptss to copy from oslo-incubator.git
+script=tools/install_venv_common.py
+
 # The list of modules to copy from oslo-incubator.git
 module=context
 module=excutils
-module=fixture
+module=fixture.config
 module=gettextutils
 module=importutils
-module=install_venv_common
 module=jsonutils
 module=local
 module=log
-module=middleware
+module=middleware.base
+module=middleware.request_id
 module=policy
 module=processutils
 module=service

tools/install_venv_common.py

@@ -125,7 +125,7 @@ class InstallVenv(object):
         parser.add_option('-n', '--no-site-packages',
                           action='store_true',
                           help="Do not inherit packages from global Python "
-                               "install")
+                               "install.")
         return parser.parse_args(argv[1:])[0]