Merge "Sync with latest oslo-incubator"

nova/api/openstack/wsgi.py

@@ -20,6 +20,7 @@ import time
 from xml.dom import minidom
 
 from lxml import etree
+from oslo.utils import strutils
 import six
 import webob
 
@@ -905,7 +906,7 @@ class Resource(wsgi.Application):
                     "%(body)s") % {'action': action,
                                    'body': unicode(body, 'utf-8'),
                                    'meth': str(meth)}
-            LOG.debug(logging.mask_password(msg))
+            LOG.debug(strutils.mask_password(msg))
         else:
             LOG.debug("Calling method '%(meth)s'",
                       {'meth': str(meth)})

nova/openstack/common/__init__.py

@@ -1,17 +0,0 @@
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import six
-
-
-six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

nova/openstack/common/_i18n.py

@@ -0,0 +1,40 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""oslo.i18n integration module.
+
+See http://docs.openstack.org/developer/oslo.i18n/usage.html
+
+"""
+
+import oslo.i18n
+
+
+# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
+# application name when this module is synced into the separate
+# repository. It is OK to have more than one translation function
+# using the same domain, since there will still only be one message
+# catalog.
+_translators = oslo.i18n.TranslatorFactory(domain='nova')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical

nova/openstack/common/cliutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -14,20 +12,33 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+# W0603: Using the global statement
+# W0621: Redefining name %s from outer scope
+# pylint: disable=W0603,W0621
+
+from __future__ import print_function
+
+import getpass
 import inspect
+import os
+import sys
+import textwrap
+
+from oslo.utils import encodeutils
+from oslo.utils import strutils
+import prettytable
+import six
+from six import moves
+
+from nova.openstack.common._i18n import _
 
 
 class MissingArgs(Exception):
-
+    """Supplied arguments are not sufficient for calling a function."""
     def __init__(self, missing):
         self.missing = missing
-
+        msg = _("Missing arguments: %s") % ", ".join(missing)
-    def __str__(self):
+        super(MissingArgs, self).__init__(msg)
-        if len(self.missing) == 1:
-            return "An argument is missing"
-        else:
-            return ("%(num)d arguments are missing" %
-                    dict(num=len(self.missing)))
 
 
 def validate_args(fn, *args, **kwargs):
@@ -36,11 +47,11 @@ def validate_args(fn, *args, **kwargs):
     >>> validate_args(lambda a: None)
     Traceback (most recent call last):
         ...
-    MissingArgs: An argument is missing
+    MissingArgs: Missing argument(s): a
     >>> validate_args(lambda a, b, c, d: None, 0, c=1)
     Traceback (most recent call last):
         ...
-    MissingArgs: 2 arguments are missing
+    MissingArgs: Missing argument(s): b, d
 
     :param fn: the function to check
     :param arg: the positional arguments supplied
@@ -52,7 +63,7 @@ def validate_args(fn, *args, **kwargs):
     required_args = argspec.args[:len(argspec.args) - num_defaults]
 
     def isbound(method):
-        return getattr(method, 'im_self', None) is not None
+        return getattr(method, '__self__', None) is not None
 
     if isbound(fn):
         required_args.pop(0)
@@ -61,3 +72,193 @@ def validate_args(fn, *args, **kwargs):
     missing = missing[len(args):]
     if missing:
         raise MissingArgs(missing)
+
+
+def arg(*args, **kwargs):
+    """Decorator for CLI args.
+
+    Example:
+
+    >>> @arg("name", help="Name of the new entity")
+    ... def entity_create(args):
+    ...     pass
+    """
+    def _decorator(func):
+        add_arg(func, *args, **kwargs)
+        return func
+    return _decorator
+
+
+def env(*args, **kwargs):
+    """Returns the first environment variable set.
+
+    If all are empty, defaults to '' or keyword arg `default`.
+    """
+    for arg in args:
+        value = os.environ.get(arg)
+        if value:
+            return value
+    return kwargs.get('default', '')
+
+
+def add_arg(func, *args, **kwargs):
+    """Bind CLI arguments to a shell.py `do_foo` function."""
+
+    if not hasattr(func, 'arguments'):
+        func.arguments = []
+
+    # NOTE(sirp): avoid dups that can occur when the module is shared across
+    # tests.
+    if (args, kwargs) not in func.arguments:
+        # Because of the semantics of decorator composition if we just append
+        # to the options list positional options will appear to be backwards.
+        func.arguments.insert(0, (args, kwargs))
+
+
+def unauthenticated(func):
+    """Adds 'unauthenticated' attribute to decorated function.
+
+    Usage:
+
+    >>> @unauthenticated
+    ... def mymethod(f):
+    ...     pass
+    """
+    func.unauthenticated = True
+    return func
+
+
+def isunauthenticated(func):
+    """Checks if the function does not require authentication.
+
+    Mark such functions with the `@unauthenticated` decorator.
+
+    :returns: bool
+    """
+    return getattr(func, 'unauthenticated', False)
+
+
+def print_list(objs, fields, formatters=None, sortby_index=0,
+               mixed_case_fields=None, field_labels=None):
+    """Print a list or objects as a table, one row per object.
+
+    :param objs: iterable of :class:`Resource`
+    :param fields: attributes that correspond to columns, in order
+    :param formatters: `dict` of callables for field formatting
+    :param sortby_index: index of the field for sorting table rows
+    :param mixed_case_fields: fields corresponding to object attributes that
+        have mixed case names (e.g., 'serverId')
+    :param field_labels: Labels to use in the heading of the table, default to
+        fields.
+    """
+    formatters = formatters or {}
+    mixed_case_fields = mixed_case_fields or []
+    field_labels = field_labels or fields
+    if len(field_labels) != len(fields):
+        raise ValueError(_("Field labels list %(labels)s has different number "
+                           "of elements than fields list %(fields)s"),
+                         {'labels': field_labels, 'fields': fields})
+
+    if sortby_index is None:
+        kwargs = {}
+    else:
+        kwargs = {'sortby': field_labels[sortby_index]}
+    pt = prettytable.PrettyTable(field_labels)
+    pt.align = 'l'
+
+    for o in objs:
+        row = []
+        for field in fields:
+            if field in formatters:
+                row.append(formatters[field](o))
+            else:
+                if field in mixed_case_fields:
+                    field_name = field.replace(' ', '_')
+                else:
+                    field_name = field.lower().replace(' ', '_')
+                data = getattr(o, field_name, '')
+                row.append(data)
+        pt.add_row(row)
+
+    print(encodeutils.safe_encode(pt.get_string(**kwargs)))
+
+
+def print_dict(dct, dict_property="Property", wrap=0):
+    """Print a `dict` as a table of two columns.
+
+    :param dct: `dict` to print
+    :param dict_property: name of the first column
+    :param wrap: wrapping for the second column
+    """
+    pt = prettytable.PrettyTable([dict_property, 'Value'])
+    pt.align = 'l'
+    for k, v in six.iteritems(dct):
+        # convert dict to str to check length
+        if isinstance(v, dict):
+            v = six.text_type(v)
+        if wrap > 0:
+            v = textwrap.fill(six.text_type(v), wrap)
+        # if value has a newline, add in multiple rows
+        # e.g. fault with stacktrace
+        if v and isinstance(v, six.string_types) and r'\n' in v:
+            lines = v.strip().split(r'\n')
+            col1 = k
+            for line in lines:
+                pt.add_row([col1, line])
+                col1 = ''
+        else:
+            pt.add_row([k, v])
+    print(encodeutils.safe_encode(pt.get_string()))
+
+
+def get_password(max_password_prompts=3):
+    """Read password from TTY."""
+    verify = strutils.bool_from_string(env("OS_VERIFY_PASSWORD"))
+    pw = None
+    if hasattr(sys.stdin, "isatty") and sys.stdin.isatty():
+        # Check for Ctrl-D
+        try:
+            for __ in moves.range(max_password_prompts):
+                pw1 = getpass.getpass("OS Password: ")
+                if verify:
+                    pw2 = getpass.getpass("Please verify: ")
+                else:
+                    pw2 = pw1
+                if pw1 == pw2 and pw1:
+                    pw = pw1
+                    break
+        except EOFError:
+            pass
+    return pw
+
+
+def service_type(stype):
+    """Adds 'service_type' attribute to decorated function.
+
+    Usage:
+
+    .. code-block:: python
+
+       @service_type('volume')
+       def mymethod(f):
+       ...
+    """
+    def inner(f):
+        f.service_type = stype
+        return f
+    return inner
+
+
+def get_service_type(f):
+    """Retrieves service type from function."""
+    return getattr(f, 'service_type', None)
+
+
+def pretty_choice_list(l):
+    return ', '.join("'%s'" % i for i in l)
+
+
+def exit(msg=''):
+    if msg:
+        print (msg, file=sys.stderr)
+    sys.exit(1)

nova/openstack/common/eventlet_backdoor.py

@@ -29,7 +29,7 @@ import eventlet.backdoor
 import greenlet
 from oslo.config import cfg
 
-from nova.openstack.common.gettextutils import _LI
+from nova.openstack.common._i18n import _LI
 from nova.openstack.common import log as logging
 
 help_for_backdoor_port = (
@@ -41,7 +41,6 @@ help_for_backdoor_port = (
     "chosen port is displayed in the service's log file.")
 eventlet_backdoor_opts = [
     cfg.StrOpt('backdoor_port',
-               default=None,
                help="Enable eventlet backdoor.  %s" % help_for_backdoor_port)
 ]
 

nova/openstack/common/fileutils.py

@@ -18,7 +18,8 @@ import errno
 import os
 import tempfile
 
-from nova.openstack.common import excutils
+from oslo.utils import excutils
+
 from nova.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)

nova/openstack/common/fixture/__init__.py

@@ -0,0 +1,17 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+
+six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

nova/openstack/common/fixture/lockutils.py

@@ -48,4 +48,4 @@ class LockFixture(fixtures.Fixture):
     def setUp(self):
         super(LockFixture, self).setUp()
         self.addCleanup(self.mgr.__exit__, None, None, None)
-        self.mgr.__enter__()
+        self.lock = self.mgr.__enter__()

nova/openstack/common/imageutils.py

@@ -21,8 +21,9 @@ Helper methods to deal with images.
 
 import re
 
-from nova.openstack.common.gettextutils import _
+from oslo.utils import strutils
-from nova.openstack.common import strutils
+
+from nova.openstack.common._i18n import _
 
 
 class QemuImgInfo(object):
@@ -62,7 +63,7 @@ class QemuImgInfo(object):
         # Standardize on underscores/lc/no dash and no spaces
         # since qemu seems to have mixed outputs here... and
         # this format allows for better integration with python
-        # - ie for usage in kwargs and such...
+        # - i.e. for usage in kwargs and such...
         field = field.lower().strip()
         for c in (" ", "-"):
             field = field.replace(c, '_')

nova/openstack/common/lockutils.py

@@ -29,7 +29,7 @@ import weakref
 from oslo.config import cfg
 
 from nova.openstack.common import fileutils
-from nova.openstack.common.gettextutils import _, _LE, _LI
+from nova.openstack.common._i18n import _, _LE, _LI
 
 
 LOG = logging.getLogger(__name__)

nova/openstack/common/log.py

@@ -38,18 +38,15 @@ import sys
 import traceback
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
+from oslo.utils import importutils
 import six
 from six import moves
 
 _PY26 = sys.version_info[0:2] == (2, 6)
 
-from nova.openstack.common.gettextutils import _
+from nova.openstack.common._i18n import _
-from nova.openstack.common import importutils
-from nova.openstack.common import jsonutils
 from nova.openstack.common import local
-# NOTE(flaper87): Pls, remove when graduating this module
-# from the incubator.
-from nova.openstack.common.strutils import mask_password  # noqa
 
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
@@ -554,7 +551,7 @@ def _setup_logging_from_conf(project, version):
                 syslog = logging.handlers.SysLogHandler(facility=facility)
             log_root.addHandler(syslog)
         except socket.error:
-            log_root.error('Unable to add syslog handler. Verify that syslog'
+            log_root.error('Unable to add syslog handler. Verify that syslog '
                            'is running.')
 
 

nova/openstack/common/loopingcall.py

@@ -21,7 +21,7 @@ import time
 from eventlet import event
 from eventlet import greenthread
 
-from nova.openstack.common.gettextutils import _LE, _LW
+from nova.openstack.common._i18n import _LE, _LW
 from nova.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)

nova/openstack/common/memorycache.py

@@ -17,7 +17,6 @@
 """Super simple fake memcache client."""
 
 from oslo.config import cfg
-
 from oslo.utils import timeutils
 
 memcache_opts = [

nova/openstack/common/middleware/base.py

@@ -12,6 +12,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+
 """Base class(es) for WSGI Middleware."""
 
 import webob.dec

nova/openstack/common/middleware/catch_errors.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2013 NEC Corporation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Middleware that provides high-level error handling.
+
+It catches all exceptions from subsequent applications in WSGI pipeline
+to hide internal errors from API response.
+"""
+import logging
+
+import webob.dec
+import webob.exc
+
+from nova.openstack.common._i18n import _LE
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+LOG = logging.getLogger(__name__)
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.CatchErrors')
+class CatchErrorsMiddleware(base.Middleware):
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        try:
+            response = req.get_response(self.application)
+        except Exception:
+            LOG.exception(_LE('An error occurred during '
+                              'processing the request: %s'))
+            response = webob.exc.HTTPInternalServerError()
+        return response

nova/openstack/common/middleware/correlation_id.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2013 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Middleware that attaches a correlation id to WSGI request"""
+
+import uuid
+
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.CorrelationId')
+class CorrelationIdMiddleware(base.Middleware):
+
+    def process_request(self, req):
+        correlation_id = (req.headers.get("X_CORRELATION_ID") or
+                          str(uuid.uuid4()))
+        req.headers['X_CORRELATION_ID'] = correlation_id

nova/openstack/common/middleware/debug.py

@@ -0,0 +1,63 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Debug middleware"""
+
+from __future__ import print_function
+
+import sys
+
+import six
+import webob.dec
+
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.Debug')
+class Debug(base.Middleware):
+    """Helper class that returns debug information.
+
+    Can be inserted into any WSGI application chain to get information about
+    the request and response.
+    """
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        print(("*" * 40) + " REQUEST ENVIRON")
+        for key, value in req.environ.items():
+            print(key, "=", value)
+        print()
+        resp = req.get_response(self.application)
+
+        print(("*" * 40) + " RESPONSE HEADERS")
+        for (key, value) in six.iteritems(resp.headers):
+            print(key, "=", value)
+        print()
+
+        resp.app_iter = self.print_generator(resp.app_iter)
+
+        return resp
+
+    @staticmethod
+    def print_generator(app_iter):
+        """Prints the contents of a wrapper string iterator when iterated."""
+        print(("*" * 40) + " BODY")
+        for part in app_iter:
+            sys.stdout.write(part)
+            sys.stdout.flush()
+            yield part
+        print()

nova/openstack/common/middleware/request_id.py

@@ -23,12 +23,15 @@ import webob.dec
 
 from nova.openstack.common import context
 from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
 
 
 ENV_REQUEST_ID = 'openstack.request_id'
 HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id'
 
 
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.RequestId')
 class RequestIdMiddleware(base.Middleware):
 
     @webob.dec.wsgify

nova/openstack/common/middleware/sizelimit.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2012 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Request Body limiting middleware.
+
+"""
+
+from oslo.config import cfg
+import webob.dec
+import webob.exc
+
+from nova.openstack.common._i18n import _
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+# default request size is 112k
+max_req_body_size = cfg.IntOpt('max_request_body_size',
+                               deprecated_name='osapi_max_request_body_size',
+                               default=114688,
+                               help='The maximum body size for each '
+                                    ' request, in bytes.')
+
+CONF = cfg.CONF
+CONF.register_opt(max_req_body_size)
+
+
+class LimitingReader(object):
+    """Reader to limit the size of an incoming request."""
+    def __init__(self, data, limit):
+        """Initiates LimitingReader object.
+
+        :param data: Underlying data object
+        :param limit: maximum number of bytes the reader should allow
+        """
+        self.data = data
+        self.limit = limit
+        self.bytes_read = 0
+
+    def __iter__(self):
+        for chunk in self.data:
+            self.bytes_read += len(chunk)
+            if self.bytes_read > self.limit:
+                msg = _("Request is too large.")
+                raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+            else:
+                yield chunk
+
+    def read(self, i=None):
+        result = self.data.read(i)
+        self.bytes_read += len(result)
+        if self.bytes_read > self.limit:
+            msg = _("Request is too large.")
+            raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+        return result
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.RequestBodySizeLimiter')
+class RequestBodySizeLimiter(base.Middleware):
+    """Limit the size of incoming requests."""
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        if (req.content_length is not None and
+                req.content_length > CONF.max_request_body_size):
+            msg = _("Request is too large.")
+            raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+        if req.content_length is None and req.is_body_readable:
+            limiter = LimitingReader(req.body_file,
+                                     CONF.max_request_body_size)
+            req.body_file = limiter
+        return self.application

nova/openstack/common/periodic_task.py

@@ -17,7 +17,7 @@ import time
 from oslo.config import cfg
 import six
 
-from nova.openstack.common.gettextutils import _, _LE, _LI
+from nova.openstack.common._i18n import _, _LE, _LI
 from nova.openstack.common import log as logging
 
 

nova/openstack/common/policy.py

@@ -77,16 +77,17 @@ as it allows particular rules to be explicitly disabled.
 
 import abc
 import ast
+import os
 import re
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
 import six
 import six.moves.urllib.parse as urlparse
 import six.moves.urllib.request as urlrequest
 
 from nova.openstack.common import fileutils
-from nova.openstack.common.gettextutils import _, _LE
+from nova.openstack.common._i18n import _, _LE, _LW
-from nova.openstack.common import jsonutils
 from nova.openstack.common import log as logging
 
 
@@ -98,6 +99,10 @@ policy_opts = [
                default='default',
                help=_('Default rule. Enforced when a requested rule is not '
                       'found.')),
+    cfg.MultiStrOpt('policy_dirs',
+                    default=['policy.d'],
+                    help=_('The directories of policy configuration files is '
+                           'stored')),
 ]
 
 CONF = cfg.CONF
@@ -216,6 +221,7 @@ class Enforcer(object):
     def clear(self):
         """Clears Enforcer rules, policy's cache and policy's path."""
         self.set_rules({})
+        fileutils.delete_cached_file(self.policy_path)
         self.default_rule = None
         self.policy_path = None
 
@@ -232,31 +238,53 @@ class Enforcer(object):
 
         if self.use_conf:
             if not self.policy_path:
-                self.policy_path = self._get_policy_path()
+                self.policy_path = self._get_policy_path(self.policy_file)
 
+            self._load_policy_file(self.policy_path, force_reload)
+            for path in CONF.policy_dirs:
+                try:
+                    path = self._get_policy_path(path)
+                except cfg.ConfigFilesNotFoundError:
+                    LOG.warn(_LW("Can not find policy directories %s"), path)
+                    continue
+                self._walk_through_policy_directory(path,
+                                                    self._load_policy_file,
+                                                    force_reload, False)
+
+    def _walk_through_policy_directory(self, path, func, *args):
+        # We do not iterate over sub-directories.
+        policy_files = next(os.walk(path))[2]
+        policy_files.sort()
+        for policy_file in [p for p in policy_files if not p.startswith('.')]:
+            func(os.path.join(path, policy_file), *args)
+
+    def _load_policy_file(self, path, force_reload, overwrite=True):
             reloaded, data = fileutils.read_cached_file(
-                self.policy_path, force_reload=force_reload)
+                path, force_reload=force_reload)
             if reloaded or not self.rules:
                 rules = Rules.load_json(data, self.default_rule)
-                self.set_rules(rules)
+                self.set_rules(rules, overwrite)
                 LOG.debug("Rules successfully reloaded")
 
-    def _get_policy_path(self):
+    def _get_policy_path(self, path):
-        """Locate the policy json data file.
+        """Locate the policy json data file/path.
 
-        :param policy_file: Custom policy file to locate.
+        :param path: It's value can be a full path or related path. When
+                     full path specified, this function just returns the full
+                     path. When related path specified, this function will
+                     search configuration directories to find one that exists.
 
         :returns: The policy path
 
-        :raises: ConfigFilesNotFoundError if the file couldn't
+        :raises: ConfigFilesNotFoundError if the file/path couldn't
                  be located.
         """
-        policy_file = CONF.find_file(self.policy_file)
+        policy_path = CONF.find_file(path)
 
-        if policy_file:
+        if policy_path:
-            return policy_file
+            return policy_path
 
-        raise cfg.ConfigFilesNotFoundError((self.policy_file,))
+        raise cfg.ConfigFilesNotFoundError((path,))
 
     def enforce(self, rule, target, creds, do_raise=False,
                 exc=None, *args, **kwargs):
@@ -784,7 +812,7 @@ def _parse_text_rule(rule):
         return state.result
     except ValueError:
         # Couldn't parse the rule
-        LOG.exception(_LE("Failed to understand rule %r") % rule)
+        LOG.exception(_LE("Failed to understand rule %s") % rule)
 
         # Fail closed
         return FalseCheck()
@@ -875,7 +903,6 @@ class GenericCheck(Check):
             'Member':%(role.name)s
         """
 
-        # TODO(termie): do dict inspection via dot syntax
         try:
             match = self.match % target
         except KeyError:
@@ -888,7 +915,10 @@ class GenericCheck(Check):
             leftval = ast.literal_eval(self.kind)
         except ValueError:
             try:
-                leftval = creds[self.kind]
+                kind_parts = self.kind.split('.')
+                leftval = creds
+                for kind_part in kind_parts:
+                    leftval = leftval[kind_part]
             except KeyError:
                 return False
         return match == six.text_type(leftval)

nova/openstack/common/processutils.py

@@ -18,7 +18,7 @@ System-level utilities and helper functions.
 """
 
 import errno
-import logging as stdlib_logging
+import logging
 import multiprocessing
 import os
 import random
@@ -27,11 +27,10 @@ import signal
 
 from eventlet.green import subprocess
 from eventlet import greenthread
+from oslo.utils import strutils
 import six
 
-from nova.openstack.common.gettextutils import _
+from nova.openstack.common._i18n import _
-from nova.openstack.common import log as logging
-from nova.openstack.common import strutils
 
 
 LOG = logging.getLogger(__name__)
@@ -116,8 +115,7 @@ def execute(*cmd, **kwargs):
                             execute this command. Defaults to false.
     :type shell:            boolean
     :param loglevel:        log level for execute commands.
-    :type loglevel:         int.  (Should be stdlib_logging.DEBUG or
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
-                            stdlib_logging.INFO)
     :returns:               (stdout, stderr) from process execution
     :raises:                :class:`UnknownArgumentError` on
                             receiving unknown arguments
@@ -133,7 +131,7 @@ def execute(*cmd, **kwargs):
     run_as_root = kwargs.pop('run_as_root', False)
     root_helper = kwargs.pop('root_helper', '')
     shell = kwargs.pop('shell', False)
-    loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+    loglevel = kwargs.pop('loglevel', logging.DEBUG)
 
     if isinstance(check_exit_code, bool):
         ignore_exit_code = not check_exit_code
@@ -142,8 +140,7 @@ def execute(*cmd, **kwargs):
         check_exit_code = [check_exit_code]
 
     if kwargs:
-        raise UnknownArgumentError(_('Got unknown keyword args '
+        raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
-                                     'to utils.execute: %r') % kwargs)
 
     if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
         if not root_helper:

nova/openstack/common/report/generators/conf.py

@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack config generators
+"""Provides OpenStack config generators
 
 This module defines a class for configuration
 generators for generating the model in
@@ -21,17 +21,17 @@ generators for generating the model in
 
 from oslo.config import cfg
 
-import nova.openstack.common.report.models.conf as cm
+from nova.openstack.common.report.models import conf as cm
 
 
 class ConfigReportGenerator(object):
     """A Configuration Data Generator
 
     This generator returns
-    :class:`openstack.common.report.models.conf.ConfigModel` ,
+    :class:`openstack.common.report.models.conf.ConfigModel`,
     by default using the configuration options stored
     in :attr:`oslo.config.cfg.CONF`, which is where
-    Openstack stores everything.
+    OpenStack stores everything.
 
     :param cnf: the configuration option object
     :type cnf: :class:`oslo.config.cfg.ConfigOpts`

nova/openstack/common/report/generators/threading.py

@@ -23,10 +23,10 @@ import sys
 
 import greenlet
 
-import nova.openstack.common.report.models.threading as tm
+from nova.openstack.common.report.models import threading as tm
 from nova.openstack.common.report.models import with_default_views as mwdv
-import nova.openstack.common.report.utils as rutils
+from nova.openstack.common.report import utils as rutils
-import nova.openstack.common.report.views.text.generic as text_views
+from nova.openstack.common.report.views.text import generic as text_views
 
 
 class ThreadReportGenerator(object):

nova/openstack/common/report/generators/version.py

@@ -12,15 +12,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack version generators
+"""Provides OpenStack version generators
 
-This module defines a class for Openstack
+This module defines a class for OpenStack
 version and package information
 generators for generating the model in
 :mod:`openstack.common.report.models.version`.
 """
 
-import nova.openstack.common.report.models.version as vm
+from nova.openstack.common.report.models import version as vm
 
 
 class PackageReportGenerator(object):

nova/openstack/common/report/guru_meditation_report.py

@@ -51,9 +51,13 @@ where stderr is logged for that given service.
 
 from __future__ import print_function
 
+import inspect
+import os
 import signal
 import sys
 
+from oslo.utils import timeutils
+
 from nova.openstack.common.report.generators import conf as cgen
 from nova.openstack.common.report.generators import threading as tgen
 from nova.openstack.common.report.generators import version as pgen
@@ -74,6 +78,8 @@ class GuruMeditation(object):
     MRO is correct.
     """
 
+    timestamp_fmt = "%Y%m%d%H%M%S"
+
     def __init__(self, version_obj, *args, **kwargs):
         self.version_obj = version_obj
 
@@ -97,13 +103,17 @@ class GuruMeditation(object):
             cls.persistent_sections = [[section_title, generator]]
 
     @classmethod
-    def setup_autorun(cls, version, signum=None):
+    def setup_autorun(cls, version, service_name=None,
+                      log_dir=None, signum=None):
         """Set Up Auto-Run
 
         This method sets up the Guru Meditation Report to automatically
-        get dumped to stderr when the given signal is received.
+        get dumped to stderr or a file in a given dir when the given signal
+        is received.
 
         :param version: the version object for the current product
+        :param service_name: this program name used to construct logfile name
+        :param logdir: path to a log directory where to create a file
         :param signum: the signal to associate with running the report
         """
 
@@ -113,18 +123,25 @@ class GuruMeditation(object):
 
         if signum:
             signal.signal(signum,
-                          lambda *args: cls.handle_signal(version, *args))
+                          lambda *args: cls.handle_signal(
+                              version, service_name, log_dir, *args))
 
     @classmethod
-    def handle_signal(cls, version, *args):
+    def handle_signal(cls, version, service_name, log_dir, *args):
         """The Signal Handler
 
         This method (indirectly) handles receiving a registered signal and
-        dumping the Guru Meditation Report to stderr.  This method is designed
+        dumping the Guru Meditation Report to stderr or a file in a given dir.
-        to be curried into a proper signal handler by currying out the version
+        If service name and log dir are not None, the report will be dumped to
+        a file named $service_name_gurumeditation_$current_time in the log_dir
+        directory.
+        This method is designed to be curried into a proper signal handler by
+        currying out the version
         parameter.
 
         :param version: the version object for the current product
+        :param service_name: this program name used to construct logfile name
+        :param logdir: path to a log directory where to create a file
         """
 
         try:
@@ -132,6 +149,19 @@ class GuruMeditation(object):
         except Exception:
             print("Unable to run Guru Meditation Report!",
                   file=sys.stderr)
+        else:
+            if log_dir:
+                service_name = service_name or os.path.basename(
+                    inspect.stack()[-1][1])
+                filename = "%s_gurumeditation_%s" % (
+                    service_name, timeutils.strtime(fmt=cls.timestamp_fmt))
+                filepath = os.path.join(log_dir, filename)
+                try:
+                    with open(filepath, "w") as dumpfile:
+                        dumpfile.write(res)
+                except Exception:
+                    print("Unable to dump Guru Meditation Report to file %s" %
+                          (filepath,), file=sys.stderr)
             else:
                 print(res, file=sys.stderr)
 

nova/openstack/common/report/models/base.py

@@ -81,6 +81,11 @@ class ReportModel(col.MutableMapping):
         return self.data.__contains__(key)
 
     def __getattr__(self, attrname):
+        # Needed for deepcopy in Python3. That will avoid an infinite loop
+        # in __getattr__ .
+        if 'data' not in self.__dict__:
+            self.data = {}
+
         try:
             return self.data[attrname]
         except KeyError:

nova/openstack/common/report/models/conf.py

@@ -12,14 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack Configuration Model
+"""Provides OpenStack Configuration Model
 
 This module defines a class representing the data
 model for :mod:`oslo.config` configuration options
 """
 
-import nova.openstack.common.report.models.with_default_views as mwdv
+from nova.openstack.common.report.models import with_default_views as mwdv
-import nova.openstack.common.report.views.text.generic as generic_text_views
+from nova.openstack.common.report.views.text import generic as generic_text_views
 
 
 class ConfigModel(mwdv.ModelWithDefaultViews):

nova/openstack/common/report/models/threading.py

@@ -20,8 +20,8 @@ thread, and stack trace data models
 
 import traceback
 
-import nova.openstack.common.report.models.with_default_views as mwdv
+from nova.openstack.common.report.models import with_default_views as mwdv
-import nova.openstack.common.report.views.text.threading as text_views
+from nova.openstack.common.report.views.text import threading as text_views
 
 
 class StackTraceModel(mwdv.ModelWithDefaultViews):
@@ -42,12 +42,12 @@ class StackTraceModel(mwdv.ModelWithDefaultViews):
                 {'filename': fn, 'line': ln, 'name': nm, 'code': cd}
                 for fn, ln, nm, cd in traceback.extract_stack(stack_state)
             ]
-
+            # FIXME(flepied): under Python3 f_exc_type doesn't exist
-            if stack_state.f_exc_type is not None:
+            # anymore so we lose information about exceptions
+            if getattr(stack_state, 'f_exc_type', None) is not None:
                 self['root_exception'] = {
                     'type': stack_state.f_exc_type,
-                    'value': stack_state.f_exc_value
+                    'value': stack_state.f_exc_value}
-                }
             else:
                 self['root_exception'] = None
         else:

nova/openstack/common/report/models/version.py

@@ -12,14 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack Version Info Model
+"""Provides OpenStack Version Info Model
 
 This module defines a class representing the data
-model for Openstack package and version information
+model for OpenStack package and version information
 """
 
-import nova.openstack.common.report.models.with_default_views as mwdv
+from nova.openstack.common.report.models import with_default_views as mwdv
-import nova.openstack.common.report.views.text.generic as generic_text_views
+from nova.openstack.common.report.views.text import generic as generic_text_views
 
 
 class PackageModel(mwdv.ModelWithDefaultViews):

nova/openstack/common/report/models/with_default_views.py

@@ -14,10 +14,10 @@
 
 import copy
 
-import nova.openstack.common.report.models.base as base_model
+from nova.openstack.common.report.models import base as base_model
-import nova.openstack.common.report.views.json.generic as jsonviews
+from nova.openstack.common.report.views.json import generic as jsonviews
-import nova.openstack.common.report.views.text.generic as textviews
+from nova.openstack.common.report.views.text import generic as textviews
-import nova.openstack.common.report.views.xml.generic as xmlviews
+from nova.openstack.common.report.views.xml import generic as xmlviews
 
 
 class ModelWithDefaultViews(base_model.ReportModel):
@@ -28,18 +28,18 @@ class ModelWithDefaultViews(base_model.ReportModel):
     when a submodel should have an attached view, but the view
     differs depending on the serialization format
 
-    Paramaters are as the superclass, with the exception
+    Parameters are as the superclass, except for any
-    of any parameters ending in '_view': these parameters
+    parameters ending in '_view': these parameters
     get stored as default views.
 
     The default 'default views' are
 
     text
-        :class:`openstack.common.views.text.generic.KeyValueView`
+        :class:`openstack.common.report.views.text.generic.KeyValueView`
     xml
-        :class:`openstack.common.views.xml.generic.KeyValueView`
+        :class:`openstack.common.report.views.xml.generic.KeyValueView`
     json
-        :class:`openstack.common.views.json.generic.KeyValueView`
+        :class:`openstack.common.report.views.json.generic.KeyValueView`
 
     .. function:: to_type()
 

nova/openstack/common/report/report.py

@@ -14,12 +14,12 @@
 
 """Provides Report classes
 
-This module defines various classes representing
+This module defines various classes representing reports and report sections.
-reports and report sections.  All reports take the
+All reports take the form of a report class containing various report
-form of a report class containing various report sections.
+sections.
 """
 
-import nova.openstack.common.report.views.text.header as header_views
+from nova.openstack.common.report.views.text import header as header_views
 
 
 class BasicReport(object):
@@ -28,7 +28,7 @@ class BasicReport(object):
     A Basic Report consists of a collection of :class:`ReportSection`
     objects, each of which contains a top-level model and generator.
     It collects these sections into a cohesive report which may then
-    be serialized by calling :func:`run`
+    be serialized by calling :func:`run`.
     """
 
     def __init__(self):
@@ -78,10 +78,9 @@ class BasicReport(object):
 class ReportSection(object):
     """A Report Section
 
-    A report section contains a generator and a top-level view.
+    A report section contains a generator and a top-level view. When something
-    When something attempts to serialize the section by calling
+    attempts to serialize the section by calling str() on it, the section runs
-    str() on it, the section runs the generator and calls the view
+    the generator and calls the view on the resulting model.
-    on the resulting model.
 
     .. seealso::
 
@@ -89,9 +88,8 @@ class ReportSection(object):
            :func:`BasicReport.add_section`
 
     :param view: the top-level view for this section
-    :param generator: the generator for this section which could be
+    :param generator: the generator for this section
-                      any callable object which takes
+      (any callable object which takes no parameters and returns a data model)
-                      no parameters and returns a data model
     """
 
     def __init__(self, view, generator):

nova/openstack/common/report/views/jinja_view.py

@@ -19,6 +19,8 @@ system for serialization.  For more information on Jinja, please
 see http://jinja.pocoo.org/ .
 """
 
+import copy
+
 import jinja2
 
 
@@ -79,6 +81,16 @@ class JinjaView(object):
     def __call__(self, model):
         return self.template.render(**model)
 
+    def __deepcopy__(self, memodict):
+        res = object.__new__(JinjaView)
+        res._text = copy.deepcopy(self._text, memodict)
+
+        # regenerate the template on a deepcopy
+        res._regentemplate = True
+        res._templatecache = None
+
+        return res
+
     @property
     def template(self):
         """Get the Compiled Template

nova/openstack/common/report/views/json/generic.py

@@ -25,8 +25,9 @@ such strings specially)
 
 import copy
 
-from nova.openstack.common import jsonutils as json
+from oslo.serialization import jsonutils as json
-import nova.openstack.common.report.utils as utils
+
+from nova.openstack.common.report import utils as utils
 
 
 class BasicKeyValueView(object):
@@ -56,10 +57,10 @@ class KeyValueView(object):
     def __call__(self, model):
         # this part deals with subviews that were already serialized
         cpy = copy.deepcopy(model)
-        for key, valstr in model.items():
+        for key in model.keys():
-            if getattr(valstr, '__is_json__', False):
+            if getattr(model[key], '__is_json__', False):
-                cpy[key] = json.loads(valstr)
+                cpy[key] = json.loads(model[key])
 
-        res = utils.StringWithAttrs(json.dumps(cpy.data))
+        res = utils.StringWithAttrs(json.dumps(cpy.data, sort_keys=True))
         res.__is_json__ = True
         return res

nova/openstack/common/report/views/text/generic.py

@@ -120,7 +120,7 @@ class KeyValueView(object):
                     if self.before_dict is not None:
                         res.insert(0, self.before_dict)
 
-                for key in root:
+                for key in sorted(root):
                     res.extend(serialize(root[key], key, indent + 1))
             elif (isinstance(root, col.Sequence) and
                     not isinstance(root, six.string_types)):
@@ -129,7 +129,7 @@ class KeyValueView(object):
                     if self.before_list is not None:
                         res.insert(0, self.before_list)
 
-                for val in root:
+                for val in sorted(root, key=str):
                     res.extend(serialize(val, None, indent + 1))
             else:
                 str_root = str(root)
@@ -172,7 +172,7 @@ class TableView(object):
         self.table_prop_name = table_prop_name
         self.column_names = column_names
         self.column_values = column_values
-        self.column_width = (72 - len(column_names) + 1) / len(column_names)
+        self.column_width = (72 - len(column_names) + 1) // len(column_names)
 
         column_headers = "|".join(
             "{ch[" + str(n) + "]: ^" + str(self.column_width) + "}"

nova/openstack/common/report/views/text/threading.py

@@ -19,7 +19,7 @@ visualizing threads, green threads, and stack traces
 in human-readable form.
 """
 
-import nova.openstack.common.report.views.jinja_view as jv
+from nova.openstack.common.report.views import jinja_view as jv
 
 
 class StackTraceView(jv.JinjaView):
@@ -52,7 +52,7 @@ class GreenThreadView(object):
     """A Green Thread View
 
     This view displays a green thread provided by the data
-    model :class:`openstack.common.report.models.threading.GreenThreadModel`  # noqa
+    model :class:`openstack.common.report.models.threading.GreenThreadModel`
     """
 
     FORMAT_STR = "------{thread_str: ^60}------" + "\n" + "{stack_trace}"

nova/openstack/common/report/views/xml/generic.py

@@ -29,7 +29,7 @@ import xml.etree.ElementTree as ET
 
 import six
 
-import nova.openstack.common.report.utils as utils
+from nova.openstack.common.report import utils as utils
 
 
 class KeyValueView(object):
@@ -66,11 +66,11 @@ class KeyValueView(object):
             res = ET.Element(rootkeyname)
 
             if isinstance(rootmodel, col.Mapping):
-                for key in rootmodel:
+                for key in sorted(rootmodel):
                     res.append(serialize(rootmodel[key], key))
             elif (isinstance(rootmodel, col.Sequence)
                     and not isinstance(rootmodel, six.string_types)):
-                for val in rootmodel:
+                for val in sorted(rootmodel, key=str):
                     res.append(serialize(val, 'item'))
             elif ET.iselement(rootmodel):
                 res.append(rootmodel)
@@ -79,7 +79,9 @@ class KeyValueView(object):
 
             return res
 
-        res = utils.StringWithAttrs(ET.tostring(serialize(cpy,
+        str_ = ET.tostring(serialize(cpy,
-                                                          self.wrapper_name)))
+                                     self.wrapper_name),
+                           encoding="utf-8").decode("utf-8")
+        res = utils.StringWithAttrs(str_)
         res.__is_xml__ = True
         return res

nova/openstack/common/service.py

@@ -38,14 +38,12 @@ from eventlet import event
 from oslo.config import cfg
 
 from nova.openstack.common import eventlet_backdoor
-from nova.openstack.common.gettextutils import _LE, _LI, _LW
+from nova.openstack.common._i18n import _LE, _LI, _LW
-from nova.openstack.common import importutils
 from nova.openstack.common import log as logging
 from nova.openstack.common import systemd
 from nova.openstack.common import threadgroup
 
 
-rpc = importutils.try_import('nova.openstack.common.rpc')
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
@@ -180,16 +178,11 @@ class ServiceLauncher(Launcher):
             status = exc.code
         finally:
             self.stop()
-            if rpc:
-                try:
-                    rpc.cleanup()
-                except Exception:
-                    # We're shutting down, so it doesn't matter at this point.
-                    LOG.exception(_LE('Exception during rpc cleanup.'))
 
         return status, signo
 
     def wait(self, ready_callback=None):
+        systemd.notify_once()
         while True:
             self.handle_signal()
             status, signo = self._wait_for_exit_or_signal(ready_callback)
@@ -267,7 +260,7 @@ class ProcessLauncher(object):
             launcher.wait()
         except SignalExit as exc:
             signame = _signo_to_signame(exc.signo)
-            LOG.info(_LI('Caught %s, exiting'), signame)
+            LOG.info(_LI('Child caught %s, exiting'), signame)
             status = exc.code
             signo = exc.signo
         except SystemExit as exc:
@@ -382,6 +375,7 @@ class ProcessLauncher(object):
     def wait(self):
         """Loop waiting on children to die and respawning as necessary."""
 
+        systemd.notify_once()
         LOG.debug('Full set of CONF:')
         CONF.log_opt_values(LOG, std_logging.DEBUG)
 
@@ -389,7 +383,10 @@ class ProcessLauncher(object):
             while True:
                 self.handle_signal()
                 self._respawn_children()
-                if self.sigcaught:
+                # No signal means that stop was called.  Don't clean up here.
+                if not self.sigcaught:
+                    return
+
                 signame = _signo_to_signame(self.sigcaught)
                 LOG.info(_LI('Caught %s, stopping children'), signame)
                 if not _is_sighup_and_daemon(self.sigcaught):
@@ -402,6 +399,11 @@ class ProcessLauncher(object):
         except eventlet.greenlet.GreenletExit:
             LOG.info(_LI("Wait called after thread killed.  Cleaning up."))
 
+        self.stop()
+
+    def stop(self):
+        """Terminate child processes and wait on each."""
+        self.running = False
         for pid in self.children:
             try:
                 os.kill(pid, signal.SIGTERM)
@@ -488,7 +490,6 @@ class Services(object):
 
         """
         service.start()
-        systemd.notify_once()
         done.wait()
 
 

nova/openstack/common/sslutils.py

@@ -17,7 +17,7 @@ import ssl
 
 from oslo.config import cfg
 
-from nova.openstack.common.gettextutils import _
+from nova.openstack.common._i18n import _
 
 
 ssl_opts = [

nova/openstack/common/threadgroup.py

@@ -85,7 +85,7 @@ class ThreadGroup(object):
     def thread_done(self, thread):
         self.threads.remove(thread)
 
-    def stop(self):
+    def _stop_threads(self):
         current = threading.current_thread()
 
         # Iterate over a copy of self.threads so thread_done doesn't
@@ -99,6 +99,7 @@ class ThreadGroup(object):
             except Exception as ex:
                 LOG.exception(ex)
 
+    def stop_timers(self):
         for x in self.timers:
             try:
                 x.stop()
@@ -106,6 +107,23 @@ class ThreadGroup(object):
                 LOG.exception(ex)
         self.timers = []
 
+    def stop(self, graceful=False):
+        """stop function has the option of graceful=True/False.
+
+        * In case of graceful=True, wait for all threads to be finished.
+          Never kill threads.
+        * In case of graceful=False, kill threads immediately.
+        """
+        self.stop_timers()
+        if graceful:
+            # In case of graceful=True, wait for all threads to be
+            # finished, never kill threads
+            self.wait()
+        else:
+            # In case of graceful=False(Default), kill threads
+            # immediately
+            self._stop_threads()
+
     def wait(self):
         for x in self.timers:
             try:

nova/openstack/common/versionutils.py

@@ -18,10 +18,12 @@ Helpers for comparing version strings.
 """
 
 import functools
+import inspect
 
 import pkg_resources
+import six
 
-from nova.openstack.common.gettextutils import _
+from nova.openstack.common._i18n import _
 from nova.openstack.common import log as logging
 
 
@@ -71,6 +73,7 @@ class deprecated(object):
     HAVANA = 'H'
     ICEHOUSE = 'I'
     JUNO = 'J'
+    KILO = 'K'
 
     _RELEASES = {
         # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
@@ -81,6 +84,7 @@ class deprecated(object):
         'H': 'Havana',
         'I': 'Icehouse',
         'J': 'Juno',
+        'K': 'Kilo',
     }
 
     _deprecated_msg_with_alternative = _(
@@ -114,16 +118,34 @@ class deprecated(object):
         self.remove_in = remove_in
         self.what = what
 
-    def __call__(self, func):
+    def __call__(self, func_or_cls):
         if not self.what:
-            self.what = func.__name__ + '()'
+            self.what = func_or_cls.__name__ + '()'
-
-        @functools.wraps(func)
-        def wrapped(*args, **kwargs):
         msg, details = self._build_message()
+
+        if inspect.isfunction(func_or_cls):
+
+            @six.wraps(func_or_cls)
+            def wrapped(*args, **kwargs):
                 LOG.deprecated(msg, details)
-            return func(*args, **kwargs)
+                return func_or_cls(*args, **kwargs)
             return wrapped
+        elif inspect.isclass(func_or_cls):
+            orig_init = func_or_cls.__init__
+
+            # TODO(tsufiev): change `functools` module to `six` as
+            # soon as six 1.7.4 (with fix for passing `assigned`
+            # argument to underlying `functools.wraps`) is released
+            # and added to the nova-incubator requrements
+            @functools.wraps(orig_init, assigned=('__name__', '__doc__'))
+            def new_init(self, *args, **kwargs):
+                LOG.deprecated(msg, details)
+                orig_init(self, *args, **kwargs)
+            func_or_cls.__init__ = new_init
+            return func_or_cls
+        else:
+            raise TypeError('deprecated can be used only with functions or '
+                            'classes')
 
     def _get_safe_to_remove_release(self, release):
         # TODO(dstanek): this method will have to be reimplemented once

nova/tests/virt/libvirt/test_driver.py

@@ -11044,6 +11044,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase):
                                             _fake_network_info(self.stubs, 1))
 
     def test_cleanup_resize_same_host(self):
+        CONF.set_override('policy_dirs', [])
         ins_ref = self._create_instance({'host': CONF.host})
 
         def fake_os_path_exists(path):
@@ -11064,6 +11065,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase):
                                             _fake_network_info(self.stubs, 1))
 
     def test_cleanup_resize_not_same_host(self):
+        CONF.set_override('policy_dirs', [])
         host = 'not' + CONF.host
         ins_ref = self._create_instance({'host': host})
 

nova/virt/libvirt/driver.py

@@ -51,6 +51,7 @@ from lxml import etree
 from oslo.config import cfg
 from oslo.utils import excutils
 from oslo.utils import importutils
+from oslo.utils import strutils
 from oslo.utils import timeutils
 from oslo.utils import units
 import six
@@ -4133,7 +4134,7 @@ class LibvirtDriver(driver.ComputeDriver):
                 'block_device_info': block_device_info})
         # NOTE(mriedem): block_device_info can contain auth_password so we
         # need to sanitize the password in the message.
-        LOG.debug(logging.mask_password(msg), instance=instance)
+        LOG.debug(strutils.mask_password(msg), instance=instance)
         conf = self._get_guest_config(instance, network_info, image_meta,
                                       disk_info, rescue, block_device_info,
                                       context)

nova/virt/libvirt/volume.py

@@ -22,6 +22,7 @@ import time
 import urllib2
 
 from oslo.config import cfg
+from oslo.utils import strutils
 import six
 import six.moves.urllib.parse as urlparse
 
@@ -238,7 +239,7 @@ class LibvirtISCSIVolumeDriver(LibvirtBaseVolumeDriver):
                {'command': iscsi_command, 'out': out, 'err': err})
         # NOTE(bpokorny): iscsi_command can contain passwords so we need to
         # sanitize the password in the message.
-        LOG.debug(logging.mask_password(msg))
+        LOG.debug(strutils.mask_password(msg))
         return (out, err)
 
     def _iscsiadm_update(self, iscsi_properties, property_key, property_value,

nova/virt/vmwareapi/vmops.py

@@ -26,6 +26,7 @@ import time
 import decorator
 from oslo.config import cfg
 from oslo.utils import excutils
+from oslo.utils import strutils
 from oslo.utils import units
 from oslo.vmware import exceptions as vexc
 
@@ -467,7 +468,7 @@ class VMwareVMOps(object):
             msg = "Block device information present: %s" % block_device_info
             # NOTE(mriedem): block_device_info can contain an auth_password
             # so we have to scrub the message before logging it.
-            LOG.debug(logging.mask_password(msg), instance=instance)
+            LOG.debug(strutils.mask_password(msg), instance=instance)
 
             for root_disk in block_device_mapping:
                 connection_info = root_disk['connection_info']

nova/virt/xenapi/vmops.py

@@ -345,7 +345,7 @@ class VMOps(object):
             msg = "block device info: %s" % block_device_info
             # NOTE(mriedem): block_device_info can contain an auth_password
             # so we have to scrub the message before logging it.
-            LOG.debug(logging.mask_password(msg), instance=instance)
+            LOG.debug(strutils.mask_password(msg), instance=instance)
             root_device_name = block_device_info['root_device_name']
 
             for bdm in block_device_info['block_device_mapping']:

requirements.txt

@@ -35,12 +35,13 @@ stevedore>=1.0.0  # Apache-2.0
 websockify>=0.6.0,<0.7
 wsgiref>=0.1.2
 oslo.config>=1.4.0  # Apache-2.0
+oslo.serialization>=1.0.0               # Apache-2.0
+oslo.utils>=1.0.0                       # Apache-2.0
 oslo.db>=1.0.0  # Apache-2.0
 oslo.rootwrap>=1.3.0
 pycadf>=0.6.0
 oslo.messaging>=1.4.0
 oslo.i18n>=1.0.0  # Apache-2.0
-oslo.utils>=1.0.0                       # Apache-2.0
 lockfile>=0.8
 simplejson>=2.2.0
 rfc3986>=0.2.0  # Apache-2.0

tools/install_venv_common.py

@@ -125,7 +125,7 @@ class InstallVenv(object):
         parser.add_option('-n', '--no-site-packages',
                           action='store_true',
                           help="Do not inherit packages from global Python "
-                               "install")
+                               "install.")
         return parser.parse_args(argv[1:])[0]