From 0bfade69eb36140ce400e37a447bc44f0664d552 Mon Sep 17 00:00:00 2001
From: Davanum Srinivas <dims@linux.vnet.ibm.com>
Date: Fri, 3 Oct 2014 12:09:02 -0400
Subject: [PATCH] Sync with latest oslo-incubator

Changes include:
55ca7c3 Split cliutils
1b44601 Allow dictionary lookup in credentials with dot notation
6c706c5 Delete graduated serialization files
3edbfb3 remove caching param from prettytable call
9ce1d96 Fix i18n import
5d40e14 Remove code that moved to oslo.i18n
a4be4ed Remove graduated config modules
6ff6b4b Switch oslo-incubator to use oslo.utils and remove old modules
70189c9 Remove graduated test and fixtures libraries
8b71f57 Delete graduated db files
6ede600 rpc, notifier: remove deprecated modules
6b65a41 Fix build break - switch from str to hash for lookup
f76f44c Delete the token and endpoint on expiry of token of client
aebb58f Fix typo to show correct log message
1131b56 Enabled mask_password to handle byte code strings
6b048e7 Let oslotest manage the six.move setting for mox

Change-Id: Ic588aec0fc0291440feaac3439836054bd3608cd
---
 nova/api/openstack/wsgi.py                    |   3 +-
 nova/openstack/common/__init__.py             |  17 --
 nova/openstack/common/_i18n.py                |  40 +++
 nova/openstack/common/cliutils.py             | 227 +++++++++++++++++-
 nova/openstack/common/eventlet_backdoor.py    |   3 +-
 nova/openstack/common/fileutils.py            |   3 +-
 nova/openstack/common/fixture/__init__.py     |  17 ++
 nova/openstack/common/fixture/lockutils.py    |   2 +-
 nova/openstack/common/imageutils.py           |   7 +-
 nova/openstack/common/lockutils.py            |   2 +-
 nova/openstack/common/log.py                  |  11 +-
 nova/openstack/common/loopingcall.py          |   2 +-
 nova/openstack/common/memorycache.py          |   1 -
 nova/openstack/common/middleware/base.py      |   1 +
 .../common/middleware/catch_errors.py         |  46 ++++
 .../common/middleware/correlation_id.py       |  31 +++
 nova/openstack/common/middleware/debug.py     |  63 +++++
 .../openstack/common/middleware/request_id.py |   3 +
 nova/openstack/common/middleware/sizelimit.py |  85 +++++++
 nova/openstack/common/periodic_task.py        |   2 +-
 nova/openstack/common/policy.py               |  62 +++--
 nova/openstack/common/processutils.py         |  15 +-
 .../common/report/generators/conf.py          |   8 +-
 .../common/report/generators/threading.py     |   6 +-
 .../common/report/generators/version.py       |   6 +-
 .../common/report/guru_meditation_report.py   |  44 +++-
 nova/openstack/common/report/models/base.py   |   5 +
 nova/openstack/common/report/models/conf.py   |   6 +-
 .../common/report/models/threading.py         |  12 +-
 .../openstack/common/report/models/version.py |   8 +-
 .../report/models/with_default_views.py       |  18 +-
 nova/openstack/common/report/report.py        |  22 +-
 .../common/report/views/jinja_view.py         |  12 +
 .../common/report/views/json/generic.py       |  13 +-
 .../common/report/views/text/generic.py       |   6 +-
 .../common/report/views/text/threading.py     |   4 +-
 .../common/report/views/xml/generic.py        |  12 +-
 nova/openstack/common/service.py              |  29 +--
 nova/openstack/common/sslutils.py             |   2 +-
 nova/openstack/common/threadgroup.py          |  20 +-
 nova/openstack/common/versionutils.py         |  40 ++-
 nova/tests/virt/libvirt/test_driver.py        |   2 +
 nova/virt/libvirt/driver.py                   |   3 +-
 nova/virt/libvirt/volume.py                   |   3 +-
 nova/virt/vmwareapi/vmops.py                  |   3 +-
 nova/virt/xenapi/vmops.py                     |   2 +-
 requirements.txt                              |   3 +-
 tools/install_venv_common.py                  |   2 +-
 48 files changed, 762 insertions(+), 172 deletions(-)
 create mode 100644 nova/openstack/common/_i18n.py
 create mode 100644 nova/openstack/common/middleware/catch_errors.py
 create mode 100644 nova/openstack/common/middleware/correlation_id.py
 create mode 100644 nova/openstack/common/middleware/debug.py
 create mode 100644 nova/openstack/common/middleware/sizelimit.py

diff --git a/nova/api/openstack/wsgi.py b/nova/api/openstack/wsgi.py
index deb9d3eb3898..c3c9d9c83d25 100644
--- a/nova/api/openstack/wsgi.py
+++ b/nova/api/openstack/wsgi.py
@@ -20,6 +20,7 @@ import time
 from xml.dom import minidom
 
 from lxml import etree
+from oslo.utils import strutils
 import six
 import webob
 
@@ -905,7 +906,7 @@ class Resource(wsgi.Application):
                     "%(body)s") % {'action': action,
                                    'body': unicode(body, 'utf-8'),
                                    'meth': str(meth)}
-            LOG.debug(logging.mask_password(msg))
+            LOG.debug(strutils.mask_password(msg))
         else:
             LOG.debug("Calling method '%(meth)s'",
                       {'meth': str(meth)})
diff --git a/nova/openstack/common/__init__.py b/nova/openstack/common/__init__.py
index d1223eaf7656..e69de29bb2d1 100644
--- a/nova/openstack/common/__init__.py
+++ b/nova/openstack/common/__init__.py
@@ -1,17 +0,0 @@
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import six
-
-
-six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))
diff --git a/nova/openstack/common/_i18n.py b/nova/openstack/common/_i18n.py
new file mode 100644
index 000000000000..c556573f4290
--- /dev/null
+++ b/nova/openstack/common/_i18n.py
@@ -0,0 +1,40 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""oslo.i18n integration module.
+
+See http://docs.openstack.org/developer/oslo.i18n/usage.html
+
+"""
+
+import oslo.i18n
+
+
+# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
+# application name when this module is synced into the separate
+# repository. It is OK to have more than one translation function
+# using the same domain, since there will still only be one message
+# catalog.
+_translators = oslo.i18n.TranslatorFactory(domain='nova')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical
diff --git a/nova/openstack/common/cliutils.py b/nova/openstack/common/cliutils.py
index 411bd58f3756..aa9354f7c0da 100644
--- a/nova/openstack/common/cliutils.py
+++ b/nova/openstack/common/cliutils.py
@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -14,20 +12,33 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+# W0603: Using the global statement
+# W0621: Redefining name %s from outer scope
+# pylint: disable=W0603,W0621
+
+from __future__ import print_function
+
+import getpass
 import inspect
+import os
+import sys
+import textwrap
+
+from oslo.utils import encodeutils
+from oslo.utils import strutils
+import prettytable
+import six
+from six import moves
+
+from nova.openstack.common._i18n import _
 
 
 class MissingArgs(Exception):
-
+    """Supplied arguments are not sufficient for calling a function."""
     def __init__(self, missing):
         self.missing = missing
-
-    def __str__(self):
-        if len(self.missing) == 1:
-            return "An argument is missing"
-        else:
-            return ("%(num)d arguments are missing" %
-                    dict(num=len(self.missing)))
+        msg = _("Missing arguments: %s") % ", ".join(missing)
+        super(MissingArgs, self).__init__(msg)
 
 
 def validate_args(fn, *args, **kwargs):
@@ -36,11 +47,11 @@ def validate_args(fn, *args, **kwargs):
     >>> validate_args(lambda a: None)
     Traceback (most recent call last):
         ...
-    MissingArgs: An argument is missing
+    MissingArgs: Missing argument(s): a
     >>> validate_args(lambda a, b, c, d: None, 0, c=1)
     Traceback (most recent call last):
         ...
-    MissingArgs: 2 arguments are missing
+    MissingArgs: Missing argument(s): b, d
 
     :param fn: the function to check
     :param arg: the positional arguments supplied
@@ -52,7 +63,7 @@ def validate_args(fn, *args, **kwargs):
     required_args = argspec.args[:len(argspec.args) - num_defaults]
 
     def isbound(method):
-        return getattr(method, 'im_self', None) is not None
+        return getattr(method, '__self__', None) is not None
 
     if isbound(fn):
         required_args.pop(0)
@@ -61,3 +72,193 @@ def validate_args(fn, *args, **kwargs):
     missing = missing[len(args):]
     if missing:
         raise MissingArgs(missing)
+
+
+def arg(*args, **kwargs):
+    """Decorator for CLI args.
+
+    Example:
+
+    >>> @arg("name", help="Name of the new entity")
+    ... def entity_create(args):
+    ...     pass
+    """
+    def _decorator(func):
+        add_arg(func, *args, **kwargs)
+        return func
+    return _decorator
+
+
+def env(*args, **kwargs):
+    """Returns the first environment variable set.
+
+    If all are empty, defaults to '' or keyword arg `default`.
+    """
+    for arg in args:
+        value = os.environ.get(arg)
+        if value:
+            return value
+    return kwargs.get('default', '')
+
+
+def add_arg(func, *args, **kwargs):
+    """Bind CLI arguments to a shell.py `do_foo` function."""
+
+    if not hasattr(func, 'arguments'):
+        func.arguments = []
+
+    # NOTE(sirp): avoid dups that can occur when the module is shared across
+    # tests.
+    if (args, kwargs) not in func.arguments:
+        # Because of the semantics of decorator composition if we just append
+        # to the options list positional options will appear to be backwards.
+        func.arguments.insert(0, (args, kwargs))
+
+
+def unauthenticated(func):
+    """Adds 'unauthenticated' attribute to decorated function.
+
+    Usage:
+
+    >>> @unauthenticated
+    ... def mymethod(f):
+    ...     pass
+    """
+    func.unauthenticated = True
+    return func
+
+
+def isunauthenticated(func):
+    """Checks if the function does not require authentication.
+
+    Mark such functions with the `@unauthenticated` decorator.
+
+    :returns: bool
+    """
+    return getattr(func, 'unauthenticated', False)
+
+
+def print_list(objs, fields, formatters=None, sortby_index=0,
+               mixed_case_fields=None, field_labels=None):
+    """Print a list or objects as a table, one row per object.
+
+    :param objs: iterable of :class:`Resource`
+    :param fields: attributes that correspond to columns, in order
+    :param formatters: `dict` of callables for field formatting
+    :param sortby_index: index of the field for sorting table rows
+    :param mixed_case_fields: fields corresponding to object attributes that
+        have mixed case names (e.g., 'serverId')
+    :param field_labels: Labels to use in the heading of the table, default to
+        fields.
+    """
+    formatters = formatters or {}
+    mixed_case_fields = mixed_case_fields or []
+    field_labels = field_labels or fields
+    if len(field_labels) != len(fields):
+        raise ValueError(_("Field labels list %(labels)s has different number "
+                           "of elements than fields list %(fields)s"),
+                         {'labels': field_labels, 'fields': fields})
+
+    if sortby_index is None:
+        kwargs = {}
+    else:
+        kwargs = {'sortby': field_labels[sortby_index]}
+    pt = prettytable.PrettyTable(field_labels)
+    pt.align = 'l'
+
+    for o in objs:
+        row = []
+        for field in fields:
+            if field in formatters:
+                row.append(formatters[field](o))
+            else:
+                if field in mixed_case_fields:
+                    field_name = field.replace(' ', '_')
+                else:
+                    field_name = field.lower().replace(' ', '_')
+                data = getattr(o, field_name, '')
+                row.append(data)
+        pt.add_row(row)
+
+    print(encodeutils.safe_encode(pt.get_string(**kwargs)))
+
+
+def print_dict(dct, dict_property="Property", wrap=0):
+    """Print a `dict` as a table of two columns.
+
+    :param dct: `dict` to print
+    :param dict_property: name of the first column
+    :param wrap: wrapping for the second column
+    """
+    pt = prettytable.PrettyTable([dict_property, 'Value'])
+    pt.align = 'l'
+    for k, v in six.iteritems(dct):
+        # convert dict to str to check length
+        if isinstance(v, dict):
+            v = six.text_type(v)
+        if wrap > 0:
+            v = textwrap.fill(six.text_type(v), wrap)
+        # if value has a newline, add in multiple rows
+        # e.g. fault with stacktrace
+        if v and isinstance(v, six.string_types) and r'\n' in v:
+            lines = v.strip().split(r'\n')
+            col1 = k
+            for line in lines:
+                pt.add_row([col1, line])
+                col1 = ''
+        else:
+            pt.add_row([k, v])
+    print(encodeutils.safe_encode(pt.get_string()))
+
+
+def get_password(max_password_prompts=3):
+    """Read password from TTY."""
+    verify = strutils.bool_from_string(env("OS_VERIFY_PASSWORD"))
+    pw = None
+    if hasattr(sys.stdin, "isatty") and sys.stdin.isatty():
+        # Check for Ctrl-D
+        try:
+            for __ in moves.range(max_password_prompts):
+                pw1 = getpass.getpass("OS Password: ")
+                if verify:
+                    pw2 = getpass.getpass("Please verify: ")
+                else:
+                    pw2 = pw1
+                if pw1 == pw2 and pw1:
+                    pw = pw1
+                    break
+        except EOFError:
+            pass
+    return pw
+
+
+def service_type(stype):
+    """Adds 'service_type' attribute to decorated function.
+
+    Usage:
+
+    .. code-block:: python
+
+       @service_type('volume')
+       def mymethod(f):
+       ...
+    """
+    def inner(f):
+        f.service_type = stype
+        return f
+    return inner
+
+
+def get_service_type(f):
+    """Retrieves service type from function."""
+    return getattr(f, 'service_type', None)
+
+
+def pretty_choice_list(l):
+    return ', '.join("'%s'" % i for i in l)
+
+
+def exit(msg=''):
+    if msg:
+        print (msg, file=sys.stderr)
+    sys.exit(1)
diff --git a/nova/openstack/common/eventlet_backdoor.py b/nova/openstack/common/eventlet_backdoor.py
index 16898db432ec..ec0d6eb04614 100644
--- a/nova/openstack/common/eventlet_backdoor.py
+++ b/nova/openstack/common/eventlet_backdoor.py
@@ -29,7 +29,7 @@ import eventlet.backdoor
 import greenlet
 from oslo.config import cfg
 
-from nova.openstack.common.gettextutils import _LI
+from nova.openstack.common._i18n import _LI
 from nova.openstack.common import log as logging
 
 help_for_backdoor_port = (
@@ -41,7 +41,6 @@ help_for_backdoor_port = (
     "chosen port is displayed in the service's log file.")
 eventlet_backdoor_opts = [
     cfg.StrOpt('backdoor_port',
-               default=None,
                help="Enable eventlet backdoor.  %s" % help_for_backdoor_port)
 ]
 
diff --git a/nova/openstack/common/fileutils.py b/nova/openstack/common/fileutils.py
index 12ae19830319..617a094986f6 100644
--- a/nova/openstack/common/fileutils.py
+++ b/nova/openstack/common/fileutils.py
@@ -18,7 +18,8 @@ import errno
 import os
 import tempfile
 
-from nova.openstack.common import excutils
+from oslo.utils import excutils
+
 from nova.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
diff --git a/nova/openstack/common/fixture/__init__.py b/nova/openstack/common/fixture/__init__.py
index e69de29bb2d1..d1223eaf7656 100644
--- a/nova/openstack/common/fixture/__init__.py
+++ b/nova/openstack/common/fixture/__init__.py
@@ -0,0 +1,17 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+
+six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))
diff --git a/nova/openstack/common/fixture/lockutils.py b/nova/openstack/common/fixture/lockutils.py
index 28e10dd4db70..420c2596d68d 100644
--- a/nova/openstack/common/fixture/lockutils.py
+++ b/nova/openstack/common/fixture/lockutils.py
@@ -48,4 +48,4 @@ class LockFixture(fixtures.Fixture):
     def setUp(self):
         super(LockFixture, self).setUp()
         self.addCleanup(self.mgr.__exit__, None, None, None)
-        self.mgr.__enter__()
+        self.lock = self.mgr.__enter__()
diff --git a/nova/openstack/common/imageutils.py b/nova/openstack/common/imageutils.py
index 31e444767164..d7bd8256f388 100644
--- a/nova/openstack/common/imageutils.py
+++ b/nova/openstack/common/imageutils.py
@@ -21,8 +21,9 @@ Helper methods to deal with images.
 
 import re
 
-from nova.openstack.common.gettextutils import _
-from nova.openstack.common import strutils
+from oslo.utils import strutils
+
+from nova.openstack.common._i18n import _
 
 
 class QemuImgInfo(object):
@@ -62,7 +63,7 @@ class QemuImgInfo(object):
         # Standardize on underscores/lc/no dash and no spaces
         # since qemu seems to have mixed outputs here... and
         # this format allows for better integration with python
-        # - ie for usage in kwargs and such...
+        # - i.e. for usage in kwargs and such...
         field = field.lower().strip()
         for c in (" ", "-"):
             field = field.replace(c, '_')
diff --git a/nova/openstack/common/lockutils.py b/nova/openstack/common/lockutils.py
index cea31544ec37..6d5258762dc5 100644
--- a/nova/openstack/common/lockutils.py
+++ b/nova/openstack/common/lockutils.py
@@ -29,7 +29,7 @@ import weakref
 from oslo.config import cfg
 
 from nova.openstack.common import fileutils
-from nova.openstack.common.gettextutils import _, _LE, _LI
+from nova.openstack.common._i18n import _, _LE, _LI
 
 
 LOG = logging.getLogger(__name__)
diff --git a/nova/openstack/common/log.py b/nova/openstack/common/log.py
index 5b8c6857912d..25e437bc0433 100644
--- a/nova/openstack/common/log.py
+++ b/nova/openstack/common/log.py
@@ -38,18 +38,15 @@ import sys
 import traceback
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
+from oslo.utils import importutils
 import six
 from six import moves
 
 _PY26 = sys.version_info[0:2] == (2, 6)
 
-from nova.openstack.common.gettextutils import _
-from nova.openstack.common import importutils
-from nova.openstack.common import jsonutils
+from nova.openstack.common._i18n import _
 from nova.openstack.common import local
-# NOTE(flaper87): Pls, remove when graduating this module
-# from the incubator.
-from nova.openstack.common.strutils import mask_password  # noqa
 
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
@@ -554,7 +551,7 @@ def _setup_logging_from_conf(project, version):
                 syslog = logging.handlers.SysLogHandler(facility=facility)
             log_root.addHandler(syslog)
         except socket.error:
-            log_root.error('Unable to add syslog handler. Verify that syslog'
+            log_root.error('Unable to add syslog handler. Verify that syslog '
                            'is running.')
 
 
diff --git a/nova/openstack/common/loopingcall.py b/nova/openstack/common/loopingcall.py
index 82411af92688..f2086b2e0ed5 100644
--- a/nova/openstack/common/loopingcall.py
+++ b/nova/openstack/common/loopingcall.py
@@ -21,7 +21,7 @@ import time
 from eventlet import event
 from eventlet import greenthread
 
-from nova.openstack.common.gettextutils import _LE, _LW
+from nova.openstack.common._i18n import _LE, _LW
 from nova.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
diff --git a/nova/openstack/common/memorycache.py b/nova/openstack/common/memorycache.py
index ede8c3855e9a..4826865a2043 100644
--- a/nova/openstack/common/memorycache.py
+++ b/nova/openstack/common/memorycache.py
@@ -17,7 +17,6 @@
 """Super simple fake memcache client."""
 
 from oslo.config import cfg
-
 from oslo.utils import timeutils
 
 memcache_opts = [
diff --git a/nova/openstack/common/middleware/base.py b/nova/openstack/common/middleware/base.py
index 20995498a30c..464a1ccd72e7 100644
--- a/nova/openstack/common/middleware/base.py
+++ b/nova/openstack/common/middleware/base.py
@@ -12,6 +12,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+
 """Base class(es) for WSGI Middleware."""
 
 import webob.dec
diff --git a/nova/openstack/common/middleware/catch_errors.py b/nova/openstack/common/middleware/catch_errors.py
new file mode 100644
index 000000000000..181d597ce278
--- /dev/null
+++ b/nova/openstack/common/middleware/catch_errors.py
@@ -0,0 +1,46 @@
+# Copyright (c) 2013 NEC Corporation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Middleware that provides high-level error handling.
+
+It catches all exceptions from subsequent applications in WSGI pipeline
+to hide internal errors from API response.
+"""
+import logging
+
+import webob.dec
+import webob.exc
+
+from nova.openstack.common._i18n import _LE
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+LOG = logging.getLogger(__name__)
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.CatchErrors')
+class CatchErrorsMiddleware(base.Middleware):
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        try:
+            response = req.get_response(self.application)
+        except Exception:
+            LOG.exception(_LE('An error occurred during '
+                              'processing the request: %s'))
+            response = webob.exc.HTTPInternalServerError()
+        return response
diff --git a/nova/openstack/common/middleware/correlation_id.py b/nova/openstack/common/middleware/correlation_id.py
new file mode 100644
index 000000000000..8e5a81a5e4d3
--- /dev/null
+++ b/nova/openstack/common/middleware/correlation_id.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2013 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Middleware that attaches a correlation id to WSGI request"""
+
+import uuid
+
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.CorrelationId')
+class CorrelationIdMiddleware(base.Middleware):
+
+    def process_request(self, req):
+        correlation_id = (req.headers.get("X_CORRELATION_ID") or
+                          str(uuid.uuid4()))
+        req.headers['X_CORRELATION_ID'] = correlation_id
diff --git a/nova/openstack/common/middleware/debug.py b/nova/openstack/common/middleware/debug.py
new file mode 100644
index 000000000000..5bd0ce002f89
--- /dev/null
+++ b/nova/openstack/common/middleware/debug.py
@@ -0,0 +1,63 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Debug middleware"""
+
+from __future__ import print_function
+
+import sys
+
+import six
+import webob.dec
+
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.Debug')
+class Debug(base.Middleware):
+    """Helper class that returns debug information.
+
+    Can be inserted into any WSGI application chain to get information about
+    the request and response.
+    """
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        print(("*" * 40) + " REQUEST ENVIRON")
+        for key, value in req.environ.items():
+            print(key, "=", value)
+        print()
+        resp = req.get_response(self.application)
+
+        print(("*" * 40) + " RESPONSE HEADERS")
+        for (key, value) in six.iteritems(resp.headers):
+            print(key, "=", value)
+        print()
+
+        resp.app_iter = self.print_generator(resp.app_iter)
+
+        return resp
+
+    @staticmethod
+    def print_generator(app_iter):
+        """Prints the contents of a wrapper string iterator when iterated."""
+        print(("*" * 40) + " BODY")
+        for part in app_iter:
+            sys.stdout.write(part)
+            sys.stdout.flush()
+            yield part
+        print()
diff --git a/nova/openstack/common/middleware/request_id.py b/nova/openstack/common/middleware/request_id.py
index c9b4be4f1fc5..03cf59416ee9 100644
--- a/nova/openstack/common/middleware/request_id.py
+++ b/nova/openstack/common/middleware/request_id.py
@@ -23,12 +23,15 @@ import webob.dec
 
 from nova.openstack.common import context
 from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
 
 
 ENV_REQUEST_ID = 'openstack.request_id'
 HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id'
 
 
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.RequestId')
 class RequestIdMiddleware(base.Middleware):
 
     @webob.dec.wsgify
diff --git a/nova/openstack/common/middleware/sizelimit.py b/nova/openstack/common/middleware/sizelimit.py
new file mode 100644
index 000000000000..208468777005
--- /dev/null
+++ b/nova/openstack/common/middleware/sizelimit.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2012 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Request Body limiting middleware.
+
+"""
+
+from oslo.config import cfg
+import webob.dec
+import webob.exc
+
+from nova.openstack.common._i18n import _
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+# default request size is 112k
+max_req_body_size = cfg.IntOpt('max_request_body_size',
+                               deprecated_name='osapi_max_request_body_size',
+                               default=114688,
+                               help='The maximum body size for each '
+                                    ' request, in bytes.')
+
+CONF = cfg.CONF
+CONF.register_opt(max_req_body_size)
+
+
+class LimitingReader(object):
+    """Reader to limit the size of an incoming request."""
+    def __init__(self, data, limit):
+        """Initiates LimitingReader object.
+
+        :param data: Underlying data object
+        :param limit: maximum number of bytes the reader should allow
+        """
+        self.data = data
+        self.limit = limit
+        self.bytes_read = 0
+
+    def __iter__(self):
+        for chunk in self.data:
+            self.bytes_read += len(chunk)
+            if self.bytes_read > self.limit:
+                msg = _("Request is too large.")
+                raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+            else:
+                yield chunk
+
+    def read(self, i=None):
+        result = self.data.read(i)
+        self.bytes_read += len(result)
+        if self.bytes_read > self.limit:
+            msg = _("Request is too large.")
+            raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+        return result
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.RequestBodySizeLimiter')
+class RequestBodySizeLimiter(base.Middleware):
+    """Limit the size of incoming requests."""
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        if (req.content_length is not None and
+                req.content_length > CONF.max_request_body_size):
+            msg = _("Request is too large.")
+            raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+        if req.content_length is None and req.is_body_readable:
+            limiter = LimitingReader(req.body_file,
+                                     CONF.max_request_body_size)
+            req.body_file = limiter
+        return self.application
diff --git a/nova/openstack/common/periodic_task.py b/nova/openstack/common/periodic_task.py
index ebbb98b614b4..2e32d0594e06 100644
--- a/nova/openstack/common/periodic_task.py
+++ b/nova/openstack/common/periodic_task.py
@@ -17,7 +17,7 @@ import time
 from oslo.config import cfg
 import six
 
-from nova.openstack.common.gettextutils import _, _LE, _LI
+from nova.openstack.common._i18n import _, _LE, _LI
 from nova.openstack.common import log as logging
 
 
diff --git a/nova/openstack/common/policy.py b/nova/openstack/common/policy.py
index 359bd7f4a468..4d7628163790 100644
--- a/nova/openstack/common/policy.py
+++ b/nova/openstack/common/policy.py
@@ -77,16 +77,17 @@ as it allows particular rules to be explicitly disabled.
 
 import abc
 import ast
+import os
 import re
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
 import six
 import six.moves.urllib.parse as urlparse
 import six.moves.urllib.request as urlrequest
 
 from nova.openstack.common import fileutils
-from nova.openstack.common.gettextutils import _, _LE
-from nova.openstack.common import jsonutils
+from nova.openstack.common._i18n import _, _LE, _LW
 from nova.openstack.common import log as logging
 
 
@@ -98,6 +99,10 @@ policy_opts = [
                default='default',
                help=_('Default rule. Enforced when a requested rule is not '
                       'found.')),
+    cfg.MultiStrOpt('policy_dirs',
+                    default=['policy.d'],
+                    help=_('The directories of policy configuration files is '
+                           'stored')),
 ]
 
 CONF = cfg.CONF
@@ -216,6 +221,7 @@ class Enforcer(object):
     def clear(self):
         """Clears Enforcer rules, policy's cache and policy's path."""
         self.set_rules({})
+        fileutils.delete_cached_file(self.policy_path)
         self.default_rule = None
         self.policy_path = None
 
@@ -232,31 +238,53 @@ class Enforcer(object):
 
         if self.use_conf:
             if not self.policy_path:
-                self.policy_path = self._get_policy_path()
+                self.policy_path = self._get_policy_path(self.policy_file)
 
+            self._load_policy_file(self.policy_path, force_reload)
+            for path in CONF.policy_dirs:
+                try:
+                    path = self._get_policy_path(path)
+                except cfg.ConfigFilesNotFoundError:
+                    LOG.warn(_LW("Can not find policy directories %s"), path)
+                    continue
+                self._walk_through_policy_directory(path,
+                                                    self._load_policy_file,
+                                                    force_reload, False)
+
+    def _walk_through_policy_directory(self, path, func, *args):
+        # We do not iterate over sub-directories.
+        policy_files = next(os.walk(path))[2]
+        policy_files.sort()
+        for policy_file in [p for p in policy_files if not p.startswith('.')]:
+            func(os.path.join(path, policy_file), *args)
+
+    def _load_policy_file(self, path, force_reload, overwrite=True):
             reloaded, data = fileutils.read_cached_file(
-                self.policy_path, force_reload=force_reload)
+                path, force_reload=force_reload)
             if reloaded or not self.rules:
                 rules = Rules.load_json(data, self.default_rule)
-                self.set_rules(rules)
+                self.set_rules(rules, overwrite)
                 LOG.debug("Rules successfully reloaded")
 
-    def _get_policy_path(self):
-        """Locate the policy json data file.
+    def _get_policy_path(self, path):
+        """Locate the policy json data file/path.
 
-        :param policy_file: Custom policy file to locate.
+        :param path: It's value can be a full path or related path. When
+                     full path specified, this function just returns the full
+                     path. When related path specified, this function will
+                     search configuration directories to find one that exists.
 
         :returns: The policy path
 
-        :raises: ConfigFilesNotFoundError if the file couldn't
+        :raises: ConfigFilesNotFoundError if the file/path couldn't
                  be located.
         """
-        policy_file = CONF.find_file(self.policy_file)
+        policy_path = CONF.find_file(path)
 
-        if policy_file:
-            return policy_file
+        if policy_path:
+            return policy_path
 
-        raise cfg.ConfigFilesNotFoundError((self.policy_file,))
+        raise cfg.ConfigFilesNotFoundError((path,))
 
     def enforce(self, rule, target, creds, do_raise=False,
                 exc=None, *args, **kwargs):
@@ -784,7 +812,7 @@ def _parse_text_rule(rule):
         return state.result
     except ValueError:
         # Couldn't parse the rule
-        LOG.exception(_LE("Failed to understand rule %r") % rule)
+        LOG.exception(_LE("Failed to understand rule %s") % rule)
 
         # Fail closed
         return FalseCheck()
@@ -875,7 +903,6 @@ class GenericCheck(Check):
             'Member':%(role.name)s
         """
 
-        # TODO(termie): do dict inspection via dot syntax
         try:
             match = self.match % target
         except KeyError:
@@ -888,7 +915,10 @@ class GenericCheck(Check):
             leftval = ast.literal_eval(self.kind)
         except ValueError:
             try:
-                leftval = creds[self.kind]
+                kind_parts = self.kind.split('.')
+                leftval = creds
+                for kind_part in kind_parts:
+                    leftval = leftval[kind_part]
             except KeyError:
                 return False
         return match == six.text_type(leftval)
diff --git a/nova/openstack/common/processutils.py b/nova/openstack/common/processutils.py
index 6e2471d7eaa2..9de8e90db2a6 100644
--- a/nova/openstack/common/processutils.py
+++ b/nova/openstack/common/processutils.py
@@ -18,7 +18,7 @@ System-level utilities and helper functions.
 """
 
 import errno
-import logging as stdlib_logging
+import logging
 import multiprocessing
 import os
 import random
@@ -27,11 +27,10 @@ import signal
 
 from eventlet.green import subprocess
 from eventlet import greenthread
+from oslo.utils import strutils
 import six
 
-from nova.openstack.common.gettextutils import _
-from nova.openstack.common import log as logging
-from nova.openstack.common import strutils
+from nova.openstack.common._i18n import _
 
 
 LOG = logging.getLogger(__name__)
@@ -116,8 +115,7 @@ def execute(*cmd, **kwargs):
                             execute this command. Defaults to false.
     :type shell:            boolean
     :param loglevel:        log level for execute commands.
-    :type loglevel:         int.  (Should be stdlib_logging.DEBUG or
-                            stdlib_logging.INFO)
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
     :returns:               (stdout, stderr) from process execution
     :raises:                :class:`UnknownArgumentError` on
                             receiving unknown arguments
@@ -133,7 +131,7 @@ def execute(*cmd, **kwargs):
     run_as_root = kwargs.pop('run_as_root', False)
     root_helper = kwargs.pop('root_helper', '')
     shell = kwargs.pop('shell', False)
-    loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+    loglevel = kwargs.pop('loglevel', logging.DEBUG)
 
     if isinstance(check_exit_code, bool):
         ignore_exit_code = not check_exit_code
@@ -142,8 +140,7 @@ def execute(*cmd, **kwargs):
         check_exit_code = [check_exit_code]
 
     if kwargs:
-        raise UnknownArgumentError(_('Got unknown keyword args '
-                                     'to utils.execute: %r') % kwargs)
+        raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
 
     if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
         if not root_helper:
diff --git a/nova/openstack/common/report/generators/conf.py b/nova/openstack/common/report/generators/conf.py
index df3f1dcd065b..f947ce2f64a5 100644
--- a/nova/openstack/common/report/generators/conf.py
+++ b/nova/openstack/common/report/generators/conf.py
@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack config generators
+"""Provides OpenStack config generators
 
 This module defines a class for configuration
 generators for generating the model in
@@ -21,17 +21,17 @@ generators for generating the model in
 
 from oslo.config import cfg
 
-import nova.openstack.common.report.models.conf as cm
+from nova.openstack.common.report.models import conf as cm
 
 
 class ConfigReportGenerator(object):
     """A Configuration Data Generator
 
     This generator returns
-    :class:`openstack.common.report.models.conf.ConfigModel` ,
+    :class:`openstack.common.report.models.conf.ConfigModel`,
     by default using the configuration options stored
     in :attr:`oslo.config.cfg.CONF`, which is where
-    Openstack stores everything.
+    OpenStack stores everything.
 
     :param cnf: the configuration option object
     :type cnf: :class:`oslo.config.cfg.ConfigOpts`
diff --git a/nova/openstack/common/report/generators/threading.py b/nova/openstack/common/report/generators/threading.py
index fadda1d0309c..5fa081538635 100644
--- a/nova/openstack/common/report/generators/threading.py
+++ b/nova/openstack/common/report/generators/threading.py
@@ -23,10 +23,10 @@ import sys
 
 import greenlet
 
-import nova.openstack.common.report.models.threading as tm
+from nova.openstack.common.report.models import threading as tm
 from nova.openstack.common.report.models import with_default_views as mwdv
-import nova.openstack.common.report.utils as rutils
-import nova.openstack.common.report.views.text.generic as text_views
+from nova.openstack.common.report import utils as rutils
+from nova.openstack.common.report.views.text import generic as text_views
 
 
 class ThreadReportGenerator(object):
diff --git a/nova/openstack/common/report/generators/version.py b/nova/openstack/common/report/generators/version.py
index 392e39725d11..65302d5b1237 100644
--- a/nova/openstack/common/report/generators/version.py
+++ b/nova/openstack/common/report/generators/version.py
@@ -12,15 +12,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack version generators
+"""Provides OpenStack version generators
 
-This module defines a class for Openstack
+This module defines a class for OpenStack
 version and package information
 generators for generating the model in
 :mod:`openstack.common.report.models.version`.
 """
 
-import nova.openstack.common.report.models.version as vm
+from nova.openstack.common.report.models import version as vm
 
 
 class PackageReportGenerator(object):
diff --git a/nova/openstack/common/report/guru_meditation_report.py b/nova/openstack/common/report/guru_meditation_report.py
index f256010fb229..6377fe5b6560 100644
--- a/nova/openstack/common/report/guru_meditation_report.py
+++ b/nova/openstack/common/report/guru_meditation_report.py
@@ -51,9 +51,13 @@ where stderr is logged for that given service.
 
 from __future__ import print_function
 
+import inspect
+import os
 import signal
 import sys
 
+from oslo.utils import timeutils
+
 from nova.openstack.common.report.generators import conf as cgen
 from nova.openstack.common.report.generators import threading as tgen
 from nova.openstack.common.report.generators import version as pgen
@@ -74,6 +78,8 @@ class GuruMeditation(object):
     MRO is correct.
     """
 
+    timestamp_fmt = "%Y%m%d%H%M%S"
+
     def __init__(self, version_obj, *args, **kwargs):
         self.version_obj = version_obj
 
@@ -97,13 +103,17 @@ class GuruMeditation(object):
             cls.persistent_sections = [[section_title, generator]]
 
     @classmethod
-    def setup_autorun(cls, version, signum=None):
+    def setup_autorun(cls, version, service_name=None,
+                      log_dir=None, signum=None):
         """Set Up Auto-Run
 
         This method sets up the Guru Meditation Report to automatically
-        get dumped to stderr when the given signal is received.
+        get dumped to stderr or a file in a given dir when the given signal
+        is received.
 
         :param version: the version object for the current product
+        :param service_name: this program name used to construct logfile name
+        :param logdir: path to a log directory where to create a file
         :param signum: the signal to associate with running the report
         """
 
@@ -113,18 +123,25 @@ class GuruMeditation(object):
 
         if signum:
             signal.signal(signum,
-                          lambda *args: cls.handle_signal(version, *args))
+                          lambda *args: cls.handle_signal(
+                              version, service_name, log_dir, *args))
 
     @classmethod
-    def handle_signal(cls, version, *args):
+    def handle_signal(cls, version, service_name, log_dir, *args):
         """The Signal Handler
 
         This method (indirectly) handles receiving a registered signal and
-        dumping the Guru Meditation Report to stderr.  This method is designed
-        to be curried into a proper signal handler by currying out the version
+        dumping the Guru Meditation Report to stderr or a file in a given dir.
+        If service name and log dir are not None, the report will be dumped to
+        a file named $service_name_gurumeditation_$current_time in the log_dir
+        directory.
+        This method is designed to be curried into a proper signal handler by
+        currying out the version
         parameter.
 
         :param version: the version object for the current product
+        :param service_name: this program name used to construct logfile name
+        :param logdir: path to a log directory where to create a file
         """
 
         try:
@@ -133,7 +150,20 @@ class GuruMeditation(object):
             print("Unable to run Guru Meditation Report!",
                   file=sys.stderr)
         else:
-            print(res, file=sys.stderr)
+            if log_dir:
+                service_name = service_name or os.path.basename(
+                    inspect.stack()[-1][1])
+                filename = "%s_gurumeditation_%s" % (
+                    service_name, timeutils.strtime(fmt=cls.timestamp_fmt))
+                filepath = os.path.join(log_dir, filename)
+                try:
+                    with open(filepath, "w") as dumpfile:
+                        dumpfile.write(res)
+                except Exception:
+                    print("Unable to dump Guru Meditation Report to file %s" %
+                          (filepath,), file=sys.stderr)
+            else:
+                print(res, file=sys.stderr)
 
     def _readd_sections(self):
         del self.sections[self.start_section_index:]
diff --git a/nova/openstack/common/report/models/base.py b/nova/openstack/common/report/models/base.py
index 90914ffe0714..7c5d7b32a343 100644
--- a/nova/openstack/common/report/models/base.py
+++ b/nova/openstack/common/report/models/base.py
@@ -81,6 +81,11 @@ class ReportModel(col.MutableMapping):
         return self.data.__contains__(key)
 
     def __getattr__(self, attrname):
+        # Needed for deepcopy in Python3. That will avoid an infinite loop
+        # in __getattr__ .
+        if 'data' not in self.__dict__:
+            self.data = {}
+
         try:
             return self.data[attrname]
         except KeyError:
diff --git a/nova/openstack/common/report/models/conf.py b/nova/openstack/common/report/models/conf.py
index 07830a6660b1..3c886493c64d 100644
--- a/nova/openstack/common/report/models/conf.py
+++ b/nova/openstack/common/report/models/conf.py
@@ -12,14 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack Configuration Model
+"""Provides OpenStack Configuration Model
 
 This module defines a class representing the data
 model for :mod:`oslo.config` configuration options
 """
 
-import nova.openstack.common.report.models.with_default_views as mwdv
-import nova.openstack.common.report.views.text.generic as generic_text_views
+from nova.openstack.common.report.models import with_default_views as mwdv
+from nova.openstack.common.report.views.text import generic as generic_text_views
 
 
 class ConfigModel(mwdv.ModelWithDefaultViews):
diff --git a/nova/openstack/common/report/models/threading.py b/nova/openstack/common/report/models/threading.py
index 411a5d9869c6..fc386cb38756 100644
--- a/nova/openstack/common/report/models/threading.py
+++ b/nova/openstack/common/report/models/threading.py
@@ -20,8 +20,8 @@ thread, and stack trace data models
 
 import traceback
 
-import nova.openstack.common.report.models.with_default_views as mwdv
-import nova.openstack.common.report.views.text.threading as text_views
+from nova.openstack.common.report.models import with_default_views as mwdv
+from nova.openstack.common.report.views.text import threading as text_views
 
 
 class StackTraceModel(mwdv.ModelWithDefaultViews):
@@ -42,12 +42,12 @@ class StackTraceModel(mwdv.ModelWithDefaultViews):
                 {'filename': fn, 'line': ln, 'name': nm, 'code': cd}
                 for fn, ln, nm, cd in traceback.extract_stack(stack_state)
             ]
-
-            if stack_state.f_exc_type is not None:
+            # FIXME(flepied): under Python3 f_exc_type doesn't exist
+            # anymore so we lose information about exceptions
+            if getattr(stack_state, 'f_exc_type', None) is not None:
                 self['root_exception'] = {
                     'type': stack_state.f_exc_type,
-                    'value': stack_state.f_exc_value
-                }
+                    'value': stack_state.f_exc_value}
             else:
                 self['root_exception'] = None
         else:
diff --git a/nova/openstack/common/report/models/version.py b/nova/openstack/common/report/models/version.py
index 2aac4c0e0141..2d8ea1de5942 100644
--- a/nova/openstack/common/report/models/version.py
+++ b/nova/openstack/common/report/models/version.py
@@ -12,14 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack Version Info Model
+"""Provides OpenStack Version Info Model
 
 This module defines a class representing the data
-model for Openstack package and version information
+model for OpenStack package and version information
 """
 
-import nova.openstack.common.report.models.with_default_views as mwdv
-import nova.openstack.common.report.views.text.generic as generic_text_views
+from nova.openstack.common.report.models import with_default_views as mwdv
+from nova.openstack.common.report.views.text import generic as generic_text_views
 
 
 class PackageModel(mwdv.ModelWithDefaultViews):
diff --git a/nova/openstack/common/report/models/with_default_views.py b/nova/openstack/common/report/models/with_default_views.py
index e53e3dfa0583..6ca8dbb14fcf 100644
--- a/nova/openstack/common/report/models/with_default_views.py
+++ b/nova/openstack/common/report/models/with_default_views.py
@@ -14,10 +14,10 @@
 
 import copy
 
-import nova.openstack.common.report.models.base as base_model
-import nova.openstack.common.report.views.json.generic as jsonviews
-import nova.openstack.common.report.views.text.generic as textviews
-import nova.openstack.common.report.views.xml.generic as xmlviews
+from nova.openstack.common.report.models import base as base_model
+from nova.openstack.common.report.views.json import generic as jsonviews
+from nova.openstack.common.report.views.text import generic as textviews
+from nova.openstack.common.report.views.xml import generic as xmlviews
 
 
 class ModelWithDefaultViews(base_model.ReportModel):
@@ -28,18 +28,18 @@ class ModelWithDefaultViews(base_model.ReportModel):
     when a submodel should have an attached view, but the view
     differs depending on the serialization format
 
-    Paramaters are as the superclass, with the exception
-    of any parameters ending in '_view': these parameters
+    Parameters are as the superclass, except for any
+    parameters ending in '_view': these parameters
     get stored as default views.
 
     The default 'default views' are
 
     text
-        :class:`openstack.common.views.text.generic.KeyValueView`
+        :class:`openstack.common.report.views.text.generic.KeyValueView`
     xml
-        :class:`openstack.common.views.xml.generic.KeyValueView`
+        :class:`openstack.common.report.views.xml.generic.KeyValueView`
     json
-        :class:`openstack.common.views.json.generic.KeyValueView`
+        :class:`openstack.common.report.views.json.generic.KeyValueView`
 
     .. function:: to_type()
 
diff --git a/nova/openstack/common/report/report.py b/nova/openstack/common/report/report.py
index 730ab4ac0c9f..9b4ae4bcfe3a 100644
--- a/nova/openstack/common/report/report.py
+++ b/nova/openstack/common/report/report.py
@@ -14,12 +14,12 @@
 
 """Provides Report classes
 
-This module defines various classes representing
-reports and report sections.  All reports take the
-form of a report class containing various report sections.
+This module defines various classes representing reports and report sections.
+All reports take the form of a report class containing various report
+sections.
 """
 
-import nova.openstack.common.report.views.text.header as header_views
+from nova.openstack.common.report.views.text import header as header_views
 
 
 class BasicReport(object):
@@ -28,7 +28,7 @@ class BasicReport(object):
     A Basic Report consists of a collection of :class:`ReportSection`
     objects, each of which contains a top-level model and generator.
     It collects these sections into a cohesive report which may then
-    be serialized by calling :func:`run`
+    be serialized by calling :func:`run`.
     """
 
     def __init__(self):
@@ -78,10 +78,9 @@ class BasicReport(object):
 class ReportSection(object):
     """A Report Section
 
-    A report section contains a generator and a top-level view.
-    When something attempts to serialize the section by calling
-    str() on it, the section runs the generator and calls the view
-    on the resulting model.
+    A report section contains a generator and a top-level view. When something
+    attempts to serialize the section by calling str() on it, the section runs
+    the generator and calls the view on the resulting model.
 
     .. seealso::
 
@@ -89,9 +88,8 @@ class ReportSection(object):
            :func:`BasicReport.add_section`
 
     :param view: the top-level view for this section
-    :param generator: the generator for this section which could be
-                      any callable object which takes
-                      no parameters and returns a data model
+    :param generator: the generator for this section
+      (any callable object which takes no parameters and returns a data model)
     """
 
     def __init__(self, view, generator):
diff --git a/nova/openstack/common/report/views/jinja_view.py b/nova/openstack/common/report/views/jinja_view.py
index a6f340e8e0ad..5f57dc34ab81 100644
--- a/nova/openstack/common/report/views/jinja_view.py
+++ b/nova/openstack/common/report/views/jinja_view.py
@@ -19,6 +19,8 @@ system for serialization.  For more information on Jinja, please
 see http://jinja.pocoo.org/ .
 """
 
+import copy
+
 import jinja2
 
 
@@ -79,6 +81,16 @@ class JinjaView(object):
     def __call__(self, model):
         return self.template.render(**model)
 
+    def __deepcopy__(self, memodict):
+        res = object.__new__(JinjaView)
+        res._text = copy.deepcopy(self._text, memodict)
+
+        # regenerate the template on a deepcopy
+        res._regentemplate = True
+        res._templatecache = None
+
+        return res
+
     @property
     def template(self):
         """Get the Compiled Template
diff --git a/nova/openstack/common/report/views/json/generic.py b/nova/openstack/common/report/views/json/generic.py
index 27fa03827e45..7c4fbc2901e4 100644
--- a/nova/openstack/common/report/views/json/generic.py
+++ b/nova/openstack/common/report/views/json/generic.py
@@ -25,8 +25,9 @@ such strings specially)
 
 import copy
 
-from nova.openstack.common import jsonutils as json
-import nova.openstack.common.report.utils as utils
+from oslo.serialization import jsonutils as json
+
+from nova.openstack.common.report import utils as utils
 
 
 class BasicKeyValueView(object):
@@ -56,10 +57,10 @@ class KeyValueView(object):
     def __call__(self, model):
         # this part deals with subviews that were already serialized
         cpy = copy.deepcopy(model)
-        for key, valstr in model.items():
-            if getattr(valstr, '__is_json__', False):
-                cpy[key] = json.loads(valstr)
+        for key in model.keys():
+            if getattr(model[key], '__is_json__', False):
+                cpy[key] = json.loads(model[key])
 
-        res = utils.StringWithAttrs(json.dumps(cpy.data))
+        res = utils.StringWithAttrs(json.dumps(cpy.data, sort_keys=True))
         res.__is_json__ = True
         return res
diff --git a/nova/openstack/common/report/views/text/generic.py b/nova/openstack/common/report/views/text/generic.py
index 7363833058ae..3b30a07079c9 100644
--- a/nova/openstack/common/report/views/text/generic.py
+++ b/nova/openstack/common/report/views/text/generic.py
@@ -120,7 +120,7 @@ class KeyValueView(object):
                     if self.before_dict is not None:
                         res.insert(0, self.before_dict)
 
-                for key in root:
+                for key in sorted(root):
                     res.extend(serialize(root[key], key, indent + 1))
             elif (isinstance(root, col.Sequence) and
                     not isinstance(root, six.string_types)):
@@ -129,7 +129,7 @@ class KeyValueView(object):
                     if self.before_list is not None:
                         res.insert(0, self.before_list)
 
-                for val in root:
+                for val in sorted(root, key=str):
                     res.extend(serialize(val, None, indent + 1))
             else:
                 str_root = str(root)
@@ -172,7 +172,7 @@ class TableView(object):
         self.table_prop_name = table_prop_name
         self.column_names = column_names
         self.column_values = column_values
-        self.column_width = (72 - len(column_names) + 1) / len(column_names)
+        self.column_width = (72 - len(column_names) + 1) // len(column_names)
 
         column_headers = "|".join(
             "{ch[" + str(n) + "]: ^" + str(self.column_width) + "}"
diff --git a/nova/openstack/common/report/views/text/threading.py b/nova/openstack/common/report/views/text/threading.py
index 4fba3eccd3ae..18ef087481c8 100644
--- a/nova/openstack/common/report/views/text/threading.py
+++ b/nova/openstack/common/report/views/text/threading.py
@@ -19,7 +19,7 @@ visualizing threads, green threads, and stack traces
 in human-readable form.
 """
 
-import nova.openstack.common.report.views.jinja_view as jv
+from nova.openstack.common.report.views import jinja_view as jv
 
 
 class StackTraceView(jv.JinjaView):
@@ -52,7 +52,7 @@ class GreenThreadView(object):
     """A Green Thread View
 
     This view displays a green thread provided by the data
-    model :class:`openstack.common.report.models.threading.GreenThreadModel`  # noqa
+    model :class:`openstack.common.report.models.threading.GreenThreadModel`
     """
 
     FORMAT_STR = "------{thread_str: ^60}------" + "\n" + "{stack_trace}"
diff --git a/nova/openstack/common/report/views/xml/generic.py b/nova/openstack/common/report/views/xml/generic.py
index 2023c854bb7d..2a66708e030b 100644
--- a/nova/openstack/common/report/views/xml/generic.py
+++ b/nova/openstack/common/report/views/xml/generic.py
@@ -29,7 +29,7 @@ import xml.etree.ElementTree as ET
 
 import six
 
-import nova.openstack.common.report.utils as utils
+from nova.openstack.common.report import utils as utils
 
 
 class KeyValueView(object):
@@ -66,11 +66,11 @@ class KeyValueView(object):
             res = ET.Element(rootkeyname)
 
             if isinstance(rootmodel, col.Mapping):
-                for key in rootmodel:
+                for key in sorted(rootmodel):
                     res.append(serialize(rootmodel[key], key))
             elif (isinstance(rootmodel, col.Sequence)
                     and not isinstance(rootmodel, six.string_types)):
-                for val in rootmodel:
+                for val in sorted(rootmodel, key=str):
                     res.append(serialize(val, 'item'))
             elif ET.iselement(rootmodel):
                 res.append(rootmodel)
@@ -79,7 +79,9 @@ class KeyValueView(object):
 
             return res
 
-        res = utils.StringWithAttrs(ET.tostring(serialize(cpy,
-                                                          self.wrapper_name)))
+        str_ = ET.tostring(serialize(cpy,
+                                     self.wrapper_name),
+                           encoding="utf-8").decode("utf-8")
+        res = utils.StringWithAttrs(str_)
         res.__is_xml__ = True
         return res
diff --git a/nova/openstack/common/service.py b/nova/openstack/common/service.py
index 80e345b9f224..f682b2b745d8 100644
--- a/nova/openstack/common/service.py
+++ b/nova/openstack/common/service.py
@@ -38,14 +38,12 @@ from eventlet import event
 from oslo.config import cfg
 
 from nova.openstack.common import eventlet_backdoor
-from nova.openstack.common.gettextutils import _LE, _LI, _LW
-from nova.openstack.common import importutils
+from nova.openstack.common._i18n import _LE, _LI, _LW
 from nova.openstack.common import log as logging
 from nova.openstack.common import systemd
 from nova.openstack.common import threadgroup
 
 
-rpc = importutils.try_import('nova.openstack.common.rpc')
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
 
@@ -180,16 +178,11 @@ class ServiceLauncher(Launcher):
             status = exc.code
         finally:
             self.stop()
-            if rpc:
-                try:
-                    rpc.cleanup()
-                except Exception:
-                    # We're shutting down, so it doesn't matter at this point.
-                    LOG.exception(_LE('Exception during rpc cleanup.'))
 
         return status, signo
 
     def wait(self, ready_callback=None):
+        systemd.notify_once()
         while True:
             self.handle_signal()
             status, signo = self._wait_for_exit_or_signal(ready_callback)
@@ -267,7 +260,7 @@ class ProcessLauncher(object):
             launcher.wait()
         except SignalExit as exc:
             signame = _signo_to_signame(exc.signo)
-            LOG.info(_LI('Caught %s, exiting'), signame)
+            LOG.info(_LI('Child caught %s, exiting'), signame)
             status = exc.code
             signo = exc.signo
         except SystemExit as exc:
@@ -382,6 +375,7 @@ class ProcessLauncher(object):
     def wait(self):
         """Loop waiting on children to die and respawning as necessary."""
 
+        systemd.notify_once()
         LOG.debug('Full set of CONF:')
         CONF.log_opt_values(LOG, std_logging.DEBUG)
 
@@ -389,9 +383,12 @@ class ProcessLauncher(object):
             while True:
                 self.handle_signal()
                 self._respawn_children()
-                if self.sigcaught:
-                    signame = _signo_to_signame(self.sigcaught)
-                    LOG.info(_LI('Caught %s, stopping children'), signame)
+                # No signal means that stop was called.  Don't clean up here.
+                if not self.sigcaught:
+                    return
+
+                signame = _signo_to_signame(self.sigcaught)
+                LOG.info(_LI('Caught %s, stopping children'), signame)
                 if not _is_sighup_and_daemon(self.sigcaught):
                     break
 
@@ -402,6 +399,11 @@ class ProcessLauncher(object):
         except eventlet.greenlet.GreenletExit:
             LOG.info(_LI("Wait called after thread killed.  Cleaning up."))
 
+        self.stop()
+
+    def stop(self):
+        """Terminate child processes and wait on each."""
+        self.running = False
         for pid in self.children:
             try:
                 os.kill(pid, signal.SIGTERM)
@@ -488,7 +490,6 @@ class Services(object):
 
         """
         service.start()
-        systemd.notify_once()
         done.wait()
 
 
diff --git a/nova/openstack/common/sslutils.py b/nova/openstack/common/sslutils.py
index 00e6173d07e2..2643f76d0b86 100644
--- a/nova/openstack/common/sslutils.py
+++ b/nova/openstack/common/sslutils.py
@@ -17,7 +17,7 @@ import ssl
 
 from oslo.config import cfg
 
-from nova.openstack.common.gettextutils import _
+from nova.openstack.common._i18n import _
 
 
 ssl_opts = [
diff --git a/nova/openstack/common/threadgroup.py b/nova/openstack/common/threadgroup.py
index d34310902b8e..ef45b5e33437 100644
--- a/nova/openstack/common/threadgroup.py
+++ b/nova/openstack/common/threadgroup.py
@@ -85,7 +85,7 @@ class ThreadGroup(object):
     def thread_done(self, thread):
         self.threads.remove(thread)
 
-    def stop(self):
+    def _stop_threads(self):
         current = threading.current_thread()
 
         # Iterate over a copy of self.threads so thread_done doesn't
@@ -99,6 +99,7 @@ class ThreadGroup(object):
             except Exception as ex:
                 LOG.exception(ex)
 
+    def stop_timers(self):
         for x in self.timers:
             try:
                 x.stop()
@@ -106,6 +107,23 @@ class ThreadGroup(object):
                 LOG.exception(ex)
         self.timers = []
 
+    def stop(self, graceful=False):
+        """stop function has the option of graceful=True/False.
+
+        * In case of graceful=True, wait for all threads to be finished.
+          Never kill threads.
+        * In case of graceful=False, kill threads immediately.
+        """
+        self.stop_timers()
+        if graceful:
+            # In case of graceful=True, wait for all threads to be
+            # finished, never kill threads
+            self.wait()
+        else:
+            # In case of graceful=False(Default), kill threads
+            # immediately
+            self._stop_threads()
+
     def wait(self):
         for x in self.timers:
             try:
diff --git a/nova/openstack/common/versionutils.py b/nova/openstack/common/versionutils.py
index 1facce7726da..c2179a821a51 100644
--- a/nova/openstack/common/versionutils.py
+++ b/nova/openstack/common/versionutils.py
@@ -18,10 +18,12 @@ Helpers for comparing version strings.
 """
 
 import functools
+import inspect
 
 import pkg_resources
+import six
 
-from nova.openstack.common.gettextutils import _
+from nova.openstack.common._i18n import _
 from nova.openstack.common import log as logging
 
 
@@ -71,6 +73,7 @@ class deprecated(object):
     HAVANA = 'H'
     ICEHOUSE = 'I'
     JUNO = 'J'
+    KILO = 'K'
 
     _RELEASES = {
         # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
@@ -81,6 +84,7 @@ class deprecated(object):
         'H': 'Havana',
         'I': 'Icehouse',
         'J': 'Juno',
+        'K': 'Kilo',
     }
 
     _deprecated_msg_with_alternative = _(
@@ -114,16 +118,34 @@ class deprecated(object):
         self.remove_in = remove_in
         self.what = what
 
-    def __call__(self, func):
+    def __call__(self, func_or_cls):
         if not self.what:
-            self.what = func.__name__ + '()'
+            self.what = func_or_cls.__name__ + '()'
+        msg, details = self._build_message()
 
-        @functools.wraps(func)
-        def wrapped(*args, **kwargs):
-            msg, details = self._build_message()
-            LOG.deprecated(msg, details)
-            return func(*args, **kwargs)
-        return wrapped
+        if inspect.isfunction(func_or_cls):
+
+            @six.wraps(func_or_cls)
+            def wrapped(*args, **kwargs):
+                LOG.deprecated(msg, details)
+                return func_or_cls(*args, **kwargs)
+            return wrapped
+        elif inspect.isclass(func_or_cls):
+            orig_init = func_or_cls.__init__
+
+            # TODO(tsufiev): change `functools` module to `six` as
+            # soon as six 1.7.4 (with fix for passing `assigned`
+            # argument to underlying `functools.wraps`) is released
+            # and added to the nova-incubator requrements
+            @functools.wraps(orig_init, assigned=('__name__', '__doc__'))
+            def new_init(self, *args, **kwargs):
+                LOG.deprecated(msg, details)
+                orig_init(self, *args, **kwargs)
+            func_or_cls.__init__ = new_init
+            return func_or_cls
+        else:
+            raise TypeError('deprecated can be used only with functions or '
+                            'classes')
 
     def _get_safe_to_remove_release(self, release):
         # TODO(dstanek): this method will have to be reimplemented once
diff --git a/nova/tests/virt/libvirt/test_driver.py b/nova/tests/virt/libvirt/test_driver.py
index 7a0115f84332..0d017a4d4dc1 100644
--- a/nova/tests/virt/libvirt/test_driver.py
+++ b/nova/tests/virt/libvirt/test_driver.py
@@ -11044,6 +11044,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase):
                                             _fake_network_info(self.stubs, 1))
 
     def test_cleanup_resize_same_host(self):
+        CONF.set_override('policy_dirs', [])
         ins_ref = self._create_instance({'host': CONF.host})
 
         def fake_os_path_exists(path):
@@ -11064,6 +11065,7 @@ class LibvirtDriverTestCase(test.NoDBTestCase):
                                             _fake_network_info(self.stubs, 1))
 
     def test_cleanup_resize_not_same_host(self):
+        CONF.set_override('policy_dirs', [])
         host = 'not' + CONF.host
         ins_ref = self._create_instance({'host': host})
 
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 089a9cbac940..dfc5a8dcf22b 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -51,6 +51,7 @@ from lxml import etree
 from oslo.config import cfg
 from oslo.utils import excutils
 from oslo.utils import importutils
+from oslo.utils import strutils
 from oslo.utils import timeutils
 from oslo.utils import units
 import six
@@ -4133,7 +4134,7 @@ class LibvirtDriver(driver.ComputeDriver):
                 'block_device_info': block_device_info})
         # NOTE(mriedem): block_device_info can contain auth_password so we
         # need to sanitize the password in the message.
-        LOG.debug(logging.mask_password(msg), instance=instance)
+        LOG.debug(strutils.mask_password(msg), instance=instance)
         conf = self._get_guest_config(instance, network_info, image_meta,
                                       disk_info, rescue, block_device_info,
                                       context)
diff --git a/nova/virt/libvirt/volume.py b/nova/virt/libvirt/volume.py
index a8913c1bb7ec..8f118ed07849 100644
--- a/nova/virt/libvirt/volume.py
+++ b/nova/virt/libvirt/volume.py
@@ -22,6 +22,7 @@ import time
 import urllib2
 
 from oslo.config import cfg
+from oslo.utils import strutils
 import six
 import six.moves.urllib.parse as urlparse
 
@@ -238,7 +239,7 @@ class LibvirtISCSIVolumeDriver(LibvirtBaseVolumeDriver):
                {'command': iscsi_command, 'out': out, 'err': err})
         # NOTE(bpokorny): iscsi_command can contain passwords so we need to
         # sanitize the password in the message.
-        LOG.debug(logging.mask_password(msg))
+        LOG.debug(strutils.mask_password(msg))
         return (out, err)
 
     def _iscsiadm_update(self, iscsi_properties, property_key, property_value,
diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py
index cd5869e26ace..f8e82a7e267a 100644
--- a/nova/virt/vmwareapi/vmops.py
+++ b/nova/virt/vmwareapi/vmops.py
@@ -26,6 +26,7 @@ import time
 import decorator
 from oslo.config import cfg
 from oslo.utils import excutils
+from oslo.utils import strutils
 from oslo.utils import units
 from oslo.vmware import exceptions as vexc
 
@@ -467,7 +468,7 @@ class VMwareVMOps(object):
             msg = "Block device information present: %s" % block_device_info
             # NOTE(mriedem): block_device_info can contain an auth_password
             # so we have to scrub the message before logging it.
-            LOG.debug(logging.mask_password(msg), instance=instance)
+            LOG.debug(strutils.mask_password(msg), instance=instance)
 
             for root_disk in block_device_mapping:
                 connection_info = root_disk['connection_info']
diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py
index 76ba7d2d45fa..6ca51d72aea1 100644
--- a/nova/virt/xenapi/vmops.py
+++ b/nova/virt/xenapi/vmops.py
@@ -345,7 +345,7 @@ class VMOps(object):
             msg = "block device info: %s" % block_device_info
             # NOTE(mriedem): block_device_info can contain an auth_password
             # so we have to scrub the message before logging it.
-            LOG.debug(logging.mask_password(msg), instance=instance)
+            LOG.debug(strutils.mask_password(msg), instance=instance)
             root_device_name = block_device_info['root_device_name']
 
             for bdm in block_device_info['block_device_mapping']:
diff --git a/requirements.txt b/requirements.txt
index 8f5eb92a0b3a..9d17e72e07bc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -35,12 +35,13 @@ stevedore>=1.0.0  # Apache-2.0
 websockify>=0.6.0,<0.7
 wsgiref>=0.1.2
 oslo.config>=1.4.0  # Apache-2.0
+oslo.serialization>=1.0.0               # Apache-2.0
+oslo.utils>=1.0.0                       # Apache-2.0
 oslo.db>=1.0.0  # Apache-2.0
 oslo.rootwrap>=1.3.0
 pycadf>=0.6.0
 oslo.messaging>=1.4.0
 oslo.i18n>=1.0.0  # Apache-2.0
-oslo.utils>=1.0.0                       # Apache-2.0
 lockfile>=0.8
 simplejson>=2.2.0
 rfc3986>=0.2.0  # Apache-2.0
diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py
index 46822e32933e..e279159abbc9 100644
--- a/tools/install_venv_common.py
+++ b/tools/install_venv_common.py
@@ -125,7 +125,7 @@ class InstallVenv(object):
         parser.add_option('-n', '--no-site-packages',
                           action='store_true',
                           help="Do not inherit packages from global Python "
-                               "install")
+                               "install.")
         return parser.parse_args(argv[1:])[0]