Sync with latest oslo-incubator

Changes include:
55ca7c3 Split cliutils
1b44601 Allow dictionary lookup in credentials with dot notation
6c706c5 Delete graduated serialization files
3edbfb3 remove caching param from prettytable call
9ce1d96 Fix i18n import
5d40e14 Remove code that moved to oslo.i18n
a4be4ed Remove graduated config modules
6ff6b4b Switch oslo-incubator to use oslo.utils and remove old modules
70189c9 Remove graduated test and fixtures libraries
8b71f57 Delete graduated db files
6ede600 rpc, notifier: remove deprecated modules
6b65a41 Fix build break - switch from str to hash for lookup
f76f44c Delete the token and endpoint on expiry of token of client
aebb58f Fix typo to show correct log message
1131b56 Enabled mask_password to handle byte code strings
6b048e7 Let oslotest manage the six.move setting for mox

Change-Id: Ic588aec0fc0291440feaac3439836054bd3608cd

nova/api/openstack/wsgi.py

@@ -20,6 +20,7 @@ import time
 from xml.dom import minidom
 
 from lxml import etree
+from oslo.utils import strutils
 import six
 import webob
 
@@ -905,7 +906,7 @@ class Resource(wsgi.Application):
                     "%(body)s") % {'action': action,
                                    'body': unicode(body, 'utf-8'),
                                    'meth': str(meth)}
-            LOG.debug(logging.mask_password(msg))
+            LOG.debug(strutils.mask_password(msg))
         else:
             LOG.debug("Calling method '%(meth)s'",
                       {'meth': str(meth)})

nova/openstack/common/__init__.py

@@ -1,17 +0,0 @@
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import six
-
-
-six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

nova/openstack/common/_i18n.py

@@ -0,0 +1,40 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""oslo.i18n integration module.
+
+See http://docs.openstack.org/developer/oslo.i18n/usage.html
+
+"""
+
+import oslo.i18n
+
+
+# NOTE(dhellmann): This reference to o-s-l-o will be replaced by the
+# application name when this module is synced into the separate
+# repository. It is OK to have more than one translation function
+# using the same domain, since there will still only be one message
+# catalog.
+_translators = oslo.i18n.TranslatorFactory(domain='nova')
+
+# The primary translation function using the well-known name "_"
+_ = _translators.primary
+
+# Translators for log levels.
+#
+# The abbreviated names are meant to reflect the usual use of a short
+# name like '_'. The "L" is for "log" and the other letter comes from
+# the level.
+_LI = _translators.log_info
+_LW = _translators.log_warning
+_LE = _translators.log_error
+_LC = _translators.log_critical

nova/openstack/common/cliutils.py

@@ -1,5 +1,3 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
 # Copyright 2012 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -14,20 +12,33 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+# W0603: Using the global statement
+# W0621: Redefining name %s from outer scope
+# pylint: disable=W0603,W0621
+
+from __future__ import print_function
+
+import getpass
 import inspect
+import os
+import sys
+import textwrap
+
+from oslo.utils import encodeutils
+from oslo.utils import strutils
+import prettytable
+import six
+from six import moves
+
+from nova.openstack.common._i18n import _
 
 
 class MissingArgs(Exception):
-
+    """Supplied arguments are not sufficient for calling a function."""
     def __init__(self, missing):
         self.missing = missing
-
-    def __str__(self):
-        if len(self.missing) == 1:
-            return "An argument is missing"
-        else:
-            return ("%(num)d arguments are missing" %
-                    dict(num=len(self.missing)))
+        msg = _("Missing arguments: %s") % ", ".join(missing)
+        super(MissingArgs, self).__init__(msg)
 
 
 def validate_args(fn, *args, **kwargs):
@@ -36,11 +47,11 @@ def validate_args(fn, *args, **kwargs):
     >>> validate_args(lambda a: None)
     Traceback (most recent call last):
         ...
-    MissingArgs: An argument is missing
+    MissingArgs: Missing argument(s): a
     >>> validate_args(lambda a, b, c, d: None, 0, c=1)
     Traceback (most recent call last):
         ...
-    MissingArgs: 2 arguments are missing
+    MissingArgs: Missing argument(s): b, d
 
     :param fn: the function to check
     :param arg: the positional arguments supplied
@@ -52,7 +63,7 @@ def validate_args(fn, *args, **kwargs):
     required_args = argspec.args[:len(argspec.args) - num_defaults]
 
     def isbound(method):
-        return getattr(method, 'im_self', None) is not None
+        return getattr(method, '__self__', None) is not None
 
     if isbound(fn):
         required_args.pop(0)
@@ -61,3 +72,193 @@ def validate_args(fn, *args, **kwargs):
     missing = missing[len(args):]
     if missing:
         raise MissingArgs(missing)
+
+
+def arg(*args, **kwargs):
+    """Decorator for CLI args.
+
+    Example:
+
+    >>> @arg("name", help="Name of the new entity")
+    ... def entity_create(args):
+    ...     pass
+    """
+    def _decorator(func):
+        add_arg(func, *args, **kwargs)
+        return func
+    return _decorator
+
+
+def env(*args, **kwargs):
+    """Returns the first environment variable set.
+
+    If all are empty, defaults to '' or keyword arg `default`.
+    """
+    for arg in args:
+        value = os.environ.get(arg)
+        if value:
+            return value
+    return kwargs.get('default', '')
+
+
+def add_arg(func, *args, **kwargs):
+    """Bind CLI arguments to a shell.py `do_foo` function."""
+
+    if not hasattr(func, 'arguments'):
+        func.arguments = []
+
+    # NOTE(sirp): avoid dups that can occur when the module is shared across
+    # tests.
+    if (args, kwargs) not in func.arguments:
+        # Because of the semantics of decorator composition if we just append
+        # to the options list positional options will appear to be backwards.
+        func.arguments.insert(0, (args, kwargs))
+
+
+def unauthenticated(func):
+    """Adds 'unauthenticated' attribute to decorated function.
+
+    Usage:
+
+    >>> @unauthenticated
+    ... def mymethod(f):
+    ...     pass
+    """
+    func.unauthenticated = True
+    return func
+
+
+def isunauthenticated(func):
+    """Checks if the function does not require authentication.
+
+    Mark such functions with the `@unauthenticated` decorator.
+
+    :returns: bool
+    """
+    return getattr(func, 'unauthenticated', False)
+
+
+def print_list(objs, fields, formatters=None, sortby_index=0,
+               mixed_case_fields=None, field_labels=None):
+    """Print a list or objects as a table, one row per object.
+
+    :param objs: iterable of :class:`Resource`
+    :param fields: attributes that correspond to columns, in order
+    :param formatters: `dict` of callables for field formatting
+    :param sortby_index: index of the field for sorting table rows
+    :param mixed_case_fields: fields corresponding to object attributes that
+        have mixed case names (e.g., 'serverId')
+    :param field_labels: Labels to use in the heading of the table, default to
+        fields.
+    """
+    formatters = formatters or {}
+    mixed_case_fields = mixed_case_fields or []
+    field_labels = field_labels or fields
+    if len(field_labels) != len(fields):
+        raise ValueError(_("Field labels list %(labels)s has different number "
+                           "of elements than fields list %(fields)s"),
+                         {'labels': field_labels, 'fields': fields})
+
+    if sortby_index is None:
+        kwargs = {}
+    else:
+        kwargs = {'sortby': field_labels[sortby_index]}
+    pt = prettytable.PrettyTable(field_labels)
+    pt.align = 'l'
+
+    for o in objs:
+        row = []
+        for field in fields:
+            if field in formatters:
+                row.append(formatters[field](o))
+            else:
+                if field in mixed_case_fields:
+                    field_name = field.replace(' ', '_')
+                else:
+                    field_name = field.lower().replace(' ', '_')
+                data = getattr(o, field_name, '')
+                row.append(data)
+        pt.add_row(row)
+
+    print(encodeutils.safe_encode(pt.get_string(**kwargs)))
+
+
+def print_dict(dct, dict_property="Property", wrap=0):
+    """Print a `dict` as a table of two columns.
+
+    :param dct: `dict` to print
+    :param dict_property: name of the first column
+    :param wrap: wrapping for the second column
+    """
+    pt = prettytable.PrettyTable([dict_property, 'Value'])
+    pt.align = 'l'
+    for k, v in six.iteritems(dct):
+        # convert dict to str to check length
+        if isinstance(v, dict):
+            v = six.text_type(v)
+        if wrap > 0:
+            v = textwrap.fill(six.text_type(v), wrap)
+        # if value has a newline, add in multiple rows
+        # e.g. fault with stacktrace
+        if v and isinstance(v, six.string_types) and r'\n' in v:
+            lines = v.strip().split(r'\n')
+            col1 = k
+            for line in lines:
+                pt.add_row([col1, line])
+                col1 = ''
+        else:
+            pt.add_row([k, v])
+    print(encodeutils.safe_encode(pt.get_string()))
+
+
+def get_password(max_password_prompts=3):
+    """Read password from TTY."""
+    verify = strutils.bool_from_string(env("OS_VERIFY_PASSWORD"))
+    pw = None
+    if hasattr(sys.stdin, "isatty") and sys.stdin.isatty():
+        # Check for Ctrl-D
+        try:
+            for __ in moves.range(max_password_prompts):
+                pw1 = getpass.getpass("OS Password: ")
+                if verify:
+                    pw2 = getpass.getpass("Please verify: ")
+                else:
+                    pw2 = pw1
+                if pw1 == pw2 and pw1:
+                    pw = pw1
+                    break
+        except EOFError:
+            pass
+    return pw
+
+
+def service_type(stype):
+    """Adds 'service_type' attribute to decorated function.
+
+    Usage:
+
+    .. code-block:: python
+
+       @service_type('volume')
+       def mymethod(f):
+       ...
+    """
+    def inner(f):
+        f.service_type = stype
+        return f
+    return inner
+
+
+def get_service_type(f):
+    """Retrieves service type from function."""
+    return getattr(f, 'service_type', None)
+
+
+def pretty_choice_list(l):
+    return ', '.join("'%s'" % i for i in l)
+
+
+def exit(msg=''):
+    if msg:
+        print (msg, file=sys.stderr)
+    sys.exit(1)

nova/openstack/common/eventlet_backdoor.py

@@ -29,7 +29,7 @@ import eventlet.backdoor
 import greenlet
 from oslo.config import cfg
 
-from nova.openstack.common.gettextutils import _LI
+from nova.openstack.common._i18n import _LI
 from nova.openstack.common import log as logging
 
 help_for_backdoor_port = (
@@ -41,7 +41,6 @@ help_for_backdoor_port = (
     "chosen port is displayed in the service's log file.")
 eventlet_backdoor_opts = [
     cfg.StrOpt('backdoor_port',
-               default=None,
                help="Enable eventlet backdoor.  %s" % help_for_backdoor_port)
 ]
 

nova/openstack/common/fileutils.py

@@ -18,7 +18,8 @@ import errno
 import os
 import tempfile
 
-from nova.openstack.common import excutils
+from oslo.utils import excutils
+
 from nova.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)

nova/openstack/common/fixture/__init__.py

@@ -0,0 +1,17 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import six
+
+
+six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

nova/openstack/common/fixture/lockutils.py

@@ -48,4 +48,4 @@ class LockFixture(fixtures.Fixture):
     def setUp(self):
         super(LockFixture, self).setUp()
         self.addCleanup(self.mgr.__exit__, None, None, None)
-        self.mgr.__enter__()
+        self.lock = self.mgr.__enter__()

nova/openstack/common/imageutils.py

@@ -21,8 +21,9 @@ Helper methods to deal with images.
 
 import re
 
-from nova.openstack.common.gettextutils import _
-from nova.openstack.common import strutils
+from oslo.utils import strutils
+
+from nova.openstack.common._i18n import _
 
 
 class QemuImgInfo(object):
@@ -62,7 +63,7 @@ class QemuImgInfo(object):
         # Standardize on underscores/lc/no dash and no spaces
         # since qemu seems to have mixed outputs here... and
         # this format allows for better integration with python
-        # - ie for usage in kwargs and such...
+        # - i.e. for usage in kwargs and such...
         field = field.lower().strip()
         for c in (" ", "-"):
             field = field.replace(c, '_')

nova/openstack/common/lockutils.py

@@ -29,7 +29,7 @@ import weakref
 from oslo.config import cfg
 
 from nova.openstack.common import fileutils
-from nova.openstack.common.gettextutils import _, _LE, _LI
+from nova.openstack.common._i18n import _, _LE, _LI
 
 
 LOG = logging.getLogger(__name__)

nova/openstack/common/log.py

@@ -38,18 +38,15 @@ import sys
 import traceback
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
+from oslo.utils import importutils
 import six
 from six import moves
 
 _PY26 = sys.version_info[0:2] == (2, 6)
 
-from nova.openstack.common.gettextutils import _
-from nova.openstack.common import importutils
-from nova.openstack.common import jsonutils
+from nova.openstack.common._i18n import _
 from nova.openstack.common import local
-# NOTE(flaper87): Pls, remove when graduating this module
-# from the incubator.
-from nova.openstack.common.strutils import mask_password  # noqa
 
 
 _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
@@ -554,7 +551,7 @@ def _setup_logging_from_conf(project, version):
                 syslog = logging.handlers.SysLogHandler(facility=facility)
             log_root.addHandler(syslog)
         except socket.error:
-            log_root.error('Unable to add syslog handler. Verify that syslog'
+            log_root.error('Unable to add syslog handler. Verify that syslog '
                            'is running.')
 
 

nova/openstack/common/loopingcall.py

@@ -21,7 +21,7 @@ import time
 from eventlet import event
 from eventlet import greenthread
 
-from nova.openstack.common.gettextutils import _LE, _LW
+from nova.openstack.common._i18n import _LE, _LW
 from nova.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)

nova/openstack/common/memorycache.py

@@ -17,7 +17,6 @@
 """Super simple fake memcache client."""
 
 from oslo.config import cfg
-
 from oslo.utils import timeutils
 
 memcache_opts = [

nova/openstack/common/middleware/base.py

@@ -12,6 +12,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+
 """Base class(es) for WSGI Middleware."""
 
 import webob.dec

nova/openstack/common/middleware/catch_errors.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2013 NEC Corporation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Middleware that provides high-level error handling.
+
+It catches all exceptions from subsequent applications in WSGI pipeline
+to hide internal errors from API response.
+"""
+import logging
+
+import webob.dec
+import webob.exc
+
+from nova.openstack.common._i18n import _LE
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+LOG = logging.getLogger(__name__)
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.CatchErrors')
+class CatchErrorsMiddleware(base.Middleware):
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        try:
+            response = req.get_response(self.application)
+        except Exception:
+            LOG.exception(_LE('An error occurred during '
+                              'processing the request: %s'))
+            response = webob.exc.HTTPInternalServerError()
+        return response

nova/openstack/common/middleware/correlation_id.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2013 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Middleware that attaches a correlation id to WSGI request"""
+
+import uuid
+
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.CorrelationId')
+class CorrelationIdMiddleware(base.Middleware):
+
+    def process_request(self, req):
+        correlation_id = (req.headers.get("X_CORRELATION_ID") or
+                          str(uuid.uuid4()))
+        req.headers['X_CORRELATION_ID'] = correlation_id

nova/openstack/common/middleware/debug.py

@@ -0,0 +1,63 @@
+# Copyright 2011 OpenStack Foundation.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Debug middleware"""
+
+from __future__ import print_function
+
+import sys
+
+import six
+import webob.dec
+
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.Debug')
+class Debug(base.Middleware):
+    """Helper class that returns debug information.
+
+    Can be inserted into any WSGI application chain to get information about
+    the request and response.
+    """
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        print(("*" * 40) + " REQUEST ENVIRON")
+        for key, value in req.environ.items():
+            print(key, "=", value)
+        print()
+        resp = req.get_response(self.application)
+
+        print(("*" * 40) + " RESPONSE HEADERS")
+        for (key, value) in six.iteritems(resp.headers):
+            print(key, "=", value)
+        print()
+
+        resp.app_iter = self.print_generator(resp.app_iter)
+
+        return resp
+
+    @staticmethod
+    def print_generator(app_iter):
+        """Prints the contents of a wrapper string iterator when iterated."""
+        print(("*" * 40) + " BODY")
+        for part in app_iter:
+            sys.stdout.write(part)
+            sys.stdout.flush()
+            yield part
+        print()

nova/openstack/common/middleware/request_id.py

@@ -23,12 +23,15 @@ import webob.dec
 
 from nova.openstack.common import context
 from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
 
 
 ENV_REQUEST_ID = 'openstack.request_id'
 HTTP_RESP_HEADER_REQUEST_ID = 'x-openstack-request-id'
 
 
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.RequestId')
 class RequestIdMiddleware(base.Middleware):
 
     @webob.dec.wsgify

nova/openstack/common/middleware/sizelimit.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2012 Red Hat, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Request Body limiting middleware.
+
+"""
+
+from oslo.config import cfg
+import webob.dec
+import webob.exc
+
+from nova.openstack.common._i18n import _
+from nova.openstack.common.middleware import base
+from nova.openstack.common import versionutils
+
+
+# default request size is 112k
+max_req_body_size = cfg.IntOpt('max_request_body_size',
+                               deprecated_name='osapi_max_request_body_size',
+                               default=114688,
+                               help='The maximum body size for each '
+                                    ' request, in bytes.')
+
+CONF = cfg.CONF
+CONF.register_opt(max_req_body_size)
+
+
+class LimitingReader(object):
+    """Reader to limit the size of an incoming request."""
+    def __init__(self, data, limit):
+        """Initiates LimitingReader object.
+
+        :param data: Underlying data object
+        :param limit: maximum number of bytes the reader should allow
+        """
+        self.data = data
+        self.limit = limit
+        self.bytes_read = 0
+
+    def __iter__(self):
+        for chunk in self.data:
+            self.bytes_read += len(chunk)
+            if self.bytes_read > self.limit:
+                msg = _("Request is too large.")
+                raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+            else:
+                yield chunk
+
+    def read(self, i=None):
+        result = self.data.read(i)
+        self.bytes_read += len(result)
+        if self.bytes_read > self.limit:
+            msg = _("Request is too large.")
+            raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+        return result
+
+
+@versionutils.deprecated(as_of=versionutils.deprecated.JUNO,
+                         in_favor_of='oslo.middleware.RequestBodySizeLimiter')
+class RequestBodySizeLimiter(base.Middleware):
+    """Limit the size of incoming requests."""
+
+    @webob.dec.wsgify
+    def __call__(self, req):
+        if (req.content_length is not None and
+                req.content_length > CONF.max_request_body_size):
+            msg = _("Request is too large.")
+            raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg)
+        if req.content_length is None and req.is_body_readable:
+            limiter = LimitingReader(req.body_file,
+                                     CONF.max_request_body_size)
+            req.body_file = limiter
+        return self.application

nova/openstack/common/periodic_task.py

@@ -17,7 +17,7 @@ import time
 from oslo.config import cfg
 import six
 
-from nova.openstack.common.gettextutils import _, _LE, _LI
+from nova.openstack.common._i18n import _, _LE, _LI
 from nova.openstack.common import log as logging
 
 

nova/openstack/common/policy.py

@@ -77,16 +77,17 @@ as it allows particular rules to be explicitly disabled.
 
 import abc
 import ast
+import os
 import re
 
 from oslo.config import cfg
+from oslo.serialization import jsonutils
 import six
 import six.moves.urllib.parse as urlparse
 import six.moves.urllib.request as urlrequest
 
 from nova.openstack.common import fileutils
-from nova.openstack.common.gettextutils import _, _LE
-from nova.openstack.common import jsonutils
+from nova.openstack.common._i18n import _, _LE, _LW
 from nova.openstack.common import log as logging
 
 
@@ -98,6 +99,10 @@ policy_opts = [
                default='default',
                help=_('Default rule. Enforced when a requested rule is not '
                       'found.')),
+    cfg.MultiStrOpt('policy_dirs',
+                    default=['policy.d'],
+                    help=_('The directories of policy configuration files is '
+                           'stored')),
 ]
 
 CONF = cfg.CONF
@@ -216,6 +221,7 @@ class Enforcer(object):
     def clear(self):
         """Clears Enforcer rules, policy's cache and policy's path."""
         self.set_rules({})
+        fileutils.delete_cached_file(self.policy_path)
         self.default_rule = None
         self.policy_path = None
 
@@ -232,31 +238,53 @@ class Enforcer(object):
 
         if self.use_conf:
             if not self.policy_path:
-                self.policy_path = self._get_policy_path()
+                self.policy_path = self._get_policy_path(self.policy_file)
 
+            self._load_policy_file(self.policy_path, force_reload)
+            for path in CONF.policy_dirs:
+                try:
+                    path = self._get_policy_path(path)
+                except cfg.ConfigFilesNotFoundError:
+                    LOG.warn(_LW("Can not find policy directories %s"), path)
+                    continue
+                self._walk_through_policy_directory(path,
+                                                    self._load_policy_file,
+                                                    force_reload, False)
+
+    def _walk_through_policy_directory(self, path, func, *args):
+        # We do not iterate over sub-directories.
+        policy_files = next(os.walk(path))[2]
+        policy_files.sort()
+        for policy_file in [p for p in policy_files if not p.startswith('.')]:
+            func(os.path.join(path, policy_file), *args)
+
+    def _load_policy_file(self, path, force_reload, overwrite=True):
             reloaded, data = fileutils.read_cached_file(
-                self.policy_path, force_reload=force_reload)
+                path, force_reload=force_reload)
             if reloaded or not self.rules:
                 rules = Rules.load_json(data, self.default_rule)
-                self.set_rules(rules)
+                self.set_rules(rules, overwrite)
                 LOG.debug("Rules successfully reloaded")
 
-    def _get_policy_path(self):
-        """Locate the policy json data file.
+    def _get_policy_path(self, path):
+        """Locate the policy json data file/path.
 
-        :param policy_file: Custom policy file to locate.
+        :param path: It's value can be a full path or related path. When
+                     full path specified, this function just returns the full
+                     path. When related path specified, this function will
+                     search configuration directories to find one that exists.
 
         :returns: The policy path
 
-        :raises: ConfigFilesNotFoundError if the file couldn't
+        :raises: ConfigFilesNotFoundError if the file/path couldn't
                  be located.
         """
-        policy_file = CONF.find_file(self.policy_file)
+        policy_path = CONF.find_file(path)
 
-        if policy_file:
-            return policy_file
+        if policy_path:
+            return policy_path
 
-        raise cfg.ConfigFilesNotFoundError((self.policy_file,))
+        raise cfg.ConfigFilesNotFoundError((path,))
 
     def enforce(self, rule, target, creds, do_raise=False,
                 exc=None, *args, **kwargs):
@@ -784,7 +812,7 @@ def _parse_text_rule(rule):
         return state.result
     except ValueError:
         # Couldn't parse the rule
-        LOG.exception(_LE("Failed to understand rule %r") % rule)
+        LOG.exception(_LE("Failed to understand rule %s") % rule)
 
         # Fail closed
         return FalseCheck()
@@ -875,7 +903,6 @@ class GenericCheck(Check):
             'Member':%(role.name)s
         """
 
-        # TODO(termie): do dict inspection via dot syntax
         try:
             match = self.match % target
         except KeyError:
@@ -888,7 +915,10 @@ class GenericCheck(Check):
             leftval = ast.literal_eval(self.kind)
         except ValueError:
             try:
-                leftval = creds[self.kind]
+                kind_parts = self.kind.split('.')
+                leftval = creds
+                for kind_part in kind_parts:
+                    leftval = leftval[kind_part]
             except KeyError:
                 return False
         return match == six.text_type(leftval)

nova/openstack/common/processutils.py

@@ -18,7 +18,7 @@ System-level utilities and helper functions.
 """
 
 import errno
-import logging as stdlib_logging
+import logging
 import multiprocessing
 import os
 import random
@@ -27,11 +27,10 @@ import signal
 
 from eventlet.green import subprocess
 from eventlet import greenthread
+from oslo.utils import strutils
 import six
 
-from nova.openstack.common.gettextutils import _
-from nova.openstack.common import log as logging
-from nova.openstack.common import strutils
+from nova.openstack.common._i18n import _
 
 
 LOG = logging.getLogger(__name__)
@@ -116,8 +115,7 @@ def execute(*cmd, **kwargs):
                             execute this command. Defaults to false.
     :type shell:            boolean
     :param loglevel:        log level for execute commands.
-    :type loglevel:         int.  (Should be stdlib_logging.DEBUG or
-                            stdlib_logging.INFO)
+    :type loglevel:         int.  (Should be logging.DEBUG or logging.INFO)
     :returns:               (stdout, stderr) from process execution
     :raises:                :class:`UnknownArgumentError` on
                             receiving unknown arguments
@@ -133,7 +131,7 @@ def execute(*cmd, **kwargs):
     run_as_root = kwargs.pop('run_as_root', False)
     root_helper = kwargs.pop('root_helper', '')
     shell = kwargs.pop('shell', False)
-    loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG)
+    loglevel = kwargs.pop('loglevel', logging.DEBUG)
 
     if isinstance(check_exit_code, bool):
         ignore_exit_code = not check_exit_code
@@ -142,8 +140,7 @@ def execute(*cmd, **kwargs):
         check_exit_code = [check_exit_code]
 
     if kwargs:
-        raise UnknownArgumentError(_('Got unknown keyword args '
-                                     'to utils.execute: %r') % kwargs)
+        raise UnknownArgumentError(_('Got unknown keyword args: %r') % kwargs)
 
     if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0:
         if not root_helper:

nova/openstack/common/report/generators/conf.py

@@ -12,7 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack config generators
+"""Provides OpenStack config generators
 
 This module defines a class for configuration
 generators for generating the model in
@@ -21,17 +21,17 @@ generators for generating the model in
 
 from oslo.config import cfg
 
-import nova.openstack.common.report.models.conf as cm
+from nova.openstack.common.report.models import conf as cm
 
 
 class ConfigReportGenerator(object):
     """A Configuration Data Generator
 
     This generator returns
-    :class:`openstack.common.report.models.conf.ConfigModel` ,
+    :class:`openstack.common.report.models.conf.ConfigModel`,
     by default using the configuration options stored
     in :attr:`oslo.config.cfg.CONF`, which is where
-    Openstack stores everything.
+    OpenStack stores everything.
 
     :param cnf: the configuration option object
     :type cnf: :class:`oslo.config.cfg.ConfigOpts`

nova/openstack/common/report/generators/threading.py

@@ -23,10 +23,10 @@ import sys
 
 import greenlet
 
-import nova.openstack.common.report.models.threading as tm
+from nova.openstack.common.report.models import threading as tm
 from nova.openstack.common.report.models import with_default_views as mwdv
-import nova.openstack.common.report.utils as rutils
-import nova.openstack.common.report.views.text.generic as text_views
+from nova.openstack.common.report import utils as rutils
+from nova.openstack.common.report.views.text import generic as text_views
 
 
 class ThreadReportGenerator(object):

nova/openstack/common/report/generators/version.py

@@ -12,15 +12,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack version generators
+"""Provides OpenStack version generators
 
-This module defines a class for Openstack
+This module defines a class for OpenStack
 version and package information
 generators for generating the model in
 :mod:`openstack.common.report.models.version`.
 """
 
-import nova.openstack.common.report.models.version as vm
+from nova.openstack.common.report.models import version as vm
 
 
 class PackageReportGenerator(object):

nova/openstack/common/report/guru_meditation_report.py

@@ -51,9 +51,13 @@ where stderr is logged for that given service.
 
 from __future__ import print_function
 
+import inspect
+import os
 import signal
 import sys
 
+from oslo.utils import timeutils
+
 from nova.openstack.common.report.generators import conf as cgen
 from nova.openstack.common.report.generators import threading as tgen
 from nova.openstack.common.report.generators import version as pgen
@@ -74,6 +78,8 @@ class GuruMeditation(object):
     MRO is correct.
     """
 
+    timestamp_fmt = "%Y%m%d%H%M%S"
+
     def __init__(self, version_obj, *args, **kwargs):
         self.version_obj = version_obj
 
@@ -97,13 +103,17 @@ class GuruMeditation(object):
             cls.persistent_sections = [[section_title, generator]]
 
     @classmethod
-    def setup_autorun(cls, version, signum=None):
+    def setup_autorun(cls, version, service_name=None,
+                      log_dir=None, signum=None):
         """Set Up Auto-Run
 
         This method sets up the Guru Meditation Report to automatically
-        get dumped to stderr when the given signal is received.
+        get dumped to stderr or a file in a given dir when the given signal
+        is received.
 
         :param version: the version object for the current product
+        :param service_name: this program name used to construct logfile name
+        :param logdir: path to a log directory where to create a file
         :param signum: the signal to associate with running the report
         """
 
@@ -113,18 +123,25 @@ class GuruMeditation(object):
 
         if signum:
             signal.signal(signum,
-                          lambda *args: cls.handle_signal(version, *args))
+                          lambda *args: cls.handle_signal(
+                              version, service_name, log_dir, *args))
 
     @classmethod
-    def handle_signal(cls, version, *args):
+    def handle_signal(cls, version, service_name, log_dir, *args):
         """The Signal Handler
 
         This method (indirectly) handles receiving a registered signal and
-        dumping the Guru Meditation Report to stderr.  This method is designed
-        to be curried into a proper signal handler by currying out the version
+        dumping the Guru Meditation Report to stderr or a file in a given dir.
+        If service name and log dir are not None, the report will be dumped to
+        a file named $service_name_gurumeditation_$current_time in the log_dir
+        directory.
+        This method is designed to be curried into a proper signal handler by
+        currying out the version
         parameter.
 
         :param version: the version object for the current product
+        :param service_name: this program name used to construct logfile name
+        :param logdir: path to a log directory where to create a file
         """
 
         try:
@@ -133,7 +150,20 @@ class GuruMeditation(object):
             print("Unable to run Guru Meditation Report!",
                   file=sys.stderr)
         else:
-            print(res, file=sys.stderr)
+            if log_dir:
+                service_name = service_name or os.path.basename(
+                    inspect.stack()[-1][1])
+                filename = "%s_gurumeditation_%s" % (
+                    service_name, timeutils.strtime(fmt=cls.timestamp_fmt))
+                filepath = os.path.join(log_dir, filename)
+                try:
+                    with open(filepath, "w") as dumpfile:
+                        dumpfile.write(res)
+                except Exception:
+                    print("Unable to dump Guru Meditation Report to file %s" %
+                          (filepath,), file=sys.stderr)
+            else:
+                print(res, file=sys.stderr)
 
     def _readd_sections(self):
         del self.sections[self.start_section_index:]

nova/openstack/common/report/models/base.py

@@ -81,6 +81,11 @@ class ReportModel(col.MutableMapping):
         return self.data.__contains__(key)
 
     def __getattr__(self, attrname):
+        # Needed for deepcopy in Python3. That will avoid an infinite loop
+        # in __getattr__ .
+        if 'data' not in self.__dict__:
+            self.data = {}
+
         try:
             return self.data[attrname]
         except KeyError:

nova/openstack/common/report/models/conf.py

@@ -12,14 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack Configuration Model
+"""Provides OpenStack Configuration Model
 
 This module defines a class representing the data
 model for :mod:`oslo.config` configuration options
 """
 
-import nova.openstack.common.report.models.with_default_views as mwdv
-import nova.openstack.common.report.views.text.generic as generic_text_views
+from nova.openstack.common.report.models import with_default_views as mwdv
+from nova.openstack.common.report.views.text import generic as generic_text_views
 
 
 class ConfigModel(mwdv.ModelWithDefaultViews):

nova/openstack/common/report/models/threading.py

@@ -20,8 +20,8 @@ thread, and stack trace data models
 
 import traceback
 
-import nova.openstack.common.report.models.with_default_views as mwdv
-import nova.openstack.common.report.views.text.threading as text_views
+from nova.openstack.common.report.models import with_default_views as mwdv
+from nova.openstack.common.report.views.text import threading as text_views
 
 
 class StackTraceModel(mwdv.ModelWithDefaultViews):
@@ -42,12 +42,12 @@ class StackTraceModel(mwdv.ModelWithDefaultViews):
                 {'filename': fn, 'line': ln, 'name': nm, 'code': cd}
                 for fn, ln, nm, cd in traceback.extract_stack(stack_state)
             ]
-
-            if stack_state.f_exc_type is not None:
+            # FIXME(flepied): under Python3 f_exc_type doesn't exist
+            # anymore so we lose information about exceptions
+            if getattr(stack_state, 'f_exc_type', None) is not None:
                 self['root_exception'] = {
                     'type': stack_state.f_exc_type,
-                    'value': stack_state.f_exc_value
-                }
+                    'value': stack_state.f_exc_value}
             else:
                 self['root_exception'] = None
         else:

nova/openstack/common/report/models/version.py

@@ -12,14 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Provides Openstack Version Info Model
+"""Provides OpenStack Version Info Model
 
 This module defines a class representing the data
-model for Openstack package and version information
+model for OpenStack package and version information
 """
 
-import nova.openstack.common.report.models.with_default_views as mwdv
-import nova.openstack.common.report.views.text.generic as generic_text_views
+from nova.openstack.common.report.models import with_default_views as mwdv
+from nova.openstack.common.report.views.text import generic as generic_text_views
 
 
 class PackageModel(mwdv.ModelWithDefaultViews):

nova/openstack/common/report/models/with_default_views.py

@@ -14,10 +14,10 @@
 
 import copy
 
-import nova.openstack.common.report.models.base as base_model
-import nova.openstack.common.report.views.json.generic as jsonviews
-import nova.openstack.common.report.views.text.generic as textviews
-import nova.openstack.common.report.views.xml.generic as xmlviews
+from nova.openstack.common.report.models import base as base_model
+from nova.openstack.common.report.views.json import generic as jsonviews
+from nova.openstack.common.report.views.text import generic as textviews
+from nova.openstack.common.report.views.xml import generic as xmlviews
 
 
 class ModelWithDefaultViews(base_model.ReportModel):
@@ -28,18 +28,18 @@ class ModelWithDefaultViews(base_model.ReportModel):
     when a submodel should have an attached view, but the view
     differs depending on the serialization format
 
-    Paramaters are as the superclass, with the exception
-    of any parameters ending in '_view': these parameters
+    Parameters are as the superclass, except for any
+    parameters ending in '_view': these parameters
     get stored as default views.
 
     The default 'default views' are
 
     text
-        :class:`openstack.common.views.text.generic.KeyValueView`
+        :class:`openstack.common.report.views.text.generic.KeyValueView`
     xml
-        :class:`openstack.common.views.xml.generic.KeyValueView`
+        :class:`openstack.common.report.views.xml.generic.KeyValueView`
     json
-        :class:`openstack.common.views.json.generic.KeyValueView`
+        :class:`openstack.common.report.views.json.generic.KeyValueView`
 
     .. function:: to_type()
 

nova/openstack/common/report/report.py

@@ -14,12 +14,12 @@
 
 """Provides Report classes
 
-This module defines various classes representing
-reports and report sections.  All reports take the
-form of a report class containing various report sections.
+This module defines various classes representing reports and report sections.
+All reports take the form of a report class containing various report
+sections.
 """
 
-import nova.openstack.common.report.views.text.header as header_views
+from nova.openstack.common.report.views.text import header as header_views
 
 
 class BasicReport(object):
@@ -28,7 +28,7 @@ class BasicReport(object):
     A Basic Report consists of a collection of :class:`ReportSection`
     objects, each of which contains a top-level model and generator.
     It collects these sections into a cohesive report which may then
-    be serialized by calling :func:`run`
+    be serialized by calling :func:`run`.
     ""&#