sync oslo and bring in versionutils

this patch synchronises oslo and brings in verisonutils so we can
deprecated swift middleware

Change-Id: I876700aac75fe5f9122e266d3c9840bd1c1691ba

ceilometer/openstack/common/_i18n.py

@@ -40,6 +40,6 @@ try:
     _LC = _translators.log_critical
 except ImportError:
     # NOTE(dims): Support for cases where a project wants to use
-    # code from ceilometer-incubator, but is not ready to be internationalized
+    # code from oslo-incubator, but is not ready to be internationalized
     # (like tempest)
     _ = _LI = _LW = _LE = _LC = lambda x: x

ceilometer/openstack/common/eventlet_backdoor.py

@@ -19,19 +19,18 @@ from __future__ import print_function
 import copy
 import errno
 import gc
+import logging
 import os
 import pprint
 import socket
 import sys
 import traceback
 
-import eventlet
 import eventlet.backdoor
 import greenlet
 from oslo.config import cfg
 
 from ceilometer.openstack.common._i18n import _LI
-from ceilometer.openstack.common import log as logging
 
 help_for_backdoor_port = (
     "Acceptable values are 0, <port>, and <start>:<end>, where 0 results "

ceilometer/openstack/common/fileutils.py

@@ -17,22 +17,25 @@ import contextlib
 import errno
 import logging
 import os
+import stat
 import tempfile
 
-from oslo_utils import excutils
+from oslo.utils import excutils
 
 LOG = logging.getLogger(__name__)
 
 _FILE_CACHE = {}
+DEFAULT_MODE = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
 
 
-def ensure_tree(path):
+def ensure_tree(path, mode=DEFAULT_MODE):
     """Create a directory (and any ancestor directories required)
 
     :param path: Directory to create
+    :param mode: Directory creation permissions
     """
     try:
-        os.makedirs(path)
+        os.makedirs(path, mode)
     except OSError as exc:
         if exc.errno == errno.EEXIST:
             if not os.path.isdir(path):

ceilometer/openstack/common/loopingcall.py

@@ -15,6 +15,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import sys
 import time
 
@@ -22,7 +23,6 @@ from eventlet import event
 from eventlet import greenthread
 
 from ceilometer.openstack.common._i18n import _LE, _LW
-from ceilometer.openstack.common import log as logging
 
 LOG = logging.getLogger(__name__)
 
@@ -84,9 +84,9 @@ class FixedIntervalLoopingCall(LoopingCallBase):
                         break
                     delay = end - start - interval
                     if delay > 0:
-                        LOG.warn(_LW('task %(func_name)s run outlasted '
+                        LOG.warn(_LW('task %(func_name)r run outlasted '
                                      'interval by %(delay).2f sec'),
-                                 {'func_name': repr(self.f), 'delay': delay})
+                                 {'func_name': self.f, 'delay': delay})
                     greenthread.sleep(-delay if delay < 0 else 0)
             except LoopingCallDone as e:
                 self.stop()
@@ -127,9 +127,9 @@ class DynamicLoopingCall(LoopingCallBase):
 
                     if periodic_interval_max is not None:
                         idle = min(idle, periodic_interval_max)
-                    LOG.debug('Dynamic looping call %(func_name)s sleeping '
+                    LOG.debug('Dynamic looping call %(func_name)r sleeping '
                               'for %(idle).02f seconds',
-                              {'func_name': repr(self.f), 'idle': idle})
+                              {'func_name': self.f, 'idle': idle})
                     greenthread.sleep(idle)
             except LoopingCallDone as e:
                 self.stop()

ceilometer/openstack/common/policy.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+#
 # Copyright (c) 2012 OpenStack Foundation.
 # All Rights Reserved.
 #
@@ -22,22 +24,43 @@ string written in the new policy language.
 In the list-of-lists representation, each check inside the innermost
 list is combined as with an "and" conjunction--for that check to pass,
 all the specified checks must pass.  These innermost lists are then
-combined as with an "or" conjunction.  This is the original way of
+combined as with an "or" conjunction. As an example, take the following
-expressing policies, but there now exists a new way: the policy
+rule, expressed in the list-of-lists representation::
-language.
-
-In the policy language, each check is specified the same way as in the
-list-of-lists representation: a simple "a:b" pair that is matched to
-the correct code to perform that check.  However, conjunction
-operators are available, allowing for more expressiveness in crafting
-policies.
-
-As an example, take the following rule, expressed in the list-of-lists
-representation::
 
     [["role:admin"], ["project_id:%(project_id)s", "role:projectadmin"]]
 
-In the policy language, this becomes::
+This is the original way of expressing policies, but there now exists a
+new way: the policy language.
+
+In the policy language, each check is specified the same way as in the
+list-of-lists representation: a simple "a:b" pair that is matched to
+the correct class to perform that check::
+
+ +===========================================================================+
+ |            TYPE                |                SYNTAX                    |
+ +===========================================================================+
+ |User's Role                     |              role:admin                  |
+ +---------------------------------------------------------------------------+
+ |Rules already defined on policy |          rule:admin_required             |
+ +---------------------------------------------------------------------------+
+ |Against URL's¹                  |         http://my-url.org/check          |
+ +---------------------------------------------------------------------------+
+ |User attributes²                |    project_id:%(target.project.id)s      |
+ +---------------------------------------------------------------------------+
+ |Strings                         |        <variable>:'xpto2035abc'          |
+ |                                |         'myproject':<variable>           |
+ +---------------------------------------------------------------------------+
+ |                                |         project_id:xpto2035abc           |
+ |Literals                        |         domain_id:20                     |
+ |                                |         True:%(user.enabled)s            |
+ +===========================================================================+
+
+¹URL checking must return 'True' to be valid
+²User attributes (obtained through the token): user_id, domain_id or project_id
+
+Conjunction operators are available, allowing for more expressiveness
+in crafting policies. So, in the policy language, the previous check in
+list-of-lists becomes::
 
     role:admin or (project_id:%(project_id)s and role:projectadmin)
 
@@ -46,26 +69,16 @@ policy rule::
 
     project_id:%(project_id)s and not role:dunce
 
-It is possible to perform policy checks on the following user
-attributes (obtained through the token): user_id, domain_id or
-project_id::
-
-    domain_id:<some_value>
-
 Attributes sent along with API calls can be used by the policy engine
 (on the right side of the expression), by using the following syntax::
 
-    <some_value>:user.id
+    <some_value>:%(user.id)s
 
 Contextual attributes of objects identified by their IDs are loaded
 from the database. They are also available to the policy engine and
 can be checked through the `target` keyword::
 
-    <some_value>:target.role.name
+    <some_value>:%(target.role.name)s
-
-All these attributes (related to users, API calls, and context) can be
-checked against each other or against constants, be it literals (True,
-<a_number>) or strings.
 
 Finally, two special policy checks should be mentioned; the policy
 check "@" will always accept an access, and the policy check "!" will
@@ -78,6 +91,7 @@ as it allows particular rules to be explicitly disabled.
 import abc
 import ast
 import copy
+import logging
 import os
 import re
 
@@ -89,7 +103,6 @@ import six.moves.urllib.request as urlrequest
 
 from ceilometer.openstack.common import fileutils
 from ceilometer.openstack.common._i18n import _, _LE, _LI
-from ceilometer.openstack.common import log as logging
 
 
 policy_opts = [

ceilometer/openstack/common/service.py

@@ -18,7 +18,7 @@
 """Generic Node base class for all workers that run on hosts."""
 
 import errno
-import logging as std_logging
+import logging
 import os
 import random
 import signal
@@ -39,7 +39,6 @@ from oslo.config import cfg
 
 from ceilometer.openstack.common import eventlet_backdoor
 from ceilometer.openstack.common._i18n import _LE, _LI, _LW
-from ceilometer.openstack.common import log as logging
 from ceilometer.openstack.common import systemd
 from ceilometer.openstack.common import threadgroup
 
@@ -163,7 +162,7 @@ class ServiceLauncher(Launcher):
         signo = 0
 
         LOG.debug('Full set of CONF:')
-        CONF.log_opt_values(LOG, std_logging.DEBUG)
+        CONF.log_opt_values(LOG, logging.DEBUG)
 
         try:
             if ready_callback:
@@ -377,7 +376,7 @@ class ProcessLauncher(object):
 
         systemd.notify_once()
         LOG.debug('Full set of CONF:')
-        CONF.log_opt_values(LOG, std_logging.DEBUG)
+        CONF.log_opt_values(LOG, logging.DEBUG)
 
         try:
             while True:
@@ -434,8 +433,8 @@ class Service(object):
     def start(self):
         pass
 
-    def stop(self):
+    def stop(self, graceful=False):
-        self.tg.stop()
+        self.tg.stop(graceful)
         self.tg.wait()
         # Signal that service cleanup is done:
         if not self._done.ready():

ceilometer/openstack/common/systemd.py

@@ -16,12 +16,11 @@
 Helper module for systemd service readiness notification.
 """
 
+import logging
 import os
 import socket
 import sys
 
-from ceilometer.openstack.common import log as logging
-
 
 LOG = logging.getLogger(__name__)
 

ceilometer/openstack/common/threadgroup.py

@@ -11,12 +11,12 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import logging
 import threading
 
 import eventlet
 from eventlet import greenpool
 
-from ceilometer.openstack.common import log as logging
 from ceilometer.openstack.common import loopingcall
 
 

ceilometer/openstack/common/versionutils.py

@@ -0,0 +1,253 @@
+# Copyright (c) 2013 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Helpers for comparing version strings.
+"""
+
+import functools
+import inspect
+import logging
+
+from oslo.config import cfg
+import pkg_resources
+import six
+
+from ceilometer.openstack.common._i18n import _
+
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+opts = [
+    cfg.BoolOpt('fatal_deprecations',
+                default=False,
+                help='Enables or disables fatal status of deprecations.'),
+]
+
+
+class deprecated(object):
+    """A decorator to mark callables as deprecated.
+
+    This decorator logs a deprecation message when the callable it decorates is
+    used. The message will include the release where the callable was
+    deprecated, the release where it may be removed and possibly an optional
+    replacement.
+
+    Examples:
+
+    1. Specifying the required deprecated release
+
+    >>> @deprecated(as_of=deprecated.ICEHOUSE)
+    ... def a(): pass
+
+    2. Specifying a replacement:
+
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()')
+    ... def b(): pass
+
+    3. Specifying the release where the functionality may be removed:
+
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1)
+    ... def c(): pass
+
+    4. Specifying the deprecated functionality will not be removed:
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0)
+    ... def d(): pass
+
+    5. Specifying a replacement, deprecated functionality will not be removed:
+    >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0)
+    ... def e(): pass
+
+    """
+
+    # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+    # expected we maintain a gap between Bexar and Folsom in this list.
+    BEXAR = 'B'
+    FOLSOM = 'F'
+    GRIZZLY = 'G'
+    HAVANA = 'H'
+    ICEHOUSE = 'I'
+    JUNO = 'J'
+    KILO = 'K'
+
+    _RELEASES = {
+        # NOTE(morganfainberg): Bexar is used for unit test purposes, it is
+        # expected we maintain a gap between Bexar and Folsom in this list.
+        'B': 'Bexar',
+        'F': 'Folsom',
+        'G': 'Grizzly',
+        'H': 'Havana',
+        'I': 'Icehouse',
+        'J': 'Juno',
+        'K': 'Kilo',
+    }
+
+    _deprecated_msg_with_alternative = _(
+        '%(what)s is deprecated as of %(as_of)s in favor of '
+        '%(in_favor_of)s and may be removed in %(remove_in)s.')
+
+    _deprecated_msg_no_alternative = _(
+        '%(what)s is deprecated as of %(as_of)s and may be '
+        'removed in %(remove_in)s. It will not be superseded.')
+
+    _deprecated_msg_with_alternative_no_removal = _(
+        '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.')
+
+    _deprecated_msg_with_no_alternative_no_removal = _(
+        '%(what)s is deprecated as of %(as_of)s. It will not be superseded.')
+
+    def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None):
+        """Initialize decorator
+
+        :param as_of: the release deprecating the callable. Constants
+            are define in this class for convenience.
+        :param in_favor_of: the replacement for the callable (optional)
+        :param remove_in: an integer specifying how many releases to wait
+            before removing (default: 2)
+        :param what: name of the thing being deprecated (default: the
+            callable's name)
+
+        """
+        self.as_of = as_of
+        self.in_favor_of = in_favor_of
+        self.remove_in = remove_in
+        self.what = what
+
+    def __call__(self, func_or_cls):
+        if not self.what:
+            self.what = func_or_cls.__name__ + '()'
+        msg, details = self._build_message()
+
+        if inspect.isfunction(func_or_cls):
+
+            @six.wraps(func_or_cls)
+            def wrapped(*args, **kwargs):
+                report_deprecated_feature(LOG, msg, details)
+                return func_or_cls(*args, **kwargs)
+            return wrapped
+        elif inspect.isclass(func_or_cls):
+            orig_init = func_or_cls.__init__
+
+            # TODO(tsufiev): change `functools` module to `six` as
+            # soon as six 1.7.4 (with fix for passing `assigned`
+            # argument to underlying `functools.wraps`) is released
+            # and added to the oslo-incubator requrements
+            @functools.wraps(orig_init, assigned=('__name__', '__doc__'))
+            def new_init(self, *args, **kwargs):
+                report_deprecated_feature(LOG, msg, details)
+                orig_init(self, *args, **kwargs)
+            func_or_cls.__init__ = new_init
+            return func_or_cls
+        else:
+            raise TypeError('deprecated can be used only with functions or '
+                            'classes')
+
+    def _get_safe_to_remove_release(self, release):
+        # TODO(dstanek): this method will have to be reimplemented once
+        #    when we get to the X release because once we get to the Y
+        #    release, what is Y+2?
+        new_release = chr(ord(release) + self.remove_in)
+        if new_release in self._RELEASES:
+            return self._RELEASES[new_release]
+        else:
+            return new_release
+
+    def _build_message(self):
+        details = dict(what=self.what,
+                       as_of=self._RELEASES[self.as_of],
+                       remove_in=self._get_safe_to_remove_release(self.as_of))
+
+        if self.in_favor_of:
+            details['in_favor_of'] = self.in_favor_of
+            if self.remove_in > 0:
+                msg = self._deprecated_msg_with_alternative
+            else:
+                # There are no plans to remove this function, but it is
+                # now deprecated.
+                msg = self._deprecated_msg_with_alternative_no_removal
+        else:
+            if self.remove_in > 0:
+                msg = self._deprecated_msg_no_alternative
+            else:
+                # There are no plans to remove this function, but it is
+                # now deprecated.
+                msg = self._deprecated_msg_with_no_alternative_no_removal
+        return msg, details
+
+
+def is_compatible(requested_version, current_version, same_major=True):
+    """Determine whether `requested_version` is satisfied by
+    `current_version`; in other words, `current_version` is >=
+    `requested_version`.
+
+    :param requested_version: version to check for compatibility
+    :param current_version: version to check against
+    :param same_major: if True, the major version must be identical between
+        `requested_version` and `current_version`. This is used when a
+        major-version difference indicates incompatibility between the two
+        versions. Since this is the common-case in practice, the default is
+        True.
+    :returns: True if compatible, False if not
+    """
+    requested_parts = pkg_resources.parse_version(requested_version)
+    current_parts = pkg_resources.parse_version(current_version)
+
+    if same_major and (requested_parts[0] != current_parts[0]):
+        return False
+
+    return current_parts >= requested_parts
+
+
+# Track the messages we have sent already. See
+# report_deprecated_feature().
+_deprecated_messages_sent = {}
+
+
+def report_deprecated_feature(logger, msg, *args, **kwargs):
+    """Call this function when a deprecated feature is used.
+
+    If the system is configured for fatal deprecations then the message
+    is logged at the 'critical' level and :class:`DeprecatedConfig` will
+    be raised.
+
+    Otherwise, the message will be logged (once) at the 'warn' level.
+
+    :raises: :class:`DeprecatedConfig` if the system is configured for
+             fatal deprecations.
+    """
+    stdmsg = _("Deprecated: %s") % msg
+    CONF.register_opts(opts)
+    if CONF.fatal_deprecations:
+        logger.critical(stdmsg, *args, **kwargs)
+        raise DeprecatedConfig(msg=stdmsg)
+
+    # Using a list because a tuple with dict can't be stored in a set.
+    sent_args = _deprecated_messages_sent.setdefault(msg, list())
+
+    if args in sent_args:
+        # Already logged this message, so don't log it again.
+        return
+
+    sent_args.append(args)
+    logger.warn(stdmsg, *args, **kwargs)
+
+
+class DeprecatedConfig(Exception):
+    message = _("Fatal call to deprecated config: %(msg)s")
+
+    def __init__(self, msg):
+        super(Exception, self).__init__(self.message % dict(msg=msg))

openstack-common.conf

@@ -4,6 +4,7 @@
 module=log
 module=policy
 module=service
+module=versionutils
 
 # The base module to hold the copy of openstack.common
 base=ceilometer