From d6daf9b98c72595e7143608936bdb953b2c66180 Mon Sep 17 00:00:00 2001 From: gordon chung Date: Fri, 30 Jan 2015 15:25:17 -0500 Subject: [PATCH] sync oslo and bring in versionutils this patch synchronises oslo and brings in verisonutils so we can deprecated swift middleware Change-Id: I876700aac75fe5f9122e266d3c9840bd1c1691ba --- ceilometer/openstack/common/_i18n.py | 2 +- .../openstack/common/eventlet_backdoor.py | 3 +- ceilometer/openstack/common/fileutils.py | 9 +- ceilometer/openstack/common/loopingcall.py | 10 +- ceilometer/openstack/common/policy.py | 65 +++-- ceilometer/openstack/common/service.py | 13 +- ceilometer/openstack/common/systemd.py | 3 +- ceilometer/openstack/common/threadgroup.py | 2 +- ceilometer/openstack/common/versionutils.py | 253 ++++++++++++++++++ openstack-common.conf | 1 + 10 files changed, 314 insertions(+), 47 deletions(-) create mode 100644 ceilometer/openstack/common/versionutils.py diff --git a/ceilometer/openstack/common/_i18n.py b/ceilometer/openstack/common/_i18n.py index 96b405a1..cec0e465 100644 --- a/ceilometer/openstack/common/_i18n.py +++ b/ceilometer/openstack/common/_i18n.py @@ -40,6 +40,6 @@ try: _LC = _translators.log_critical except ImportError: # NOTE(dims): Support for cases where a project wants to use - # code from ceilometer-incubator, but is not ready to be internationalized + # code from oslo-incubator, but is not ready to be internationalized # (like tempest) _ = _LI = _LW = _LE = _LC = lambda x: x diff --git a/ceilometer/openstack/common/eventlet_backdoor.py b/ceilometer/openstack/common/eventlet_backdoor.py index b6e6dd9e..d6f574a3 100644 --- a/ceilometer/openstack/common/eventlet_backdoor.py +++ b/ceilometer/openstack/common/eventlet_backdoor.py @@ -19,19 +19,18 @@ from __future__ import print_function import copy import errno import gc +import logging import os import pprint import socket import sys import traceback -import eventlet import eventlet.backdoor import greenlet from oslo.config import cfg from ceilometer.openstack.common._i18n import _LI -from ceilometer.openstack.common import log as logging help_for_backdoor_port = ( "Acceptable values are 0, , and :, where 0 results " diff --git a/ceilometer/openstack/common/fileutils.py b/ceilometer/openstack/common/fileutils.py index 64cee1ce..6cacf22b 100644 --- a/ceilometer/openstack/common/fileutils.py +++ b/ceilometer/openstack/common/fileutils.py @@ -17,22 +17,25 @@ import contextlib import errno import logging import os +import stat import tempfile -from oslo_utils import excutils +from oslo.utils import excutils LOG = logging.getLogger(__name__) _FILE_CACHE = {} +DEFAULT_MODE = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO -def ensure_tree(path): +def ensure_tree(path, mode=DEFAULT_MODE): """Create a directory (and any ancestor directories required) :param path: Directory to create + :param mode: Directory creation permissions """ try: - os.makedirs(path) + os.makedirs(path, mode) except OSError as exc: if exc.errno == errno.EEXIST: if not os.path.isdir(path): diff --git a/ceilometer/openstack/common/loopingcall.py b/ceilometer/openstack/common/loopingcall.py index 7d6ad452..a12564a2 100644 --- a/ceilometer/openstack/common/loopingcall.py +++ b/ceilometer/openstack/common/loopingcall.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import logging import sys import time @@ -22,7 +23,6 @@ from eventlet import event from eventlet import greenthread from ceilometer.openstack.common._i18n import _LE, _LW -from ceilometer.openstack.common import log as logging LOG = logging.getLogger(__name__) @@ -84,9 +84,9 @@ class FixedIntervalLoopingCall(LoopingCallBase): break delay = end - start - interval if delay > 0: - LOG.warn(_LW('task %(func_name)s run outlasted ' + LOG.warn(_LW('task %(func_name)r run outlasted ' 'interval by %(delay).2f sec'), - {'func_name': repr(self.f), 'delay': delay}) + {'func_name': self.f, 'delay': delay}) greenthread.sleep(-delay if delay < 0 else 0) except LoopingCallDone as e: self.stop() @@ -127,9 +127,9 @@ class DynamicLoopingCall(LoopingCallBase): if periodic_interval_max is not None: idle = min(idle, periodic_interval_max) - LOG.debug('Dynamic looping call %(func_name)s sleeping ' + LOG.debug('Dynamic looping call %(func_name)r sleeping ' 'for %(idle).02f seconds', - {'func_name': repr(self.f), 'idle': idle}) + {'func_name': self.f, 'idle': idle}) greenthread.sleep(idle) except LoopingCallDone as e: self.stop() diff --git a/ceilometer/openstack/common/policy.py b/ceilometer/openstack/common/policy.py index f31131d2..35c5a7e9 100644 --- a/ceilometer/openstack/common/policy.py +++ b/ceilometer/openstack/common/policy.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +# # Copyright (c) 2012 OpenStack Foundation. # All Rights Reserved. # @@ -22,22 +24,43 @@ string written in the new policy language. In the list-of-lists representation, each check inside the innermost list is combined as with an "and" conjunction--for that check to pass, all the specified checks must pass. These innermost lists are then -combined as with an "or" conjunction. This is the original way of -expressing policies, but there now exists a new way: the policy -language. - -In the policy language, each check is specified the same way as in the -list-of-lists representation: a simple "a:b" pair that is matched to -the correct code to perform that check. However, conjunction -operators are available, allowing for more expressiveness in crafting -policies. - -As an example, take the following rule, expressed in the list-of-lists -representation:: +combined as with an "or" conjunction. As an example, take the following +rule, expressed in the list-of-lists representation:: [["role:admin"], ["project_id:%(project_id)s", "role:projectadmin"]] -In the policy language, this becomes:: +This is the original way of expressing policies, but there now exists a +new way: the policy language. + +In the policy language, each check is specified the same way as in the +list-of-lists representation: a simple "a:b" pair that is matched to +the correct class to perform that check:: + + +===========================================================================+ + | TYPE | SYNTAX | + +===========================================================================+ + |User's Role | role:admin | + +---------------------------------------------------------------------------+ + |Rules already defined on policy | rule:admin_required | + +---------------------------------------------------------------------------+ + |Against URL's¹ | http://my-url.org/check | + +---------------------------------------------------------------------------+ + |User attributes² | project_id:%(target.project.id)s | + +---------------------------------------------------------------------------+ + |Strings | :'xpto2035abc' | + | | 'myproject': | + +---------------------------------------------------------------------------+ + | | project_id:xpto2035abc | + |Literals | domain_id:20 | + | | True:%(user.enabled)s | + +===========================================================================+ + +¹URL checking must return 'True' to be valid +²User attributes (obtained through the token): user_id, domain_id or project_id + +Conjunction operators are available, allowing for more expressiveness +in crafting policies. So, in the policy language, the previous check in +list-of-lists becomes:: role:admin or (project_id:%(project_id)s and role:projectadmin) @@ -46,26 +69,16 @@ policy rule:: project_id:%(project_id)s and not role:dunce -It is possible to perform policy checks on the following user -attributes (obtained through the token): user_id, domain_id or -project_id:: - - domain_id: - Attributes sent along with API calls can be used by the policy engine (on the right side of the expression), by using the following syntax:: - :user.id + :%(user.id)s Contextual attributes of objects identified by their IDs are loaded from the database. They are also available to the policy engine and can be checked through the `target` keyword:: - :target.role.name - -All these attributes (related to users, API calls, and context) can be -checked against each other or against constants, be it literals (True, -) or strings. + :%(target.role.name)s Finally, two special policy checks should be mentioned; the policy check "@" will always accept an access, and the policy check "!" will @@ -78,6 +91,7 @@ as it allows particular rules to be explicitly disabled. import abc import ast import copy +import logging import os import re @@ -89,7 +103,6 @@ import six.moves.urllib.request as urlrequest from ceilometer.openstack.common import fileutils from ceilometer.openstack.common._i18n import _, _LE, _LI -from ceilometer.openstack.common import log as logging policy_opts = [ diff --git a/ceilometer/openstack/common/service.py b/ceilometer/openstack/common/service.py index 542670ac..abdab60f 100644 --- a/ceilometer/openstack/common/service.py +++ b/ceilometer/openstack/common/service.py @@ -18,7 +18,7 @@ """Generic Node base class for all workers that run on hosts.""" import errno -import logging as std_logging +import logging import os import random import signal @@ -39,7 +39,6 @@ from oslo.config import cfg from ceilometer.openstack.common import eventlet_backdoor from ceilometer.openstack.common._i18n import _LE, _LI, _LW -from ceilometer.openstack.common import log as logging from ceilometer.openstack.common import systemd from ceilometer.openstack.common import threadgroup @@ -163,7 +162,7 @@ class ServiceLauncher(Launcher): signo = 0 LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) + CONF.log_opt_values(LOG, logging.DEBUG) try: if ready_callback: @@ -377,7 +376,7 @@ class ProcessLauncher(object): systemd.notify_once() LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) + CONF.log_opt_values(LOG, logging.DEBUG) try: while True: @@ -397,7 +396,7 @@ class ProcessLauncher(object): self.running = True self.sigcaught = None except eventlet.greenlet.GreenletExit: - LOG.info(_LI("Wait called after thread killed. Cleaning up.")) + LOG.info(_LI("Wait called after thread killed. Cleaning up.")) self.stop() @@ -434,8 +433,8 @@ class Service(object): def start(self): pass - def stop(self): - self.tg.stop() + def stop(self, graceful=False): + self.tg.stop(graceful) self.tg.wait() # Signal that service cleanup is done: if not self._done.ready(): diff --git a/ceilometer/openstack/common/systemd.py b/ceilometer/openstack/common/systemd.py index a4677117..36243b34 100644 --- a/ceilometer/openstack/common/systemd.py +++ b/ceilometer/openstack/common/systemd.py @@ -16,12 +16,11 @@ Helper module for systemd service readiness notification. """ +import logging import os import socket import sys -from ceilometer.openstack.common import log as logging - LOG = logging.getLogger(__name__) diff --git a/ceilometer/openstack/common/threadgroup.py b/ceilometer/openstack/common/threadgroup.py index 11235809..52c83fd5 100644 --- a/ceilometer/openstack/common/threadgroup.py +++ b/ceilometer/openstack/common/threadgroup.py @@ -11,12 +11,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import logging import threading import eventlet from eventlet import greenpool -from ceilometer.openstack.common import log as logging from ceilometer.openstack.common import loopingcall diff --git a/ceilometer/openstack/common/versionutils.py b/ceilometer/openstack/common/versionutils.py new file mode 100644 index 00000000..1fb637c3 --- /dev/null +++ b/ceilometer/openstack/common/versionutils.py @@ -0,0 +1,253 @@ +# Copyright (c) 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helpers for comparing version strings. +""" + +import functools +import inspect +import logging + +from oslo.config import cfg +import pkg_resources +import six + +from ceilometer.openstack.common._i18n import _ + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +opts = [ + cfg.BoolOpt('fatal_deprecations', + default=False, + help='Enables or disables fatal status of deprecations.'), +] + + +class deprecated(object): + """A decorator to mark callables as deprecated. + + This decorator logs a deprecation message when the callable it decorates is + used. The message will include the release where the callable was + deprecated, the release where it may be removed and possibly an optional + replacement. + + Examples: + + 1. Specifying the required deprecated release + + >>> @deprecated(as_of=deprecated.ICEHOUSE) + ... def a(): pass + + 2. Specifying a replacement: + + >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()') + ... def b(): pass + + 3. Specifying the release where the functionality may be removed: + + >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=+1) + ... def c(): pass + + 4. Specifying the deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, remove_in=0) + ... def d(): pass + + 5. Specifying a replacement, deprecated functionality will not be removed: + >>> @deprecated(as_of=deprecated.ICEHOUSE, in_favor_of='f()', remove_in=0) + ... def e(): pass + + """ + + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + BEXAR = 'B' + FOLSOM = 'F' + GRIZZLY = 'G' + HAVANA = 'H' + ICEHOUSE = 'I' + JUNO = 'J' + KILO = 'K' + + _RELEASES = { + # NOTE(morganfainberg): Bexar is used for unit test purposes, it is + # expected we maintain a gap between Bexar and Folsom in this list. + 'B': 'Bexar', + 'F': 'Folsom', + 'G': 'Grizzly', + 'H': 'Havana', + 'I': 'Icehouse', + 'J': 'Juno', + 'K': 'Kilo', + } + + _deprecated_msg_with_alternative = _( + '%(what)s is deprecated as of %(as_of)s in favor of ' + '%(in_favor_of)s and may be removed in %(remove_in)s.') + + _deprecated_msg_no_alternative = _( + '%(what)s is deprecated as of %(as_of)s and may be ' + 'removed in %(remove_in)s. It will not be superseded.') + + _deprecated_msg_with_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s in favor of %(in_favor_of)s.') + + _deprecated_msg_with_no_alternative_no_removal = _( + '%(what)s is deprecated as of %(as_of)s. It will not be superseded.') + + def __init__(self, as_of, in_favor_of=None, remove_in=2, what=None): + """Initialize decorator + + :param as_of: the release deprecating the callable. Constants + are define in this class for convenience. + :param in_favor_of: the replacement for the callable (optional) + :param remove_in: an integer specifying how many releases to wait + before removing (default: 2) + :param what: name of the thing being deprecated (default: the + callable's name) + + """ + self.as_of = as_of + self.in_favor_of = in_favor_of + self.remove_in = remove_in + self.what = what + + def __call__(self, func_or_cls): + if not self.what: + self.what = func_or_cls.__name__ + '()' + msg, details = self._build_message() + + if inspect.isfunction(func_or_cls): + + @six.wraps(func_or_cls) + def wrapped(*args, **kwargs): + report_deprecated_feature(LOG, msg, details) + return func_or_cls(*args, **kwargs) + return wrapped + elif inspect.isclass(func_or_cls): + orig_init = func_or_cls.__init__ + + # TODO(tsufiev): change `functools` module to `six` as + # soon as six 1.7.4 (with fix for passing `assigned` + # argument to underlying `functools.wraps`) is released + # and added to the oslo-incubator requrements + @functools.wraps(orig_init, assigned=('__name__', '__doc__')) + def new_init(self, *args, **kwargs): + report_deprecated_feature(LOG, msg, details) + orig_init(self, *args, **kwargs) + func_or_cls.__init__ = new_init + return func_or_cls + else: + raise TypeError('deprecated can be used only with functions or ' + 'classes') + + def _get_safe_to_remove_release(self, release): + # TODO(dstanek): this method will have to be reimplemented once + # when we get to the X release because once we get to the Y + # release, what is Y+2? + new_release = chr(ord(release) + self.remove_in) + if new_release in self._RELEASES: + return self._RELEASES[new_release] + else: + return new_release + + def _build_message(self): + details = dict(what=self.what, + as_of=self._RELEASES[self.as_of], + remove_in=self._get_safe_to_remove_release(self.as_of)) + + if self.in_favor_of: + details['in_favor_of'] = self.in_favor_of + if self.remove_in > 0: + msg = self._deprecated_msg_with_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_alternative_no_removal + else: + if self.remove_in > 0: + msg = self._deprecated_msg_no_alternative + else: + # There are no plans to remove this function, but it is + # now deprecated. + msg = self._deprecated_msg_with_no_alternative_no_removal + return msg, details + + +def is_compatible(requested_version, current_version, same_major=True): + """Determine whether `requested_version` is satisfied by + `current_version`; in other words, `current_version` is >= + `requested_version`. + + :param requested_version: version to check for compatibility + :param current_version: version to check against + :param same_major: if True, the major version must be identical between + `requested_version` and `current_version`. This is used when a + major-version difference indicates incompatibility between the two + versions. Since this is the common-case in practice, the default is + True. + :returns: True if compatible, False if not + """ + requested_parts = pkg_resources.parse_version(requested_version) + current_parts = pkg_resources.parse_version(current_version) + + if same_major and (requested_parts[0] != current_parts[0]): + return False + + return current_parts >= requested_parts + + +# Track the messages we have sent already. See +# report_deprecated_feature(). +_deprecated_messages_sent = {} + + +def report_deprecated_feature(logger, msg, *args, **kwargs): + """Call this function when a deprecated feature is used. + + If the system is configured for fatal deprecations then the message + is logged at the 'critical' level and :class:`DeprecatedConfig` will + be raised. + + Otherwise, the message will be logged (once) at the 'warn' level. + + :raises: :class:`DeprecatedConfig` if the system is configured for + fatal deprecations. + """ + stdmsg = _("Deprecated: %s") % msg + CONF.register_opts(opts) + if CONF.fatal_deprecations: + logger.critical(stdmsg, *args, **kwargs) + raise DeprecatedConfig(msg=stdmsg) + + # Using a list because a tuple with dict can't be stored in a set. + sent_args = _deprecated_messages_sent.setdefault(msg, list()) + + if args in sent_args: + # Already logged this message, so don't log it again. + return + + sent_args.append(args) + logger.warn(stdmsg, *args, **kwargs) + + +class DeprecatedConfig(Exception): + message = _("Fatal call to deprecated config: %(msg)s") + + def __init__(self, msg): + super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/openstack-common.conf b/openstack-common.conf index 40b7f86d..686b570f 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -4,6 +4,7 @@ module=log module=policy module=service +module=versionutils # The base module to hold the copy of openstack.common base=ceilometer