From 0789d7005603ea89249cf4ce9f638d2794db2bd0 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Fri, 20 Mar 2015 16:07:13 -0400 Subject: [PATCH] Sync with latest oslo-incubator Changes are as follows: f29e865 3/17/15 Store ProcessLauncher signal handlers on class level bf92010 2/19/15 Optimization of waiting subprocesses in ProcessLauncher 2aacb11 1/29/15 Change oslo.config to oslo_config 2fbf506 1/29/15 Remove oslo.log code and clean up versionutils API 0cc741a 1/26/15 switch to oslo_i18n fbd77a7 1/21/15 Allow overriding name for periodic tasks 9896e0e 1/21/15 Separate add_periodic_task from the metaclass __init__ 80c4751 1/5/15 Remove unnecessary import of eventlet 442fc22 12/30/14 Added graceful argument on Service.stop method cf429e5 12/28/14 Remove extra white space in log message 5985b35 12/11/14 Prefer delayed %r formatting over explicit repr use Closes-bug: #1434744 Change-Id: Icf0b52170b3457d570fd87fd2875a1e759b97579 --- magnum/openstack/common/_i18n.py | 6 +- magnum/openstack/common/eventlet_backdoor.py | 5 +- magnum/openstack/common/loopingcall.py | 10 +-- magnum/openstack/common/periodic_task.py | 72 +++++++++++++------- magnum/openstack/common/service.py | 39 ++++++----- magnum/openstack/common/systemd.py | 3 +- magnum/openstack/common/threadgroup.py | 2 +- magnum/openstack/common/versionutils.py | 67 ++++++++++++++++-- openstack-common.conf | 1 - 9 files changed, 140 insertions(+), 65 deletions(-) diff --git a/magnum/openstack/common/_i18n.py b/magnum/openstack/common/_i18n.py index ecb0161cc2..0b0a247721 100644 --- a/magnum/openstack/common/_i18n.py +++ b/magnum/openstack/common/_i18n.py @@ -17,14 +17,14 @@ See http://docs.openstack.org/developer/oslo.i18n/usage.html """ try: - import oslo.i18n + import oslo_i18n # NOTE(dhellmann): This reference to o-s-l-o will be replaced by the # application name when this module is synced into the separate # repository. It is OK to have more than one translation function # using the same domain, since there will still only be one message # catalog. - _translators = oslo.i18n.TranslatorFactory(domain='magnum') + _translators = oslo_i18n.TranslatorFactory(domain='magnum') # The primary translation function using the well-known name "_" _ = _translators.primary @@ -40,6 +40,6 @@ try: _LC = _translators.log_critical except ImportError: # NOTE(dims): Support for cases where a project wants to use - # code from magnum-incubator, but is not ready to be internationalized + # code from oslo-incubator, but is not ready to be internationalized # (like tempest) _ = _LI = _LW = _LE = _LC = lambda x: x diff --git a/magnum/openstack/common/eventlet_backdoor.py b/magnum/openstack/common/eventlet_backdoor.py index 033e442025..1ab08a0600 100644 --- a/magnum/openstack/common/eventlet_backdoor.py +++ b/magnum/openstack/common/eventlet_backdoor.py @@ -19,19 +19,18 @@ from __future__ import print_function import copy import errno import gc +import logging import os import pprint import socket import sys import traceback -import eventlet import eventlet.backdoor import greenlet from oslo_config import cfg from magnum.openstack.common._i18n import _LI -from magnum.openstack.common import log as logging help_for_backdoor_port = ( "Acceptable values are 0, , and :, where 0 results " @@ -51,7 +50,7 @@ LOG = logging.getLogger(__name__) def list_opts(): - """Entry point for oslo.config-generator. + """Entry point for oslo-config-generator. """ return [(None, copy.deepcopy(eventlet_backdoor_opts))] diff --git a/magnum/openstack/common/loopingcall.py b/magnum/openstack/common/loopingcall.py index 5b9594c8c1..d85e5dd24a 100644 --- a/magnum/openstack/common/loopingcall.py +++ b/magnum/openstack/common/loopingcall.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import logging import sys import time @@ -22,7 +23,6 @@ from eventlet import event from eventlet import greenthread from magnum.openstack.common._i18n import _LE, _LW -from magnum.openstack.common import log as logging LOG = logging.getLogger(__name__) @@ -84,9 +84,9 @@ class FixedIntervalLoopingCall(LoopingCallBase): break delay = end - start - interval if delay > 0: - LOG.warn(_LW('task %(func_name)s run outlasted ' + LOG.warn(_LW('task %(func_name)r run outlasted ' 'interval by %(delay).2f sec'), - {'func_name': repr(self.f), 'delay': delay}) + {'func_name': self.f, 'delay': delay}) greenthread.sleep(-delay if delay < 0 else 0) except LoopingCallDone as e: self.stop() @@ -127,9 +127,9 @@ class DynamicLoopingCall(LoopingCallBase): if periodic_interval_max is not None: idle = min(idle, periodic_interval_max) - LOG.debug('Dynamic looping call %(func_name)s sleeping ' + LOG.debug('Dynamic looping call %(func_name)r sleeping ' 'for %(idle).02f seconds', - {'func_name': repr(self.f), 'idle': idle}) + {'func_name': self.f, 'idle': idle}) greenthread.sleep(idle) except LoopingCallDone as e: self.stop() diff --git a/magnum/openstack/common/periodic_task.py b/magnum/openstack/common/periodic_task.py index cec3482fcc..d9857e581a 100644 --- a/magnum/openstack/common/periodic_task.py +++ b/magnum/openstack/common/periodic_task.py @@ -12,6 +12,7 @@ # under the License. import copy +import logging import random import time @@ -19,7 +20,6 @@ from oslo_config import cfg import six from magnum.openstack.common._i18n import _, _LE, _LI -from magnum.openstack.common import log as logging periodic_opts = [ @@ -38,7 +38,7 @@ DEFAULT_INTERVAL = 60.0 def list_opts(): - """Entry point for oslo.config-generator.""" + """Entry point for oslo-config-generator.""" return [(None, copy.deepcopy(periodic_opts))] @@ -55,14 +55,15 @@ def periodic_task(*args, **kwargs): interval of 60 seconds. 2. With arguments: - @periodic_task(spacing=N [, run_immediately=[True|False]]) + @periodic_task(spacing=N [, run_immediately=[True|False]] + [, name=[None|"string"]) this will be run on approximately every N seconds. If this number is negative the periodic task will be disabled. If the run_immediately argument is provided and has a value of 'True', the first run of the task will be shortly after task scheduler starts. If run_immediately is omitted or set to 'False', the first time the task runs will be approximately N seconds after the task scheduler - starts. + starts. If name is not provided, __name__ of function is used. """ def decorator(f): # Test for old style invocation @@ -76,6 +77,7 @@ def periodic_task(*args, **kwargs): f._periodic_enabled = False else: f._periodic_enabled = kwargs.pop('enabled', True) + f._periodic_name = kwargs.pop('name', f.__name__) # Control frequency f._periodic_spacing = kwargs.pop('spacing', 0) @@ -105,6 +107,36 @@ def periodic_task(*args, **kwargs): class _PeriodicTasksMeta(type): + def _add_periodic_task(cls, task): + """Add a periodic task to the list of periodic tasks. + + The task should already be decorated by @periodic_task. + + :return: whether task was actually enabled + """ + name = task._periodic_name + + if task._periodic_spacing < 0: + LOG.info(_LI('Skipping periodic task %(task)s because ' + 'its interval is negative'), + {'task': name}) + return False + if not task._periodic_enabled: + LOG.info(_LI('Skipping periodic task %(task)s because ' + 'it is disabled'), + {'task': name}) + return False + + # A periodic spacing of zero indicates that this task should + # be run on the default interval to avoid running too + # frequently. + if task._periodic_spacing == 0: + task._periodic_spacing = DEFAULT_INTERVAL + + cls._periodic_tasks.append((name, task)) + cls._periodic_spacing[name] = task._periodic_spacing + return True + def __init__(cls, names, bases, dict_): """Metaclass that allows us to collect decorated periodic tasks.""" super(_PeriodicTasksMeta, cls).__init__(names, bases, dict_) @@ -125,28 +157,7 @@ class _PeriodicTasksMeta(type): for value in cls.__dict__.values(): if getattr(value, '_periodic_task', False): - task = value - name = task.__name__ - - if task._periodic_spacing < 0: - LOG.info(_LI('Skipping periodic task %(task)s because ' - 'its interval is negative'), - {'task': name}) - continue - if not task._periodic_enabled: - LOG.info(_LI('Skipping periodic task %(task)s because ' - 'it is disabled'), - {'task': name}) - continue - - # A periodic spacing of zero indicates that this task should - # be run on the default interval to avoid running too - # frequently. - if task._periodic_spacing == 0: - task._periodic_spacing = DEFAULT_INTERVAL - - cls._periodic_tasks.append((name, task)) - cls._periodic_spacing[name] = task._periodic_spacing + cls._add_periodic_task(value) def _nearest_boundary(last_run, spacing): @@ -178,6 +189,15 @@ class PeriodicTasks(object): for name, task in self._periodic_tasks: self._periodic_last_run[name] = task._periodic_last_run + def add_periodic_task(self, task): + """Add a periodic task to the list of periodic tasks. + + The task should already be decorated by @periodic_task. + """ + if self.__class__._add_periodic_task(task): + self._periodic_last_run[task._periodic_name] = ( + task._periodic_last_run) + def run_periodic_tasks(self, context, raise_on_error=False): """Tasks to be run at a periodic interval.""" idle_for = DEFAULT_INTERVAL diff --git a/magnum/openstack/common/service.py b/magnum/openstack/common/service.py index 2d0a71c80b..4105fcac92 100644 --- a/magnum/openstack/common/service.py +++ b/magnum/openstack/common/service.py @@ -18,7 +18,7 @@ """Generic Node base class for all workers that run on hosts.""" import errno -import logging as std_logging +import logging import os import random import signal @@ -39,7 +39,6 @@ from oslo_config import cfg from magnum.openstack.common import eventlet_backdoor from magnum.openstack.common._i18n import _LE, _LI, _LW -from magnum.openstack.common import log as logging from magnum.openstack.common import systemd from magnum.openstack.common import threadgroup @@ -163,7 +162,7 @@ class ServiceLauncher(Launcher): signo = 0 LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) + CONF.log_opt_values(LOG, logging.DEBUG) try: if ready_callback: @@ -200,22 +199,26 @@ class ServiceWrapper(object): class ProcessLauncher(object): - def __init__(self, wait_interval=0.01): - """Constructor. + _signal_handlers_set = set() + + @classmethod + def _handle_class_signals(cls, *args, **kwargs): + for handler in cls._signal_handlers_set: + handler(*args, **kwargs) + + def __init__(self): + """Constructor.""" - :param wait_interval: The interval to sleep for between checks - of child process exit. - """ self.children = {} self.sigcaught = None self.running = True - self.wait_interval = wait_interval rfd, self.writepipe = os.pipe() self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') self.handle_signal() def handle_signal(self): - _set_signals_handler(self._handle_signal) + self._signal_handlers_set.add(self._handle_signal) + _set_signals_handler(self._handle_class_signals) def _handle_signal(self, signo, frame): self.sigcaught = signo @@ -334,8 +337,8 @@ class ProcessLauncher(object): def _wait_child(self): try: - # Don't block if no child processes have exited - pid, status = os.waitpid(0, os.WNOHANG) + # Block while any of child processes have exited + pid, status = os.waitpid(0, 0) if not pid: return None except OSError as exc: @@ -364,10 +367,6 @@ class ProcessLauncher(object): while self.running: wrap = self._wait_child() if not wrap: - # Yield to other threads if no children have exited - # Sleep for a short time to avoid excessive CPU usage - # (see bug #1095346) - eventlet.greenthread.sleep(self.wait_interval) continue while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap) @@ -377,7 +376,7 @@ class ProcessLauncher(object): systemd.notify_once() LOG.debug('Full set of CONF:') - CONF.log_opt_values(LOG, std_logging.DEBUG) + CONF.log_opt_values(LOG, logging.DEBUG) try: while True: @@ -397,7 +396,7 @@ class ProcessLauncher(object): self.running = True self.sigcaught = None except eventlet.greenlet.GreenletExit: - LOG.info(_LI("Wait called after thread killed. Cleaning up.")) + LOG.info(_LI("Wait called after thread killed. Cleaning up.")) self.stop() @@ -434,8 +433,8 @@ class Service(object): def start(self): pass - def stop(self): - self.tg.stop() + def stop(self, graceful=False): + self.tg.stop(graceful) self.tg.wait() # Signal that service cleanup is done: if not self._done.ready(): diff --git a/magnum/openstack/common/systemd.py b/magnum/openstack/common/systemd.py index 24c12a6039..36243b342a 100644 --- a/magnum/openstack/common/systemd.py +++ b/magnum/openstack/common/systemd.py @@ -16,12 +16,11 @@ Helper module for systemd service readiness notification. """ +import logging import os import socket import sys -from magnum.openstack.common import log as logging - LOG = logging.getLogger(__name__) diff --git a/magnum/openstack/common/threadgroup.py b/magnum/openstack/common/threadgroup.py index 0b3a0e3fb1..f061f8b7b2 100644 --- a/magnum/openstack/common/threadgroup.py +++ b/magnum/openstack/common/threadgroup.py @@ -11,12 +11,12 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import logging import threading import eventlet from eventlet import greenpool -from magnum.openstack.common import log as logging from magnum.openstack.common import loopingcall diff --git a/magnum/openstack/common/versionutils.py b/magnum/openstack/common/versionutils.py index 67f8699c93..067dce4ea8 100644 --- a/magnum/openstack/common/versionutils.py +++ b/magnum/openstack/common/versionutils.py @@ -17,17 +17,33 @@ Helpers for comparing version strings. """ +import copy import functools import inspect +import logging +from oslo_config import cfg import pkg_resources import six from magnum.openstack.common._i18n import _ -from magnum.openstack.common import log as logging LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +deprecated_opts = [ + cfg.BoolOpt('fatal_deprecations', + default=False, + help='Enables or disables fatal status of deprecations.'), +] + + +def list_opts(): + """Entry point for oslo.config-generator. + """ + return [(None, copy.deepcopy(deprecated_opts))] class deprecated(object): @@ -74,6 +90,7 @@ class deprecated(object): ICEHOUSE = 'I' JUNO = 'J' KILO = 'K' + LIBERTY = 'L' _RELEASES = { # NOTE(morganfainberg): Bexar is used for unit test purposes, it is @@ -85,6 +102,7 @@ class deprecated(object): 'I': 'Icehouse', 'J': 'Juno', 'K': 'Kilo', + 'L': 'Liberty', } _deprecated_msg_with_alternative = _( @@ -127,7 +145,7 @@ class deprecated(object): @six.wraps(func_or_cls) def wrapped(*args, **kwargs): - LOG.deprecated(msg, details) + report_deprecated_feature(LOG, msg, details) return func_or_cls(*args, **kwargs) return wrapped elif inspect.isclass(func_or_cls): @@ -136,10 +154,10 @@ class deprecated(object): # TODO(tsufiev): change `functools` module to `six` as # soon as six 1.7.4 (with fix for passing `assigned` # argument to underlying `functools.wraps`) is released - # and added to the magnum-incubator requrements + # and added to the oslo-incubator requrements @functools.wraps(orig_init, assigned=('__name__', '__doc__')) def new_init(self, *args, **kwargs): - LOG.deprecated(msg, details) + report_deprecated_feature(LOG, msg, details) orig_init(self, *args, **kwargs) func_or_cls.__init__ = new_init return func_or_cls @@ -201,3 +219,44 @@ def is_compatible(requested_version, current_version, same_major=True): return False return current_parts >= requested_parts + + +# Track the messages we have sent already. See +# report_deprecated_feature(). +_deprecated_messages_sent = {} + + +def report_deprecated_feature(logger, msg, *args, **kwargs): + """Call this function when a deprecated feature is used. + + If the system is configured for fatal deprecations then the message + is logged at the 'critical' level and :class:`DeprecatedConfig` will + be raised. + + Otherwise, the message will be logged (once) at the 'warn' level. + + :raises: :class:`DeprecatedConfig` if the system is configured for + fatal deprecations. + """ + stdmsg = _("Deprecated: %s") % msg + CONF.register_opts(deprecated_opts) + if CONF.fatal_deprecations: + logger.critical(stdmsg, *args, **kwargs) + raise DeprecatedConfig(msg=stdmsg) + + # Using a list because a tuple with dict can't be stored in a set. + sent_args = _deprecated_messages_sent.setdefault(msg, list()) + + if args in sent_args: + # Already logged this message, so don't log it again. + return + + sent_args.append(args) + logger.warn(stdmsg, *args, **kwargs) + + +class DeprecatedConfig(Exception): + message = _("Fatal call to deprecated config: %(msg)s") + + def __init__(self, msg): + super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/openstack-common.conf b/openstack-common.conf index 781cd5e646..3430a68eb6 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -2,7 +2,6 @@ # The list of modules to copy from oslo-incubator.git module=eventlet_backdoor -module=log module=loopingcall module=periodic_task module=service