461 lines
17 KiB
Python
461 lines
17 KiB
Python
# Copyright 2010 United States Government as represented by the
|
|
# Administrator of the National Aeronautics and Space Administration.
|
|
# All Rights Reserved.
|
|
# Copyright 2013 Red Hat, Inc.
|
|
# Copyright 2013 New Dream Network, LLC (DreamHost)
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
__all__ = [
|
|
'ExecutorLoadFailure',
|
|
'MessageHandlingServer',
|
|
'MessagingServerError',
|
|
'ServerListenError',
|
|
]
|
|
|
|
import abc
|
|
import functools
|
|
import inspect
|
|
import logging
|
|
import threading
|
|
import traceback
|
|
|
|
from oslo_config import cfg
|
|
from oslo_service import service
|
|
from oslo_utils import eventletutils
|
|
from oslo_utils import timeutils
|
|
import six
|
|
from stevedore import driver
|
|
|
|
from oslo_messaging._drivers import base as driver_base
|
|
from oslo_messaging._i18n import _LW, _LI
|
|
from oslo_messaging import exceptions
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# The default number of seconds of waiting after which we will emit a log
|
|
# message
|
|
DEFAULT_LOG_AFTER = 30
|
|
|
|
|
|
_pool_opts = [
|
|
cfg.IntOpt('executor_thread_pool_size',
|
|
default=64,
|
|
deprecated_name="rpc_thread_pool_size",
|
|
help='Size of executor thread pool.'),
|
|
]
|
|
|
|
|
|
class MessagingServerError(exceptions.MessagingException):
|
|
"""Base class for all MessageHandlingServer exceptions."""
|
|
|
|
|
|
class ExecutorLoadFailure(MessagingServerError):
|
|
"""Raised if an executor can't be loaded."""
|
|
|
|
def __init__(self, executor, ex):
|
|
msg = 'Failed to load executor "%s": %s' % (executor, ex)
|
|
super(ExecutorLoadFailure, self).__init__(msg)
|
|
self.executor = executor
|
|
self.ex = ex
|
|
|
|
|
|
class ServerListenError(MessagingServerError):
|
|
"""Raised if we failed to listen on a target."""
|
|
|
|
def __init__(self, target, ex):
|
|
msg = 'Failed to listen on target "%s": %s' % (target, ex)
|
|
super(ServerListenError, self).__init__(msg)
|
|
self.target = target
|
|
self.ex = ex
|
|
|
|
|
|
class TaskTimeout(MessagingServerError):
|
|
"""Raised if we timed out waiting for a task to complete."""
|
|
|
|
|
|
class _OrderedTask(object):
|
|
"""A task which must be executed in a particular order.
|
|
|
|
A caller may wait for this task to complete by calling
|
|
`wait_for_completion`.
|
|
|
|
A caller may run this task with `run_once`, which will ensure that however
|
|
many times the task is called it only runs once. Simultaneous callers will
|
|
block until the running task completes, which means that any caller can be
|
|
sure that the task has completed after run_once returns.
|
|
"""
|
|
|
|
INIT = 0 # The task has not yet started
|
|
RUNNING = 1 # The task is running somewhere
|
|
COMPLETE = 2 # The task has run somewhere
|
|
|
|
def __init__(self, name):
|
|
"""Create a new _OrderedTask.
|
|
|
|
:param name: The name of this task. Used in log messages.
|
|
"""
|
|
super(_OrderedTask, self).__init__()
|
|
|
|
self._name = name
|
|
self._cond = threading.Condition()
|
|
self._state = self.INIT
|
|
|
|
def _wait(self, condition, msg, log_after, timeout_timer):
|
|
"""Wait while condition() is true. Write a log message if condition()
|
|
has not become false within `log_after` seconds. Raise TaskTimeout if
|
|
timeout_timer expires while waiting.
|
|
"""
|
|
|
|
log_timer = None
|
|
if log_after != 0:
|
|
log_timer = timeutils.StopWatch(duration=log_after)
|
|
log_timer.start()
|
|
|
|
while condition():
|
|
if log_timer is not None and log_timer.expired():
|
|
LOG.warning(_LW('Possible hang: %s'), msg)
|
|
LOG.debug(''.join(traceback.format_stack()))
|
|
# Only log once. After than we wait indefinitely without
|
|
# logging.
|
|
log_timer = None
|
|
|
|
if timeout_timer is not None and timeout_timer.expired():
|
|
raise TaskTimeout(msg)
|
|
|
|
timeouts = []
|
|
if log_timer is not None:
|
|
timeouts.append(log_timer.leftover())
|
|
if timeout_timer is not None:
|
|
timeouts.append(timeout_timer.leftover())
|
|
|
|
wait = None
|
|
if timeouts:
|
|
wait = min(timeouts)
|
|
self._cond.wait(wait)
|
|
|
|
@property
|
|
def complete(self):
|
|
return self._state == self.COMPLETE
|
|
|
|
def wait_for_completion(self, caller, log_after, timeout_timer):
|
|
"""Wait until this task has completed.
|
|
|
|
:param caller: The name of the task which is waiting.
|
|
:param log_after: Emit a log message if waiting longer than `log_after`
|
|
seconds.
|
|
:param timeout_timer: Raise TaskTimeout if StopWatch object
|
|
`timeout_timer` expires while waiting.
|
|
"""
|
|
with self._cond:
|
|
msg = '%s is waiting for %s to complete' % (caller, self._name)
|
|
self._wait(lambda: not self.complete,
|
|
msg, log_after, timeout_timer)
|
|
|
|
def run_once(self, fn, log_after, timeout_timer):
|
|
"""Run a task exactly once. If it is currently running in another
|
|
thread, wait for it to complete. If it has already run, return
|
|
immediately without running it again.
|
|
|
|
:param fn: The task to run. It must be a callable taking no arguments.
|
|
It may optionally return another callable, which also takes
|
|
no arguments, which will be executed after completion has
|
|
been signaled to other threads.
|
|
:param log_after: Emit a log message if waiting longer than `log_after`
|
|
seconds.
|
|
:param timeout_timer: Raise TaskTimeout if StopWatch object
|
|
`timeout_timer` expires while waiting.
|
|
"""
|
|
with self._cond:
|
|
if self._state == self.INIT:
|
|
self._state = self.RUNNING
|
|
# Note that nothing waits on RUNNING, so no need to notify
|
|
|
|
# We need to release the condition lock before calling out to
|
|
# prevent deadlocks. Reacquire it immediately afterwards.
|
|
self._cond.release()
|
|
try:
|
|
post_fn = fn()
|
|
finally:
|
|
self._cond.acquire()
|
|
self._state = self.COMPLETE
|
|
self._cond.notify_all()
|
|
|
|
if post_fn is not None:
|
|
# Release the condition lock before calling out to prevent
|
|
# deadlocks. Reacquire it immediately afterwards.
|
|
self._cond.release()
|
|
try:
|
|
post_fn()
|
|
finally:
|
|
self._cond.acquire()
|
|
elif self._state == self.RUNNING:
|
|
msg = ('%s is waiting for another thread to complete'
|
|
% self._name)
|
|
self._wait(lambda: self._state == self.RUNNING,
|
|
msg, log_after, timeout_timer)
|
|
|
|
|
|
class _OrderedTaskRunner(object):
|
|
"""Mixin for a class which executes ordered tasks."""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(_OrderedTaskRunner, self).__init__(*args, **kwargs)
|
|
|
|
# Get a list of methods on this object which have the _ordered
|
|
# attribute
|
|
self._tasks = [name
|
|
for (name, member) in inspect.getmembers(self)
|
|
if inspect.ismethod(member) and
|
|
getattr(member, '_ordered', False)]
|
|
self.reset_states()
|
|
|
|
self._reset_lock = threading.Lock()
|
|
|
|
def reset_states(self):
|
|
# Create new task states for tasks in reset
|
|
self._states = {task: _OrderedTask(task) for task in self._tasks}
|
|
|
|
@staticmethod
|
|
def decorate_ordered(fn, state, after, reset_after):
|
|
|
|
@functools.wraps(fn)
|
|
def wrapper(self, *args, **kwargs):
|
|
# If the reset_after state has already completed, reset state so
|
|
# we can run again.
|
|
# NOTE(mdbooth): This is ugly and requires external locking to be
|
|
# deterministic when using multiple threads. Consider a thread that
|
|
# does: server.stop(), server.wait(). If another thread causes a
|
|
# reset between stop() and wait(), this will not have the intended
|
|
# behaviour. It is safe without external locking, if the caller
|
|
# instantiates a new object.
|
|
with self._reset_lock:
|
|
if (reset_after is not None and
|
|
self._states[reset_after].complete):
|
|
self.reset_states()
|
|
|
|
# Store the states we started with in case the state wraps on us
|
|
# while we're sleeping. We must wait and run_once in the same
|
|
# epoch. If the epoch ended while we were sleeping, run_once will
|
|
# safely do nothing.
|
|
states = self._states
|
|
|
|
log_after = kwargs.pop('log_after', DEFAULT_LOG_AFTER)
|
|
timeout = kwargs.pop('timeout', None)
|
|
|
|
timeout_timer = None
|
|
if timeout is not None:
|
|
timeout_timer = timeutils.StopWatch(duration=timeout)
|
|
timeout_timer.start()
|
|
|
|
# Wait for the given preceding state to complete
|
|
if after is not None:
|
|
states[after].wait_for_completion(state,
|
|
log_after, timeout_timer)
|
|
|
|
# Run this state
|
|
states[state].run_once(lambda: fn(self, *args, **kwargs),
|
|
log_after, timeout_timer)
|
|
return wrapper
|
|
|
|
|
|
def ordered(after=None, reset_after=None):
|
|
"""A method which will be executed as an ordered task. The method will be
|
|
called exactly once, however many times it is called. If it is called
|
|
multiple times simultaneously it will only be called once, but all callers
|
|
will wait until execution is complete.
|
|
|
|
If `after` is given, this method will not run until `after` has completed.
|
|
|
|
If `reset_after` is given and the target method has completed, allow this
|
|
task to run again by resetting all task states.
|
|
|
|
:param after: Optionally, the name of another `ordered` method. Wait for
|
|
the completion of `after` before executing this method.
|
|
:param reset_after: Optionally, the name of another `ordered` method. Reset
|
|
all states when calling this method if `reset_after`
|
|
has completed.
|
|
"""
|
|
def _ordered(fn):
|
|
# Set an attribute on the method so we can find it later
|
|
setattr(fn, '_ordered', True)
|
|
state = fn.__name__
|
|
|
|
return _OrderedTaskRunner.decorate_ordered(fn, state, after,
|
|
reset_after)
|
|
return _ordered
|
|
|
|
|
|
@six.add_metaclass(abc.ABCMeta)
|
|
class MessageHandlingServer(service.ServiceBase, _OrderedTaskRunner):
|
|
"""Server for handling messages.
|
|
|
|
Connect a transport to a dispatcher that knows how to process the
|
|
message using an executor that knows how the app wants to create
|
|
new tasks.
|
|
"""
|
|
|
|
def __init__(self, transport, dispatcher, executor='blocking'):
|
|
"""Construct a message handling server.
|
|
|
|
The dispatcher parameter is a DispatcherBase instance which is used
|
|
for routing request to endpoint for processing.
|
|
|
|
The executor parameter controls how incoming messages will be received
|
|
and dispatched. By default, the most simple executor is used - the
|
|
blocking executor. It handles only one message at once. It's
|
|
recommended to use threading or eventlet.
|
|
|
|
:param transport: the messaging transport
|
|
:type transport: Transport
|
|
:param dispatcher: has a dispatch() method which is invoked for each
|
|
incoming request
|
|
:type dispatcher: DispatcherBase
|
|
:param executor: name of message executor - for example
|
|
'eventlet', 'blocking'
|
|
:type executor: str
|
|
"""
|
|
self.conf = transport.conf
|
|
self.conf.register_opts(_pool_opts)
|
|
|
|
self.transport = transport
|
|
self.dispatcher = dispatcher
|
|
self.executor_type = executor
|
|
if self.executor_type == 'blocking':
|
|
# NOTE(sileht): We keep blocking as default to not enforce the
|
|
# application to use threading or eventlet. Because application
|
|
# have to be preprepared accordingly for each one (monkeypatching,
|
|
# threadsafe, ...)
|
|
LOG.info(_LI("blocking executor handles only one message at "
|
|
"once. threading or eventlet executor is "
|
|
"recommended."))
|
|
|
|
self.listener = None
|
|
|
|
try:
|
|
mgr = driver.DriverManager('oslo.messaging.executors',
|
|
self.executor_type)
|
|
except RuntimeError as ex:
|
|
raise ExecutorLoadFailure(self.executor_type, ex)
|
|
|
|
self._executor_cls = mgr.driver
|
|
|
|
self._work_executor = None
|
|
|
|
self._started = False
|
|
|
|
super(MessageHandlingServer, self).__init__()
|
|
|
|
def _on_incoming(self, incoming):
|
|
"""Handles on_incoming event
|
|
|
|
:param incoming: incoming request.
|
|
"""
|
|
self._work_executor.submit(self._process_incoming, incoming)
|
|
|
|
@abc.abstractmethod
|
|
def _process_incoming(self, incoming):
|
|
"""Perform processing incoming request
|
|
|
|
:param incoming: incoming request.
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def _create_listener(self):
|
|
"""Creates listener object for polling requests
|
|
:return: MessageListenerAdapter
|
|
"""
|
|
|
|
@ordered(reset_after='stop')
|
|
def start(self, override_pool_size=None):
|
|
"""Start handling incoming messages.
|
|
|
|
This method causes the server to begin polling the transport for
|
|
incoming messages and passing them to the dispatcher. Message
|
|
processing will continue until the stop() method is called.
|
|
|
|
The executor controls how the server integrates with the applications
|
|
I/O handling strategy - it may choose to poll for messages in a new
|
|
process, thread or co-operatively scheduled coroutine or simply by
|
|
registering a callback with an event loop. Similarly, the executor may
|
|
choose to dispatch messages in a new thread, coroutine or simply the
|
|
current thread.
|
|
"""
|
|
# Warn that restarting will be deprecated
|
|
if self._started:
|
|
LOG.warning(_LW('Restarting a MessageHandlingServer is inherently '
|
|
'racy. It is deprecated, and will become a noop '
|
|
'in a future release of oslo.messaging. If you '
|
|
'need to restart MessageHandlingServer you should '
|
|
'instantiate a new object.'))
|
|
self._started = True
|
|
|
|
executor_opts = {}
|
|
|
|
if self.executor_type == "threading":
|
|
executor_opts["max_workers"] = (
|
|
override_pool_size or self.conf.executor_thread_pool_size
|
|
)
|
|
elif self.executor_type == "eventlet":
|
|
eventletutils.warn_eventlet_not_patched(
|
|
expected_patched_modules=['thread'],
|
|
what="the 'oslo.messaging eventlet executor'")
|
|
executor_opts["max_workers"] = (
|
|
override_pool_size or self.conf.executor_thread_pool_size
|
|
)
|
|
|
|
self._work_executor = self._executor_cls(**executor_opts)
|
|
|
|
try:
|
|
self.listener = self._create_listener()
|
|
except driver_base.TransportDriverError as ex:
|
|
raise ServerListenError(self.target, ex)
|
|
|
|
self.listener.start(self._on_incoming)
|
|
|
|
@ordered(after='start')
|
|
def stop(self):
|
|
"""Stop handling incoming messages.
|
|
|
|
Once this method returns, no new incoming messages will be handled by
|
|
the server. However, the server may still be in the process of handling
|
|
some messages, and underlying driver resources associated to this
|
|
server are still in use. See 'wait' for more details.
|
|
"""
|
|
self.listener.stop()
|
|
self._started = False
|
|
|
|
@ordered(after='stop')
|
|
def wait(self):
|
|
"""Wait for message processing to complete.
|
|
|
|
After calling stop(), there may still be some existing messages
|
|
which have not been completely processed. The wait() method blocks
|
|
until all message processing has completed.
|
|
|
|
Once it's finished, the underlying driver resources associated to this
|
|
server are released (like closing useless network connections).
|
|
"""
|
|
self._work_executor.shutdown(wait=True)
|
|
|
|
# Close listener connection after processing all messages
|
|
self.listener.cleanup()
|
|
|
|
def reset(self):
|
|
"""Reset service.
|
|
|
|
Called in case service running in daemon mode receives SIGHUP.
|
|
"""
|
|
# TODO(sergey.vilgelm): implement this method
|
|
pass
|