162f6e987b
We can reduce a workload of rabbitmq through implementation of expiration mechanism for idle connections in the pool with next properties: conn_pool_ttl (default 20 min) conn_pool_min_size: the pool size limit for expire() (default 2) The problem is timeless idle connections in the pool, which can be created via some single huge workload of RPCServer. One SEND connection is heartbeat thread + some network activity every n second. So, we can reduce it. Here is two ways to implement an expiration: [1] Create a separated thread for checking expire date of connections [2] Make call expire() on pool.get() or pool.put() The [1] has some threading overhead, but probably insignificant because the thread can sleep 99% time and wake up every 20 mins (by default). Anyway current implementation is [2]. Change-Id: Ie8781d10549a044656824ceb78b2fe2e4f7f8b43
275 lines
9.1 KiB
Python
275 lines
9.1 KiB
Python
|
|
# Copyright 2013 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import abc
|
|
import threading
|
|
|
|
from oslo_config import cfg
|
|
from oslo_utils import excutils
|
|
from oslo_utils import timeutils
|
|
import six
|
|
|
|
|
|
from oslo_messaging import exceptions
|
|
|
|
base_opts = [
|
|
cfg.IntOpt('rpc_conn_pool_size', default=30,
|
|
deprecated_group='DEFAULT',
|
|
help='Size of RPC connection pool.'),
|
|
cfg.IntOpt('conn_pool_min_size', default=2,
|
|
help='The pool size limit for connections expiration policy'),
|
|
cfg.IntOpt('conn_pool_ttl', default=1200,
|
|
help='The time-to-live in sec of idle connections in the pool')
|
|
]
|
|
|
|
|
|
def batch_poll_helper(func):
|
|
"""Decorator to poll messages in batch
|
|
|
|
This decorator helps driver that polls message one by one,
|
|
to returns a list of message.
|
|
"""
|
|
def wrapper(in_self, timeout=None, batch_size=1, batch_timeout=None):
|
|
incomings = []
|
|
driver_prefetch = in_self.prefetch_size
|
|
if driver_prefetch > 0:
|
|
batch_size = min(batch_size, driver_prefetch)
|
|
|
|
with timeutils.StopWatch(timeout) as timeout_watch:
|
|
# poll first message
|
|
msg = func(in_self, timeout=timeout_watch.leftover(True))
|
|
if msg is not None:
|
|
incomings.append(msg)
|
|
if batch_size == 1 or msg is None:
|
|
return incomings
|
|
|
|
# update batch_timeout according to timeout for whole operation
|
|
timeout_left = timeout_watch.leftover(True)
|
|
if timeout_left is not None and (
|
|
batch_timeout is None or timeout_left < batch_timeout):
|
|
batch_timeout = timeout_left
|
|
|
|
with timeutils.StopWatch(batch_timeout) as batch_timeout_watch:
|
|
# poll remained batch messages
|
|
while len(incomings) < batch_size and msg is not None:
|
|
msg = func(in_self, timeout=batch_timeout_watch.leftover(True))
|
|
if msg is not None:
|
|
incomings.append(msg)
|
|
|
|
return incomings
|
|
return wrapper
|
|
|
|
|
|
class TransportDriverError(exceptions.MessagingException):
|
|
"""Base class for transport driver specific exceptions."""
|
|
|
|
|
|
@six.add_metaclass(abc.ABCMeta)
|
|
class IncomingMessage(object):
|
|
|
|
def __init__(self, ctxt, message):
|
|
self.ctxt = ctxt
|
|
self.message = message
|
|
|
|
def acknowledge(self):
|
|
"""Acknowledge the message."""
|
|
|
|
@abc.abstractmethod
|
|
def requeue(self):
|
|
"""Requeue the message."""
|
|
|
|
|
|
@six.add_metaclass(abc.ABCMeta)
|
|
class RpcIncomingMessage(IncomingMessage):
|
|
|
|
@abc.abstractmethod
|
|
def reply(self, reply=None, failure=None):
|
|
"""Send a reply or failure back to the client."""
|
|
|
|
|
|
@six.add_metaclass(abc.ABCMeta)
|
|
class PollStyleListener(object):
|
|
def __init__(self, prefetch_size=-1):
|
|
self.prefetch_size = prefetch_size
|
|
|
|
@abc.abstractmethod
|
|
def poll(self, timeout=None, batch_size=1, batch_timeout=None):
|
|
"""Blocking until 'batch_size' message is pending and return
|
|
[IncomingMessage].
|
|
Waits for first message. Then waits for next batch_size-1 messages
|
|
during batch window defined by batch_timeout
|
|
This method block current thread until message comes, stop() is
|
|
executed by another thread or timemout is elapsed.
|
|
"""
|
|
|
|
def stop(self):
|
|
"""Stop listener.
|
|
Stop the listener message polling
|
|
"""
|
|
pass
|
|
|
|
def cleanup(self):
|
|
"""Cleanup listener.
|
|
Close connection (socket) used by listener if any.
|
|
As this is listener specific method, overwrite it in to derived class
|
|
if cleanup of listener required.
|
|
"""
|
|
pass
|
|
|
|
|
|
@six.add_metaclass(abc.ABCMeta)
|
|
class Listener(object):
|
|
def __init__(self, batch_size, batch_timeout,
|
|
prefetch_size=-1):
|
|
"""Init Listener
|
|
|
|
:param batch_size: desired number of messages passed to
|
|
single on_incoming_callback notification
|
|
:param batch_timeout: defines how long should we wait for batch_size
|
|
messages if we already have some messages waiting for processing
|
|
:param prefetch_size: defines how many massages we want to prefetch
|
|
from backend (depend on driver type) by single request
|
|
"""
|
|
self.on_incoming_callback = None
|
|
self.batch_timeout = batch_timeout
|
|
self.prefetch_size = prefetch_size
|
|
if prefetch_size > 0:
|
|
batch_size = min(batch_size, prefetch_size)
|
|
self.batch_size = batch_size
|
|
|
|
def start(self, on_incoming_callback):
|
|
"""Start listener.
|
|
Start the listener message polling
|
|
|
|
:param on_incoming_callback: callback function to be executed when
|
|
listener received messages. Messages should be processed and
|
|
acked/nacked by callback
|
|
"""
|
|
self.on_incoming_callback = on_incoming_callback
|
|
|
|
def stop(self):
|
|
"""Stop listener.
|
|
Stop the listener message polling
|
|
"""
|
|
self.on_incoming_callback = None
|
|
|
|
@abc.abstractmethod
|
|
def cleanup(self):
|
|
"""Cleanup listener.
|
|
Close connection (socket) used by listener if any.
|
|
As this is listener specific method, overwrite it in to derived class
|
|
if cleanup of listener required.
|
|
"""
|
|
|
|
|
|
class PollStyleListenerAdapter(Listener):
|
|
def __init__(self, poll_style_listener, batch_size, batch_timeout):
|
|
super(PollStyleListenerAdapter, self).__init__(
|
|
batch_size, batch_timeout, poll_style_listener.prefetch_size
|
|
)
|
|
self._poll_style_listener = poll_style_listener
|
|
self._listen_thread = threading.Thread(target=self._runner)
|
|
self._listen_thread.daemon = True
|
|
self._started = False
|
|
|
|
def start(self, on_incoming_callback):
|
|
"""Start listener.
|
|
Start the listener message polling
|
|
|
|
:param on_incoming_callback: callback function to be executed when
|
|
listener received messages. Messages should be processed and
|
|
acked/nacked by callback
|
|
"""
|
|
super(PollStyleListenerAdapter, self).start(on_incoming_callback)
|
|
self._started = True
|
|
self._listen_thread.start()
|
|
|
|
@excutils.forever_retry_uncaught_exceptions
|
|
def _runner(self):
|
|
while self._started:
|
|
incoming = self._poll_style_listener.poll(
|
|
batch_size=self.batch_size, batch_timeout=self.batch_timeout)
|
|
|
|
if incoming:
|
|
self.on_incoming_callback(incoming)
|
|
|
|
# listener is stopped but we need to process all already consumed
|
|
# messages
|
|
while True:
|
|
incoming = self._poll_style_listener.poll(
|
|
batch_size=self.batch_size, batch_timeout=self.batch_timeout)
|
|
|
|
if not incoming:
|
|
return
|
|
self.on_incoming_callback(incoming)
|
|
|
|
def stop(self):
|
|
"""Stop listener.
|
|
Stop the listener message polling
|
|
"""
|
|
self._started = False
|
|
self._poll_style_listener.stop()
|
|
self._listen_thread.join()
|
|
super(PollStyleListenerAdapter, self).stop()
|
|
|
|
def cleanup(self):
|
|
"""Cleanup listener.
|
|
Close connection (socket) used by listener if any.
|
|
As this is listener specific method, overwrite it in to derived class
|
|
if cleanup of listener required.
|
|
"""
|
|
self._poll_style_listener.cleanup()
|
|
|
|
|
|
@six.add_metaclass(abc.ABCMeta)
|
|
class BaseDriver(object):
|
|
prefetch_size = 0
|
|
|
|
def __init__(self, conf, url,
|
|
default_exchange=None, allowed_remote_exmods=None):
|
|
self.conf = conf
|
|
self._url = url
|
|
self._default_exchange = default_exchange
|
|
self._allowed_remote_exmods = allowed_remote_exmods or []
|
|
|
|
def require_features(self, requeue=False):
|
|
if requeue:
|
|
raise NotImplementedError('Message requeueing not supported by '
|
|
'this transport driver')
|
|
|
|
@abc.abstractmethod
|
|
def send(self, target, ctxt, message,
|
|
wait_for_reply=None, timeout=None, envelope=False):
|
|
"""Send a message to the given target."""
|
|
|
|
@abc.abstractmethod
|
|
def send_notification(self, target, ctxt, message, version):
|
|
"""Send a notification message to the given target."""
|
|
|
|
@abc.abstractmethod
|
|
def listen(self, target, batch_size, batch_timeout):
|
|
"""Construct a Listener for the given target."""
|
|
|
|
@abc.abstractmethod
|
|
def listen_for_notifications(self, targets_and_priorities, pool,
|
|
batch_size, batch_timeout):
|
|
"""Construct a notification Listener for the given list of
|
|
tuple of (target, priority).
|
|
"""
|
|
|
|
@abc.abstractmethod
|
|
def cleanup(self):
|
|
"""Release all resources."""
|