Introduce TTL for idle connections

We can reduce a workload of rabbitmq through implementation
of expiration mechanism for idle connections in the pool with
next properties:

 conn_pool_ttl (default 20 min)
 conn_pool_min_size: the pool size limit for expire() (default 2)

The problem is timeless idle connections in the pool, which can be created
via some single huge workload of RPCServer. One SEND connection is heartbeat
thread + some network activity every n second. So, we can reduce it.

Here is two ways to implement an expiration:

 [1] Create a separated thread for checking expire date of connections
 [2] Make call expire() on pool.get() or pool.put()

The [1] has some threading overhead, but probably insignificant
because the thread can sleep 99% time and wake up every 20 mins (by default).
Anyway current implementation is [2].

Change-Id: Ie8781d10549a044656824ceb78b2fe2e4f7f8b43
This commit is contained in:
Kirill Bespalov 2016-07-11 15:25:23 +03:00
parent afd5f8233c
commit 162f6e987b
5 changed files with 76 additions and 24 deletions

View File

@ -25,10 +25,13 @@ import six
from oslo_messaging import exceptions
base_opts = [
cfg.IntOpt('rpc_conn_pool_size',
default=30,
cfg.IntOpt('rpc_conn_pool_size', default=30,
deprecated_group='DEFAULT',
help='Size of RPC connection pool.'),
cfg.IntOpt('conn_pool_min_size', default=2,
help='The pool size limit for connections expiration policy'),
cfg.IntOpt('conn_pool_ttl', default=1200,
help='The time-to-live in sec of idle connections in the pool')
]

View File

@ -50,6 +50,12 @@ kafka_opts = [
cfg.IntOpt('pool_size', default=10,
help='Pool Size for Kafka Consumers'),
cfg.IntOpt('conn_pool_min_size', default=2,
help='The pool size limit for connections expiration policy'),
cfg.IntOpt('conn_pool_ttl', default=1200,
help='The time-to-live in sec of idle connections in the pool')
]
CONF = cfg.CONF
@ -301,8 +307,13 @@ class KafkaDriver(base.BaseDriver):
super(KafkaDriver, self).__init__(
conf, url, default_exchange, allowed_remote_exmods)
# the pool configuration properties
max_size = self.conf.oslo_messaging_kafka.pool_size
min_size = self.conf.oslo_messaging_kafka.conn_pool_min_size
ttl = self.conf.oslo_messaging_kafka.conn_pool_ttl
self.connection_pool = driver_pool.ConnectionPool(
self.conf, self.conf.oslo_messaging_kafka.pool_size,
self.conf, max_size, min_size, ttl,
self._url, Connection)
self.listeners = []

View File

@ -1321,8 +1321,13 @@ class RabbitDriver(amqpdriver.AMQPDriverBase):
self.prefetch_size = (
conf.oslo_messaging_rabbit.rabbit_qos_prefetch_count)
# the pool configuration properties
max_size = conf.oslo_messaging_rabbit.rpc_conn_pool_size
min_size = conf.oslo_messaging_rabbit.conn_pool_min_size
ttl = conf.oslo_messaging_rabbit.conn_pool_ttl
connection_pool = pool.ConnectionPool(
conf, conf.oslo_messaging_rabbit.rpc_conn_pool_size,
conf, max_size, min_size, ttl,
url, Connection)
super(RabbitDriver, self).__init__(

View File

@ -1,4 +1,3 @@
# Copyright 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -19,6 +18,7 @@ import sys
import threading
from oslo_log import log as logging
from oslo_utils import timeutils
import six
from oslo_messaging._drivers import common
@ -38,28 +38,48 @@ else:
@six.add_metaclass(abc.ABCMeta)
class Pool(object):
"""A thread-safe object pool.
Modelled after the eventlet.pools.Pool interface, but designed to be safe
when using native threads without the GIL.
Resizing is not supported.
"""
def __init__(self, max_size=4):
def __init__(self, max_size=4, min_size=2, ttl=1200, on_expire=None):
super(Pool, self).__init__()
self._min_size = min_size
self._max_size = max_size
self._item_ttl = ttl
self._current_size = 0
self._cond = threading.Condition()
self._items = collections.deque()
self._on_expire = on_expire
def expire(self):
"""Remove expired items from left (the oldest item) to
right (the newest item).
"""
with self._cond:
while len(self._items) > self._min_size:
try:
ttl_watch, item = self._items.popleft()
if ttl_watch.expired():
self._on_expire and self._on_expire(item)
self._current_size -= 1
else:
self._items.appendleft((ttl_watch, item))
return
except IndexError:
break
def put(self, item):
"""Return an item to the pool."""
with self._cond:
self._items.appendleft(item)
ttl_watch = timeutils.StopWatch(duration=self._item_ttl)
ttl_watch.start()
self._items.append((ttl_watch, item))
self._cond.notify()
def get(self):
@ -70,7 +90,9 @@ class Pool(object):
with self._cond:
while True:
try:
return self._items.popleft()
ttl_watch, item = self._items.pop()
self.expire()
return item
except IndexError:
pass
@ -90,12 +112,12 @@ class Pool(object):
def iter_free(self):
"""Iterate over free items."""
with self._cond:
while True:
try:
yield self._items.popleft()
except IndexError:
break
while True:
try:
_, item = self._items.pop()
yield item
except IndexError:
raise StopIteration
@abc.abstractmethod
def create(self):
@ -104,17 +126,20 @@ class Pool(object):
class ConnectionPool(Pool):
"""Class that implements a Pool of Connections."""
def __init__(self, conf, rpc_conn_pool_size, url, connection_cls):
def __init__(self, conf, max_size, min_size, ttl, url, connection_cls):
self.connection_cls = connection_cls
self.conf = conf
self.url = url
super(ConnectionPool, self).__init__(rpc_conn_pool_size)
self.reply_proxy = None
super(ConnectionPool, self).__init__(max_size, min_size, ttl,
self._on_expire)
# TODO(comstud): Timeout connections not used in a while
def create(self, purpose=None):
if purpose is None:
purpose = common.PURPOSE_SEND
def _on_expire(self, connection):
connection.close()
LOG.debug("Idle connection has expired and been closed."
" Pool size: %d" % len(self._items))
def create(self, purpose=common.PURPOSE_SEND):
LOG.debug('Pool creating new connection')
return self.connection_cls(self.conf, self.url, purpose)

View File

@ -0,0 +1,8 @@
---
features:
- |
| Idle connections in the pool will be expired and closed.
| Default ttl is 1200s. Next configuration params was added
* *conn_pool_ttl* (defaul 1200)
* *conn_pool_min_size* (default 2)