From 20123399825c4ccacb78ebde56645856a6895a2f Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 8 Jan 2015 20:29:47 -0800 Subject: [PATCH] Make more memcache options configurable More memcache options can be set in the memcache.conf or proxy-server.conf * connect_timeout * pool_timeout * tries * io_timeout Options set in proxy-server.conf are considered more specific to the memcache middleware. DocImpact Change-Id: I194d0f4d88c6cb8c797a37dcab48f2d8473e7a4e --- etc/memcache.conf-sample | 9 ++ etc/proxy-server.conf-sample | 2 + swift/common/middleware/memcache.py | 25 +++- test/unit/__init__.py | 13 +- test/unit/common/middleware/test_memcache.py | 132 +++++++++++++++++++ 5 files changed, 176 insertions(+), 5 deletions(-) diff --git a/etc/memcache.conf-sample b/etc/memcache.conf-sample index 24d24c8103..7ec55f100f 100644 --- a/etc/memcache.conf-sample +++ b/etc/memcache.conf-sample @@ -16,3 +16,12 @@ # # Sets the maximum number of connections to each memcached server per worker # memcache_max_connections = 2 +# +# Timeout for connection +# connect_timeout = 0.3 +# Timeout for pooled connection +# pool_timeout = 1.0 +# number of servers to retry on failures getting a pooled connection +# tries = 3 +# Timeout for read and writes +# io_timeout = 2.0 diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 8eff0c8709..1091268391 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -348,6 +348,8 @@ use = egg:swift#memcache # # Sets the maximum number of connections to each memcached server per worker # memcache_max_connections = 2 +# +# More options documented in memcache.conf-sample [filter:ratelimit] use = egg:swift#ratelimit diff --git a/swift/common/middleware/memcache.py b/swift/common/middleware/memcache.py index 19a2f4197f..1b66716b15 100644 --- a/swift/common/middleware/memcache.py +++ b/swift/common/middleware/memcache.py @@ -16,7 +16,8 @@ import os from ConfigParser import ConfigParser, NoSectionError, NoOptionError -from swift.common.memcached import MemcacheRing +from swift.common.memcached import (MemcacheRing, CONN_TIMEOUT, POOL_TIMEOUT, + IO_TIMEOUT, TRY_COUNT) class MemcacheMiddleware(object): @@ -36,6 +37,7 @@ class MemcacheMiddleware(object): except ValueError: max_conns = 0 + memcache_options = {} if (not self.memcache_servers or serialization_format is None or max_conns <= 0): @@ -43,6 +45,12 @@ class MemcacheMiddleware(object): 'memcache.conf') memcache_conf = ConfigParser() if memcache_conf.read(path): + # if memcache.conf exists we'll start with those base options + try: + memcache_options = dict(memcache_conf.items('memcache')) + except NoSectionError: + pass + if not self.memcache_servers: try: self.memcache_servers = \ @@ -65,6 +73,17 @@ class MemcacheMiddleware(object): except (NoSectionError, NoOptionError, ValueError): pass + # while memcache.conf options are the base for the memcache + # middleware, if you set the same option also in the filter + # section of the proxy config it is more specific. + memcache_options.update(conf) + connect_timeout = float(memcache_options.get( + 'connect_timeout', CONN_TIMEOUT)) + pool_timeout = float(memcache_options.get( + 'pool_timeout', POOL_TIMEOUT)) + tries = int(memcache_options.get('tries', TRY_COUNT)) + io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT)) + if not self.memcache_servers: self.memcache_servers = '127.0.0.1:11211' if max_conns <= 0: @@ -76,6 +95,10 @@ class MemcacheMiddleware(object): self.memcache = MemcacheRing( [s.strip() for s in self.memcache_servers.split(',') if s.strip()], + connect_timeout=connect_timeout, + pool_timeout=pool_timeout, + tries=tries, + io_timeout=io_timeout, allow_pickle=(serialization_format == 0), allow_unpickle=(serialization_format <= 1), max_conns=max_conns) diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 43aa677502..0e10d3bac0 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -29,8 +29,7 @@ from eventlet.green import socket from tempfile import mkdtemp from shutil import rmtree from test import get_config -from swift.common import swob -from swift.common.utils import config_true_value, LogAdapter +from swift.common import swob, utils from swift.common.ring import Ring, RingData from hashlib import md5 from eventlet import sleep, Timeout @@ -42,6 +41,11 @@ import cPickle as pickle from gzip import GzipFile import mock as mocklib +# try not to import this module from swift +if not os.path.basename(sys.argv[0]).startswith('swift'): + # never patch HASH_PATH_SUFFIX AGAIN! + utils.HASH_PATH_SUFFIX = 'endcap' + def patch_policies(thing_or_policies=None, legacy_only=False): if legacy_only: @@ -485,7 +489,7 @@ class DebugLogger(FakeLogger): print self.formatter.format(record) -class DebugLogAdapter(LogAdapter): +class DebugLogAdapter(utils.LogAdapter): def _send_to_logger(name): def stub_fn(self, *args, **kwargs): @@ -527,7 +531,8 @@ def fake_syslog_handler(): logging.handlers.SysLogHandler = FakeLogger -if config_true_value(get_config('unit_test').get('fake_syslog', 'False')): +if utils.config_true_value( + get_config('unit_test').get('fake_syslog', 'False')): fake_syslog_handler() diff --git a/test/unit/common/middleware/test_memcache.py b/test/unit/common/middleware/test_memcache.py index 3273ee5a65..d45fca7e89 100644 --- a/test/unit/common/middleware/test_memcache.py +++ b/test/unit/common/middleware/test_memcache.py @@ -13,12 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +from textwrap import dedent import unittest from ConfigParser import NoSectionError, NoOptionError +import mock + from swift.common.middleware import memcache from swift.common.memcached import MemcacheRing from swift.common.swob import Request +from swift.common.wsgi import loadapp + +from test.unit import with_tempdir, patch_policies class FakeApp(object): @@ -49,6 +56,16 @@ def get_config_parser(memcache_servers='1.2.3.4:5', class SetConfigParser(object): + def items(self, section_name): + if section_name != section: + raise NoSectionError(section_name) + return { + 'memcache_servers': memcache_servers, + 'memcache_serialization_support': + memcache_serialization_support, + 'memcache_max_connections': memcache_max_connections, + } + def read(self, path): return True @@ -295,5 +312,120 @@ class TestCacheMiddleware(unittest.TestCase): self.assertEquals( thefilter.memcache._client_cache['10.10.10.10:10'].max_size, 3) + @patch_policies + def _loadapp(self, proxy_config_path): + """ + Load a proxy from an app.conf to get the memcache_ring + + :returns: the memcache_ring of the memcache middleware filter + """ + with mock.patch('swift.proxy.server.Ring'): + app = loadapp(proxy_config_path) + memcache_ring = None + while True: + memcache_ring = getattr(app, 'memcache', None) + if memcache_ring: + break + app = app.app + return memcache_ring + + @with_tempdir + def test_real_config(self, tempdir): + config = """ + [pipeline:main] + pipeline = cache proxy-server + + [app:proxy-server] + use = egg:swift#proxy + + [filter:cache] + use = egg:swift#memcache + """ + config_path = os.path.join(tempdir, 'test.conf') + with open(config_path, 'w') as f: + f.write(dedent(config)) + memcache_ring = self._loadapp(config_path) + # only one server by default + self.assertEqual(memcache_ring._client_cache.keys(), + ['127.0.0.1:11211']) + # extra options + self.assertEqual(memcache_ring._connect_timeout, 0.3) + self.assertEqual(memcache_ring._pool_timeout, 1.0) + # tries is limited to server count + self.assertEqual(memcache_ring._tries, 1) + self.assertEqual(memcache_ring._io_timeout, 2.0) + + @with_tempdir + def test_real_config_with_options(self, tempdir): + config = """ + [pipeline:main] + pipeline = cache proxy-server + + [app:proxy-server] + use = egg:swift#proxy + + [filter:cache] + use = egg:swift#memcache + memcache_servers = 10.0.0.1:11211,10.0.0.2:11211,10.0.0.3:11211, + 10.0.0.4:11211 + connect_timeout = 1.0 + pool_timeout = 0.5 + tries = 4 + io_timeout = 1.0 + """ + config_path = os.path.join(tempdir, 'test.conf') + with open(config_path, 'w') as f: + f.write(dedent(config)) + memcache_ring = self._loadapp(config_path) + self.assertEqual(sorted(memcache_ring._client_cache.keys()), + ['10.0.0.%d:11211' % i for i in range(1, 5)]) + # extra options + self.assertEqual(memcache_ring._connect_timeout, 1.0) + self.assertEqual(memcache_ring._pool_timeout, 0.5) + # tries is limited to server count + self.assertEqual(memcache_ring._tries, 4) + self.assertEqual(memcache_ring._io_timeout, 1.0) + + @with_tempdir + def test_real_memcache_config(self, tempdir): + proxy_config = """ + [DEFAULT] + swift_dir = %s + + [pipeline:main] + pipeline = cache proxy-server + + [app:proxy-server] + use = egg:swift#proxy + + [filter:cache] + use = egg:swift#memcache + connect_timeout = 1.0 + """ % tempdir + proxy_config_path = os.path.join(tempdir, 'test.conf') + with open(proxy_config_path, 'w') as f: + f.write(dedent(proxy_config)) + + memcache_config = """ + [memcache] + memcache_servers = 10.0.0.1:11211,10.0.0.2:11211,10.0.0.3:11211, + 10.0.0.4:11211 + connect_timeout = 0.5 + io_timeout = 1.0 + """ + memcache_config_path = os.path.join(tempdir, 'memcache.conf') + with open(memcache_config_path, 'w') as f: + f.write(dedent(memcache_config)) + memcache_ring = self._loadapp(proxy_config_path) + self.assertEqual(sorted(memcache_ring._client_cache.keys()), + ['10.0.0.%d:11211' % i for i in range(1, 5)]) + # proxy option takes precedence + self.assertEqual(memcache_ring._connect_timeout, 1.0) + # default tries are not limited by servers + self.assertEqual(memcache_ring._tries, 3) + # memcache conf options are defaults + self.assertEqual(memcache_ring._io_timeout, 1.0) + + if __name__ == '__main__': unittest.main()