From 408588389576f5a239651034bee25c0961b12c81 Mon Sep 17 00:00:00 2001 From: Paul Luse Date: Fri, 18 Apr 2014 10:16:11 -0700 Subject: [PATCH] Add Ratelimit parameters to /info We previously registered with no parameters, added parms so they are displayed as follows (example): "ratelimit": {"max_sleep_time_seconds": 60.0, "container_listing_ratelimits": [[0, 100.0], [10, 50.0], [50, 20.0]], "container_ratelimits": [[0, 100.0], [10, 50.0], [50, 20.0]], "account_ratelimit": 1.0} Note that not all parameters are exposed (intentionally) via /info Change-Id: I36c7ef15af17e3eb8ebb93429035bd06d089a945 Closes-Bug: 1308989 --- swift/common/middleware/ratelimit.py | 42 +++++-- test/unit/common/middleware/test_ratelimit.py | 106 ++++++++++++++---- 2 files changed, 117 insertions(+), 31 deletions(-) diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index b88fbccc88..6e8dd87298 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -24,7 +24,16 @@ from swift.common.memcached import MemcacheConnectionError from swift.common.swob import Request, Response -def interpret_conf_limits(conf, name_prefix): +def interpret_conf_limits(conf, name_prefix, info=None): + """ + Parses general parms for rate limits looking for things that + start with the provided name_prefix within the provided conf + and returns lists for both internal use and for /info + + :param conf: conf dict to parse + :param name_prefix: prefix of config parms to look for + :param info: set to return extra stuff for /info registration + """ conf_limits = [] for conf_key in conf: if conf_key.startswith(name_prefix): @@ -34,6 +43,7 @@ def interpret_conf_limits(conf, name_prefix): conf_limits.sort() ratelimits = [] + conf_limits_info = list(conf_limits) while conf_limits: cur_size, cur_rate = conf_limits.pop(0) if conf_limits: @@ -49,8 +59,10 @@ def interpret_conf_limits(conf, name_prefix): line_func = lambda x: cur_rate ratelimits.append((cur_size, cur_rate, line_func)) - - return ratelimits + if info is None: + return ratelimits + else: + return ratelimits, conf_limits_info def get_maxrate(ratelimits, size): @@ -84,11 +96,10 @@ class RateLimitMiddleware(object): BLACK_LIST_SLEEP = 1 def __init__(self, app, conf, logger=None): + self.app = app - if logger: - self.logger = logger - else: - self.logger = get_logger(conf, log_route='ratelimit') + self.logger = logger or get_logger(conf, log_route='ratelimit') + self.memcache_client = None self.account_ratelimit = float(conf.get('account_ratelimit', 0)) self.max_sleep_time_seconds = \ float(conf.get('max_sleep_time_seconds', 60)) @@ -102,7 +113,6 @@ class RateLimitMiddleware(object): self.ratelimit_blacklist = \ [acc.strip() for acc in conf.get('account_blacklist', '').split(',') if acc.strip()] - self.memcache_client = None self.container_ratelimits = interpret_conf_limits( conf, 'container_ratelimit_') self.container_listing_ratelimits = interpret_conf_limits( @@ -289,8 +299,22 @@ def filter_factory(global_conf, **local_conf): """ conf = global_conf.copy() conf.update(local_conf) - register_swift_info('ratelimit') + + account_ratelimit = float(conf.get('account_ratelimit', 0)) + max_sleep_time_seconds = \ + float(conf.get('max_sleep_time_seconds', 60)) + container_ratelimits, cont_limit_info = interpret_conf_limits( + conf, 'container_ratelimit_', info=1) + container_listing_ratelimits, cont_list_limit_info = \ + interpret_conf_limits(conf, 'container_listing_ratelimit_', info=1) + # not all limits are exposed (intentionally) + register_swift_info('ratelimit', + account_ratelimit=account_ratelimit, + max_sleep_time_seconds=max_sleep_time_seconds, + container_ratelimits=cont_limit_info, + container_listing_ratelimits=cont_list_limit_info) def limit_filter(app): return RateLimitMiddleware(app, conf) + return limit_filter diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 25b1258000..a502bafc99 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -26,6 +26,7 @@ from swift.proxy.controllers.base import get_container_memcache_key, \ headers_to_container_info from swift.common.memcached import MemcacheConnectionError from swift.common.swob import Request +from swift.common import utils class FakeMemcache(object): @@ -109,14 +110,6 @@ def start_response(*args): pass -def dummy_filter_factory(global_conf, **local_conf): - conf = global_conf.copy() - conf.update(local_conf) - - def limit_filter(app): - return ratelimit.RateLimitMiddleware(app, conf, logger=FakeLogger()) - return limit_filter - time_ticker = 0 time_override = [] @@ -173,7 +166,8 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'container_ratelimit_10': 200, 'container_ratelimit_50': 100, 'container_ratelimit_75': 30} - test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) + test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) + test_ratelimit.logger = FakeLogger() self.assertEquals(ratelimit.get_maxrate( test_ratelimit.container_ratelimits, 0), None) self.assertEquals(ratelimit.get_maxrate( @@ -192,8 +186,7 @@ class TestRateLimit(unittest.TestCase): fake_memcache = FakeMemcache() fake_memcache.store[get_container_memcache_key('a', 'c')] = \ {'object_count': '5'} - the_app = ratelimit.RateLimitMiddleware(None, conf_dict, - logger=FakeLogger()) + the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.environ = {} @@ -246,8 +239,7 @@ class TestRateLimit(unittest.TestCase): fake_memcache = FakeMemcache() fake_memcache.store[get_container_memcache_key('a', 'c')] = \ {'container_size': 5} - the_app = ratelimit.RateLimitMiddleware(None, conf_dict, - logger=FakeLogger()) + the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.method = 'PUT' @@ -303,7 +295,7 @@ class TestRateLimit(unittest.TestCase): 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} - self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) + self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/a/c') req.environ['swift.cache'] = FakeMemcache() @@ -337,7 +329,8 @@ class TestRateLimit(unittest.TestCase): 'max_sleep_time_seconds': 2, 'account_whitelist': 'a', 'account_blacklist': 'b'} - self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) + self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) + self.test_ratelimit.logger = FakeLogger() self.test_ratelimit.BLACK_LIST_SLEEP = 0 ratelimit.http_connect = mock_http_connect(204) req = Request.blank('/v/b/c') @@ -372,7 +365,7 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'account_ratelimit': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} - self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) + self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c') @@ -403,7 +396,7 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'container_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} - self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) + self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c/o') @@ -437,7 +430,7 @@ class TestRateLimit(unittest.TestCase): conf_dict = {'container_listing_ratelimit_0': current_rate, 'clock_accuracy': 100, 'max_sleep_time_seconds': 1} - self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp()) + self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) ratelimit.http_connect = mock_http_connect(204) self.test_ratelimit.log_sleep_time_seconds = .00001 req = Request.blank('/v/a/c') @@ -461,6 +454,14 @@ class TestRateLimit(unittest.TestCase): mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], '204 No Content') + mc = self.test_ratelimit.memcache_client + try: + self.test_ratelimit.memcache_client = None + self.assertEquals( + self.test_ratelimit.handle_ratelimit(req, 'n', 'c', None), + None) + finally: + self.test_ratelimit.memcache_client = mc def test_ratelimit_max_rate_multiple_acc(self): num_calls = 4 @@ -469,8 +470,7 @@ class TestRateLimit(unittest.TestCase): 'max_sleep_time_seconds': 2} fake_memcache = FakeMemcache() - the_app = ratelimit.RateLimitMiddleware(None, conf_dict, - logger=FakeLogger()) + the_app = ratelimit.filter_factory(conf_dict)(FakeApp()) the_app.memcache_client = fake_memcache req = lambda: None req.method = 'PUT' @@ -512,8 +512,7 @@ class TestRateLimit(unittest.TestCase): 'SERVER_PROTOCOL': 'HTTP/1.0'} app = lambda *args, **kwargs: ['fake_app'] - rate_mid = ratelimit.RateLimitMiddleware(app, {}, - logger=FakeLogger()) + rate_mid = ratelimit.filter_factory({})(app) class a_callable(object): @@ -556,5 +555,68 @@ class TestRateLimit(unittest.TestCase): time_took = time.time() - begin self.assertEquals(round(time_took, 1), 0) # no memcache, no limit + +class TestSwiftInfo(unittest.TestCase): + def setUp(self): + utils._swift_info = {} + utils._swift_admin_info = {} + + def test_registered_defaults(self): + + def check_key_is_absnet(key): + try: + swift_info[key] + except KeyError as err: + if key not in err: + raise + + test_limits = {'account_ratelimit': 1, + 'max_sleep_time_seconds': 60, + 'container_ratelimit_0': 0, + 'container_ratelimit_10': 10, + 'container_ratelimit_50': 50, + 'container_listing_ratelimit_0': 0, + 'container_listing_ratelimit_10': 10, + 'container_listing_ratelimit_50': 50} + + ratelimit.filter_factory(test_limits)('have to pass in an app') + swift_info = utils.get_swift_info() + self.assertTrue('ratelimit' in swift_info) + self.assertEqual(swift_info['ratelimit'] + ['account_ratelimit'], 1.0) + self.assertEqual(swift_info['ratelimit'] + ['max_sleep_time_seconds'], 60.0) + self.assertEqual(swift_info['ratelimit'] + ['container_ratelimits'][0][0], 0) + self.assertEqual(swift_info['ratelimit'] + ['container_ratelimits'][0][1], 0.0) + self.assertEqual(swift_info['ratelimit'] + ['container_ratelimits'][1][0], 10) + self.assertEqual(swift_info['ratelimit'] + ['container_ratelimits'][1][1], 10.0) + self.assertEqual(swift_info['ratelimit'] + ['container_ratelimits'][2][0], 50) + self.assertEqual(swift_info['ratelimit'] + ['container_ratelimits'][2][1], 50.0) + self.assertEqual(swift_info['ratelimit'] + ['container_listing_ratelimits'][0][0], 0) + self.assertEqual(swift_info['ratelimit'] + ['container_listing_ratelimits'][0][1], 0.0) + self.assertEqual(swift_info['ratelimit'] + ['container_listing_ratelimits'][1][0], 10) + self.assertEqual(swift_info['ratelimit'] + ['container_listing_ratelimits'][1][1], 10.0) + self.assertEqual(swift_info['ratelimit'] + ['container_listing_ratelimits'][2][0], 50) + self.assertEqual(swift_info['ratelimit'] + ['container_listing_ratelimits'][2][1], 50.0) + + # these were left out on purpose + for key in ['log_sleep_time_seconds', 'clock_accuracy', + 'rate_buffer_seconds', 'ratelimit_whitelis', + 'ratelimit_blacklist']: + check_key_is_absnet(key) + + if __name__ == '__main__': unittest.main()