Add Ratelimit parameters to /info
We previously registered with no parameters, added parms so they
are displayed as follows (example):
"ratelimit": {"max_sleep_time_seconds": 60.0,
"container_listing_ratelimits": [[0, 100.0], [10, 50.0], [50, 20.0]],
"container_ratelimits": [[0, 100.0], [10, 50.0], [50, 20.0]],
"account_ratelimit": 1.0}
Note that not all parameters are exposed (intentionally) via /info
Change-Id: I36c7ef15af17e3eb8ebb93429035bd06d089a945
Closes-Bug: 1308989
This commit is contained in:
Paul Luse
John Dickinson
parent
373e06a6a0
commit
4085883895
|
|
@ -24,7 +24,16 @@ from swift.common.memcached import MemcacheConnectionError
|
|||
from swift.common.swob import Request, Response
|
||||
|
||||
|
||||
def interpret_conf_limits(conf, name_prefix):
|
||||
def interpret_conf_limits(conf, name_prefix, info=None):
|
||||
"""
|
||||
Parses general parms for rate limits looking for things that
|
||||
start with the provided name_prefix within the provided conf
|
||||
and returns lists for both internal use and for /info
|
||||
|
||||
:param conf: conf dict to parse
|
||||
:param name_prefix: prefix of config parms to look for
|
||||
:param info: set to return extra stuff for /info registration
|
||||
"""
|
||||
conf_limits = []
|
||||
for conf_key in conf:
|
||||
if conf_key.startswith(name_prefix):
|
||||
|
|
@ -34,6 +43,7 @@ def interpret_conf_limits(conf, name_prefix):
|
|||
|
||||
conf_limits.sort()
|
||||
ratelimits = []
|
||||
conf_limits_info = list(conf_limits)
|
||||
while conf_limits:
|
||||
cur_size, cur_rate = conf_limits.pop(0)
|
||||
if conf_limits:
|
||||
|
|
@ -49,8 +59,10 @@ def interpret_conf_limits(conf, name_prefix):
|
|||
line_func = lambda x: cur_rate
|
||||
|
||||
ratelimits.append((cur_size, cur_rate, line_func))
|
||||
|
||||
return ratelimits
|
||||
if info is None:
|
||||
return ratelimits
|
||||
else:
|
||||
return ratelimits, conf_limits_info
|
||||
|
||||
|
||||
def get_maxrate(ratelimits, size):
|
||||
|
|
@ -84,11 +96,10 @@ class RateLimitMiddleware(object):
|
|||
BLACK_LIST_SLEEP = 1
|
||||
|
||||
def __init__(self, app, conf, logger=None):
|
||||
|
||||
self.app = app
|
||||
if logger:
|
||||
self.logger = logger
|
||||
else:
|
||||
self.logger = get_logger(conf, log_route='ratelimit')
|
||||
self.logger = logger or get_logger(conf, log_route='ratelimit')
|
||||
self.memcache_client = None
|
||||
self.account_ratelimit = float(conf.get('account_ratelimit', 0))
|
||||
self.max_sleep_time_seconds = \
|
||||
float(conf.get('max_sleep_time_seconds', 60))
|
||||
|
|
@ -102,7 +113,6 @@ class RateLimitMiddleware(object):
|
|||
self.ratelimit_blacklist = \
|
||||
[acc.strip() for acc in
|
||||
conf.get('account_blacklist', '').split(',') if acc.strip()]
|
||||
self.memcache_client = None
|
||||
self.container_ratelimits = interpret_conf_limits(
|
||||
conf, 'container_ratelimit_')
|
||||
self.container_listing_ratelimits = interpret_conf_limits(
|
||||
|
|
@ -289,8 +299,22 @@ def filter_factory(global_conf, **local_conf):
|
|||
"""
|
||||
conf = global_conf.copy()
|
||||
conf.update(local_conf)
|
||||
register_swift_info('ratelimit')
|
||||
|
||||
account_ratelimit = float(conf.get('account_ratelimit', 0))
|
||||
max_sleep_time_seconds = \
|
||||
float(conf.get('max_sleep_time_seconds', 60))
|
||||
container_ratelimits, cont_limit_info = interpret_conf_limits(
|
||||
conf, 'container_ratelimit_', info=1)
|
||||
container_listing_ratelimits, cont_list_limit_info = \
|
||||
interpret_conf_limits(conf, 'container_listing_ratelimit_', info=1)
|
||||
# not all limits are exposed (intentionally)
|
||||
register_swift_info('ratelimit',
|
||||
account_ratelimit=account_ratelimit,
|
||||
max_sleep_time_seconds=max_sleep_time_seconds,
|
||||
container_ratelimits=cont_limit_info,
|
||||
container_listing_ratelimits=cont_list_limit_info)
|
||||
|
||||
def limit_filter(app):
|
||||
return RateLimitMiddleware(app, conf)
|
||||
|
||||
return limit_filter
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from swift.proxy.controllers.base import get_container_memcache_key, \
|
|||
headers_to_container_info
|
||||
from swift.common.memcached import MemcacheConnectionError
|
||||
from swift.common.swob import Request
|
||||
from swift.common import utils
|
||||
|
||||
|
||||
class FakeMemcache(object):
|
||||
|
|
@ -109,14 +110,6 @@ def start_response(*args):
|
|||
pass
|
||||
|
||||
|
||||
def dummy_filter_factory(global_conf, **local_conf):
|
||||
conf = global_conf.copy()
|
||||
conf.update(local_conf)
|
||||
|
||||
def limit_filter(app):
|
||||
return ratelimit.RateLimitMiddleware(app, conf, logger=FakeLogger())
|
||||
return limit_filter
|
||||
|
||||
time_ticker = 0
|
||||
time_override = []
|
||||
|
||||
|
|
@ -173,7 +166,8 @@ class TestRateLimit(unittest.TestCase):
|
|||
conf_dict = {'container_ratelimit_10': 200,
|
||||
'container_ratelimit_50': 100,
|
||||
'container_ratelimit_75': 30}
|
||||
test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
|
||||
test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
test_ratelimit.logger = FakeLogger()
|
||||
self.assertEquals(ratelimit.get_maxrate(
|
||||
test_ratelimit.container_ratelimits, 0), None)
|
||||
self.assertEquals(ratelimit.get_maxrate(
|
||||
|
|
@ -192,8 +186,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
fake_memcache = FakeMemcache()
|
||||
fake_memcache.store[get_container_memcache_key('a', 'c')] = \
|
||||
{'object_count': '5'}
|
||||
the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
|
||||
logger=FakeLogger())
|
||||
the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
the_app.memcache_client = fake_memcache
|
||||
req = lambda: None
|
||||
req.environ = {}
|
||||
|
|
@ -246,8 +239,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
fake_memcache = FakeMemcache()
|
||||
fake_memcache.store[get_container_memcache_key('a', 'c')] = \
|
||||
{'container_size': 5}
|
||||
the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
|
||||
logger=FakeLogger())
|
||||
the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
the_app.memcache_client = fake_memcache
|
||||
req = lambda: None
|
||||
req.method = 'PUT'
|
||||
|
|
@ -303,7 +295,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
'max_sleep_time_seconds': 2,
|
||||
'account_whitelist': 'a',
|
||||
'account_blacklist': 'b'}
|
||||
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
|
||||
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
ratelimit.http_connect = mock_http_connect(204)
|
||||
req = Request.blank('/v/a/c')
|
||||
req.environ['swift.cache'] = FakeMemcache()
|
||||
|
|
@ -337,7 +329,8 @@ class TestRateLimit(unittest.TestCase):
|
|||
'max_sleep_time_seconds': 2,
|
||||
'account_whitelist': 'a',
|
||||
'account_blacklist': 'b'}
|
||||
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
|
||||
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
self.test_ratelimit.logger = FakeLogger()
|
||||
self.test_ratelimit.BLACK_LIST_SLEEP = 0
|
||||
ratelimit.http_connect = mock_http_connect(204)
|
||||
req = Request.blank('/v/b/c')
|
||||
|
|
@ -372,7 +365,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
conf_dict = {'account_ratelimit': current_rate,
|
||||
'clock_accuracy': 100,
|
||||
'max_sleep_time_seconds': 1}
|
||||
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
|
||||
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
ratelimit.http_connect = mock_http_connect(204)
|
||||
self.test_ratelimit.log_sleep_time_seconds = .00001
|
||||
req = Request.blank('/v/a/c')
|
||||
|
|
@ -403,7 +396,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
conf_dict = {'container_ratelimit_0': current_rate,
|
||||
'clock_accuracy': 100,
|
||||
'max_sleep_time_seconds': 1}
|
||||
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
|
||||
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
ratelimit.http_connect = mock_http_connect(204)
|
||||
self.test_ratelimit.log_sleep_time_seconds = .00001
|
||||
req = Request.blank('/v/a/c/o')
|
||||
|
|
@ -437,7 +430,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
conf_dict = {'container_listing_ratelimit_0': current_rate,
|
||||
'clock_accuracy': 100,
|
||||
'max_sleep_time_seconds': 1}
|
||||
self.test_ratelimit = dummy_filter_factory(conf_dict)(FakeApp())
|
||||
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
ratelimit.http_connect = mock_http_connect(204)
|
||||
self.test_ratelimit.log_sleep_time_seconds = .00001
|
||||
req = Request.blank('/v/a/c')
|
||||
|
|
@ -461,6 +454,14 @@ class TestRateLimit(unittest.TestCase):
|
|||
mock_sleep(.1)
|
||||
r = self.test_ratelimit(req.environ, start_response)
|
||||
self.assertEquals(r[0], '204 No Content')
|
||||
mc = self.test_ratelimit.memcache_client
|
||||
try:
|
||||
self.test_ratelimit.memcache_client = None
|
||||
self.assertEquals(
|
||||
self.test_ratelimit.handle_ratelimit(req, 'n', 'c', None),
|
||||
None)
|
||||
finally:
|
||||
self.test_ratelimit.memcache_client = mc
|
||||
|
||||
def test_ratelimit_max_rate_multiple_acc(self):
|
||||
num_calls = 4
|
||||
|
|
@ -469,8 +470,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
'max_sleep_time_seconds': 2}
|
||||
fake_memcache = FakeMemcache()
|
||||
|
||||
the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
|
||||
logger=FakeLogger())
|
||||
the_app = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||
the_app.memcache_client = fake_memcache
|
||||
req = lambda: None
|
||||
req.method = 'PUT'
|
||||
|
|
@ -512,8 +512,7 @@ class TestRateLimit(unittest.TestCase):
|
|||
'SERVER_PROTOCOL': 'HTTP/1.0'}
|
||||
|
||||
app = lambda *args, **kwargs: ['fake_app']
|
||||
rate_mid = ratelimit.RateLimitMiddleware(app, {},
|
||||
logger=FakeLogger())
|
||||
rate_mid = ratelimit.filter_factory({})(app)
|
||||
|
||||
class a_callable(object):
|
||||
|
||||
|
|
@ -556,5 +555,68 @@ class TestRateLimit(unittest.TestCase):
|
|||
time_took = time.time() - begin
|
||||
self.assertEquals(round(time_took, 1), 0) # no memcache, no limit
|
||||
|
||||
|
||||
class TestSwiftInfo(unittest.TestCase):
|
||||
def setUp(self):
|
||||
utils._swift_info = {}
|
||||
utils._swift_admin_info = {}
|
||||
|
||||
def test_registered_defaults(self):
|
||||
|
||||
def check_key_is_absnet(key):
|
||||
try:
|
||||
swift_info[key]
|
||||
except KeyError as err:
|
||||
if key not in err:
|
||||
raise
|
||||
|
||||
test_limits = {'account_ratelimit': 1,
|
||||
'max_sleep_time_seconds': 60,
|
||||
'container_ratelimit_0': 0,
|
||||
'container_ratelimit_10': 10,
|
||||
'container_ratelimit_50': 50,
|
||||
'container_listing_ratelimit_0': 0,
|
||||
'container_listing_ratelimit_10': 10,
|
||||
'container_listing_ratelimit_50': 50}
|
||||
|
||||
ratelimit.filter_factory(test_limits)('have to pass in an app')
|
||||
swift_info = utils.get_swift_info()
|
||||
self.assertTrue('ratelimit' in swift_info)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['account_ratelimit'], 1.0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['max_sleep_time_seconds'], 60.0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_ratelimits'][0][0], 0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_ratelimits'][0][1], 0.0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_ratelimits'][1][0], 10)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_ratelimits'][1][1], 10.0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_ratelimits'][2][0], 50)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_ratelimits'][2][1], 50.0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_listing_ratelimits'][0][0], 0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_listing_ratelimits'][0][1], 0.0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_listing_ratelimits'][1][0], 10)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_listing_ratelimits'][1][1], 10.0)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_listing_ratelimits'][2][0], 50)
|
||||
self.assertEqual(swift_info['ratelimit']
|
||||
['container_listing_ratelimits'][2][1], 50.0)
|
||||
|
||||
# these were left out on purpose
|
||||
for key in ['log_sleep_time_seconds', 'clock_accuracy',
|
||||
'rate_buffer_seconds', 'ratelimit_whitelis',
|
||||
'ratelimit_blacklist']:
|
||||
check_key_is_absnet(key)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue