Change black/white-listing to use sysmeta.
The way we do this now involves a conf change and a proxy reload which is a pain. You can now just set these: X-Account-Sysmeta-Global-Write-Ratelimit: WHITELIST or X-Account-Sysmeta-Global-Write-Ratelimit: BLACKLIST NOTE: The existing proxy config settings: account_whitelist and account_blacklist will continue to work. Change-Id: I532663f1d2c75d03170c5fdb9b330416822fbc88
This commit is contained in:
parent
7528f2b221
commit
172a9b369f
@ -265,11 +265,6 @@ rate but better average accuracy. The default is 5.
|
|||||||
.IP \fBaccount_ratelimit\fR
|
.IP \fBaccount_ratelimit\fR
|
||||||
If set, will limit PUT and DELETE requests to /account_name/container_name. Number is
|
If set, will limit PUT and DELETE requests to /account_name/container_name. Number is
|
||||||
in requests per second. If set to 0 means disabled. The default is 0.
|
in requests per second. If set to 0 means disabled. The default is 0.
|
||||||
.IP \fBaccount_whitelist\fR
|
|
||||||
Comma separated lists of account names that will not be rate limited. The default is ''.
|
|
||||||
.IP \fBaccount_blacklist\fR
|
|
||||||
Comma separated lists of account names that will not be allowed. Returns a 497 response.
|
|
||||||
The default is ''.
|
|
||||||
.IP \fBcontainer_ratelimit_size\fR
|
.IP \fBcontainer_ratelimit_size\fR
|
||||||
When set with container_limit_x = r: for containers of size x, limit requests per second
|
When set with container_limit_x = r: for containers of size x, limit requests per second
|
||||||
to r. Will limit PUT, DELETE, and POST requests to /a/c/o. The default is ''.
|
to r. Will limit PUT, DELETE, and POST requests to /a/c/o. The default is ''.
|
||||||
|
@ -42,11 +42,6 @@ account_ratelimit 0 If set, will limit PUT and DELETE
|
|||||||
requests to
|
requests to
|
||||||
/account_name/container_name. Number
|
/account_name/container_name. Number
|
||||||
is in requests per second.
|
is in requests per second.
|
||||||
account_whitelist '' Comma separated lists of account names
|
|
||||||
that will not be rate limited.
|
|
||||||
account_blacklist '' Comma separated lists of account names
|
|
||||||
that will not be allowed. Returns a
|
|
||||||
497 response.
|
|
||||||
container_ratelimit_size '' When set with container_ratelimit_x =
|
container_ratelimit_size '' When set with container_ratelimit_x =
|
||||||
r: for containers of size x, limit
|
r: for containers of size x, limit
|
||||||
requests per second to r. Will limit
|
requests per second to r. Will limit
|
||||||
@ -85,6 +80,7 @@ Container Size Rate Limit
|
|||||||
Account Specific Ratelimiting
|
Account Specific Ratelimiting
|
||||||
-----------------------------
|
-----------------------------
|
||||||
|
|
||||||
|
|
||||||
The above ratelimiting is to prevent the "many writes to a single container"
|
The above ratelimiting is to prevent the "many writes to a single container"
|
||||||
bottleneck from causing a problem. There could also be a problem where a single
|
bottleneck from causing a problem. There could also be a problem where a single
|
||||||
account is just using too much of the cluster's resources. In this case, the
|
account is just using too much of the cluster's resources. In this case, the
|
||||||
@ -98,3 +94,17 @@ to the applicable account/container limits from above. This header will be
|
|||||||
hidden from the user, because of the gatekeeper middleware, and can only be set
|
hidden from the user, because of the gatekeeper middleware, and can only be set
|
||||||
using a direct client to the account nodes. It accepts a float value and will
|
using a direct client to the account nodes. It accepts a float value and will
|
||||||
only limit requests if the value is > 0.
|
only limit requests if the value is > 0.
|
||||||
|
|
||||||
|
-------------------
|
||||||
|
Black/White-listing
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
To blacklist or whitelist an account set:
|
||||||
|
|
||||||
|
X-Account-Sysmeta-Global-Write-Ratelimit: BLACKLIST
|
||||||
|
|
||||||
|
or
|
||||||
|
|
||||||
|
X-Account-Sysmeta-Global-Write-Ratelimit: WHITELIST
|
||||||
|
|
||||||
|
in the account headers.
|
||||||
|
@ -361,6 +361,9 @@ use = egg:swift#ratelimit
|
|||||||
# account_ratelimit of 0 means disabled
|
# account_ratelimit of 0 means disabled
|
||||||
# account_ratelimit = 0
|
# account_ratelimit = 0
|
||||||
|
|
||||||
|
# DEPRECATED- these will continue to work but will be replaced
|
||||||
|
# by the X-Account-Sysmeta-Global-Write-Ratelimit flag.
|
||||||
|
# Please see ratelimiting docs for details.
|
||||||
# these are comma separated lists of account names
|
# these are comma separated lists of account names
|
||||||
# account_whitelist = a,b
|
# account_whitelist = a,b
|
||||||
# account_blacklist = c,d
|
# account_blacklist = c,d
|
||||||
|
@ -127,7 +127,8 @@ class RateLimitMiddleware(object):
|
|||||||
return rv
|
return rv
|
||||||
|
|
||||||
def get_ratelimitable_key_tuples(self, req, account_name,
|
def get_ratelimitable_key_tuples(self, req, account_name,
|
||||||
container_name=None, obj_name=None):
|
container_name=None, obj_name=None,
|
||||||
|
global_ratelimit=None):
|
||||||
"""
|
"""
|
||||||
Returns a list of key (used in memcache), ratelimit tuples. Keys
|
Returns a list of key (used in memcache), ratelimit tuples. Keys
|
||||||
should be checked in order.
|
should be checked in order.
|
||||||
@ -136,9 +137,12 @@ class RateLimitMiddleware(object):
|
|||||||
:param account_name: account name from path
|
:param account_name: account name from path
|
||||||
:param container_name: container name from path
|
:param container_name: container name from path
|
||||||
:param obj_name: object name from path
|
:param obj_name: object name from path
|
||||||
|
:param global_ratelimit: this account has an account wide
|
||||||
|
ratelimit on all writes combined
|
||||||
"""
|
"""
|
||||||
keys = []
|
keys = []
|
||||||
# COPYs are not limited
|
# COPYs are not limited
|
||||||
|
|
||||||
if self.account_ratelimit and \
|
if self.account_ratelimit and \
|
||||||
account_name and container_name and not obj_name and \
|
account_name and container_name and not obj_name and \
|
||||||
req.method in ('PUT', 'DELETE'):
|
req.method in ('PUT', 'DELETE'):
|
||||||
@ -166,16 +170,13 @@ class RateLimitMiddleware(object):
|
|||||||
container_rate))
|
container_rate))
|
||||||
|
|
||||||
if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
|
if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
|
||||||
account_info = get_account_info(req.environ, self.app)
|
if global_ratelimit:
|
||||||
account_global_ratelimit = \
|
|
||||||
account_info.get('sysmeta', {}).get('global-write-ratelimit')
|
|
||||||
if account_global_ratelimit:
|
|
||||||
try:
|
try:
|
||||||
account_global_ratelimit = float(account_global_ratelimit)
|
global_ratelimit = float(global_ratelimit)
|
||||||
if account_global_ratelimit > 0:
|
if global_ratelimit > 0:
|
||||||
keys.append((
|
keys.append((
|
||||||
"ratelimit/global-write/%s" % account_name,
|
"ratelimit/global-write/%s" % account_name,
|
||||||
account_global_ratelimit))
|
global_ratelimit))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@ -229,18 +230,30 @@ class RateLimitMiddleware(object):
|
|||||||
'''
|
'''
|
||||||
if not self.memcache_client:
|
if not self.memcache_client:
|
||||||
return None
|
return None
|
||||||
if account_name in self.ratelimit_blacklist:
|
|
||||||
|
try:
|
||||||
|
account_info = get_account_info(req.environ, self.app)
|
||||||
|
account_global_ratelimit = \
|
||||||
|
account_info.get('sysmeta', {}).get('global-write-ratelimit')
|
||||||
|
except ValueError:
|
||||||
|
account_global_ratelimit = None
|
||||||
|
|
||||||
|
if account_name in self.ratelimit_whitelist or \
|
||||||
|
account_global_ratelimit == 'WHITELIST':
|
||||||
|
return None
|
||||||
|
|
||||||
|
if account_name in self.ratelimit_blacklist or \
|
||||||
|
account_global_ratelimit == 'BLACKLIST':
|
||||||
self.logger.error(_('Returning 497 because of blacklisting: %s'),
|
self.logger.error(_('Returning 497 because of blacklisting: %s'),
|
||||||
account_name)
|
account_name)
|
||||||
eventlet.sleep(self.BLACK_LIST_SLEEP)
|
eventlet.sleep(self.BLACK_LIST_SLEEP)
|
||||||
return Response(status='497 Blacklisted',
|
return Response(status='497 Blacklisted',
|
||||||
body='Your account has been blacklisted',
|
body='Your account has been blacklisted',
|
||||||
request=req)
|
request=req)
|
||||||
if account_name in self.ratelimit_whitelist:
|
|
||||||
return None
|
|
||||||
for key, max_rate in self.get_ratelimitable_key_tuples(
|
for key, max_rate in self.get_ratelimitable_key_tuples(
|
||||||
req, account_name, container_name=container_name,
|
req, account_name, container_name=container_name,
|
||||||
obj_name=obj_name):
|
obj_name=obj_name, global_ratelimit=account_global_ratelimit):
|
||||||
try:
|
try:
|
||||||
need_to_sleep = self._get_sleep_time(key, max_rate)
|
need_to_sleep = self._get_sleep_time(key, max_rate)
|
||||||
if self.log_sleep_time_seconds and \
|
if self.log_sleep_time_seconds and \
|
||||||
|
@ -310,6 +310,7 @@ def get_account_info(env, app, swift_source=None):
|
|||||||
|
|
||||||
This call bypasses auth. Success does not imply that the request has
|
This call bypasses auth. Success does not imply that the request has
|
||||||
authorization to the account.
|
authorization to the account.
|
||||||
|
:raises ValueError: when path can't be split(path, 2, 4)
|
||||||
"""
|
"""
|
||||||
(version, account, _junk, _junk) = \
|
(version, account, _junk, _junk) = \
|
||||||
split_path(env['PATH_INFO'], 2, 4, True)
|
split_path(env['PATH_INFO'], 2, 4, True)
|
||||||
|
@ -208,25 +208,16 @@ class TestRateLimit(unittest.TestCase):
|
|||||||
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
|
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
|
||||||
req, 'a', 'c', 'o')), 1)
|
req, 'a', 'c', 'o')), 1)
|
||||||
|
|
||||||
def get_fake_ratelimit(*args, **kwargs):
|
req.method = 'PUT'
|
||||||
return {'sysmeta': {'global-write-ratelimit': 10}}
|
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
|
||||||
|
req, 'a', 'c', None, global_ratelimit=10)), 2)
|
||||||
|
self.assertEquals(the_app.get_ratelimitable_key_tuples(
|
||||||
|
req, 'a', 'c', None, global_ratelimit=10)[1],
|
||||||
|
('ratelimit/global-write/a', 10))
|
||||||
|
|
||||||
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
req.method = 'PUT'
|
||||||
get_fake_ratelimit):
|
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
|
||||||
req.method = 'PUT'
|
req, 'a', 'c', None, global_ratelimit='notafloat')), 1)
|
||||||
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
|
|
||||||
req, 'a', 'c', None)), 2)
|
|
||||||
self.assertEquals(the_app.get_ratelimitable_key_tuples(
|
|
||||||
req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10))
|
|
||||||
|
|
||||||
def get_fake_ratelimit(*args, **kwargs):
|
|
||||||
return {'sysmeta': {'global-write-ratelimit': 'notafloat'}}
|
|
||||||
|
|
||||||
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
|
||||||
get_fake_ratelimit):
|
|
||||||
req.method = 'PUT'
|
|
||||||
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
|
|
||||||
req, 'a', 'c', None)), 1)
|
|
||||||
|
|
||||||
def test_memcached_container_info_dict(self):
|
def test_memcached_container_info_dict(self):
|
||||||
mdict = headers_to_container_info({'x-container-object-count': '45'})
|
mdict = headers_to_container_info({'x-container-object-count': '45'})
|
||||||
@ -291,7 +282,26 @@ class TestRateLimit(unittest.TestCase):
|
|||||||
self._run(make_app_call, num_calls, current_rate, check_time=False)
|
self._run(make_app_call, num_calls, current_rate, check_time=False)
|
||||||
self.assertEquals(round(time.time() - begin, 1), 9.8)
|
self.assertEquals(round(time.time() - begin, 1), 9.8)
|
||||||
|
|
||||||
def test_ratelimit_whitelist(self):
|
def test_ratelimit_old_white_black_list(self):
|
||||||
|
global time_ticker
|
||||||
|
current_rate = 2
|
||||||
|
conf_dict = {'account_ratelimit': current_rate,
|
||||||
|
'max_sleep_time_seconds': 2,
|
||||||
|
'account_whitelist': 'a',
|
||||||
|
'account_blacklist': 'b'}
|
||||||
|
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
|
||||||
|
req = Request.blank('/')
|
||||||
|
with mock.patch.object(self.test_ratelimit,
|
||||||
|
'memcache_client', FakeMemcache()):
|
||||||
|
self.assertEqual(
|
||||||
|
self.test_ratelimit.handle_ratelimit(req, 'a', 'c', 'o'),
|
||||||
|
None)
|
||||||
|
self.assertEqual(
|
||||||
|
self.test_ratelimit.handle_ratelimit(
|
||||||
|
req, 'b', 'c', 'o').status_int,
|
||||||
|
497)
|
||||||
|
|
||||||
|
def test_ratelimit_whitelist_sysmeta(self):
|
||||||
global time_ticker
|
global time_ticker
|
||||||
current_rate = 2
|
current_rate = 2
|
||||||
conf_dict = {'account_ratelimit': current_rate,
|
conf_dict = {'account_ratelimit': current_rate,
|
||||||
@ -312,18 +322,25 @@ class TestRateLimit(unittest.TestCase):
|
|||||||
def run(self):
|
def run(self):
|
||||||
self.result = self.parent.test_ratelimit(req.environ,
|
self.result = self.parent.test_ratelimit(req.environ,
|
||||||
start_response)
|
start_response)
|
||||||
nt = 5
|
|
||||||
threads = []
|
def get_fake_ratelimit(*args, **kwargs):
|
||||||
for i in range(nt):
|
return {'sysmeta': {'global-write-ratelimit': 'WHITELIST'}}
|
||||||
rc = rate_caller(self)
|
|
||||||
rc.start()
|
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
||||||
threads.append(rc)
|
get_fake_ratelimit):
|
||||||
for thread in threads:
|
nt = 5
|
||||||
thread.join()
|
threads = []
|
||||||
the_498s = [
|
for i in range(nt):
|
||||||
t for t in threads if ''.join(t.result).startswith('Slow down')]
|
rc = rate_caller(self)
|
||||||
self.assertEquals(len(the_498s), 0)
|
rc.start()
|
||||||
self.assertEquals(time_ticker, 0)
|
threads.append(rc)
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
the_498s = [
|
||||||
|
t for t in threads
|
||||||
|
if ''.join(t.result).startswith('Slow down')]
|
||||||
|
self.assertEquals(len(the_498s), 0)
|
||||||
|
self.assertEquals(time_ticker, 0)
|
||||||
|
|
||||||
def test_ratelimit_blacklist(self):
|
def test_ratelimit_blacklist(self):
|
||||||
global time_ticker
|
global time_ticker
|
||||||
@ -348,18 +365,25 @@ class TestRateLimit(unittest.TestCase):
|
|||||||
def run(self):
|
def run(self):
|
||||||
self.result = self.parent.test_ratelimit(req.environ,
|
self.result = self.parent.test_ratelimit(req.environ,
|
||||||
start_response)
|
start_response)
|
||||||
nt = 5
|
|
||||||
threads = []
|
def get_fake_ratelimit(*args, **kwargs):
|
||||||
for i in range(nt):
|
return {'sysmeta': {'global-write-ratelimit': 'BLACKLIST'}}
|
||||||
rc = rate_caller(self)
|
|
||||||
rc.start()
|
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
||||||
threads.append(rc)
|
get_fake_ratelimit):
|
||||||
for thread in threads:
|
nt = 5
|
||||||
thread.join()
|
threads = []
|
||||||
the_497s = [
|
for i in range(nt):
|
||||||
t for t in threads if ''.join(t.result).startswith('Your account')]
|
rc = rate_caller(self)
|
||||||
self.assertEquals(len(the_497s), 5)
|
rc.start()
|
||||||
self.assertEquals(time_ticker, 0)
|
threads.append(rc)
|
||||||
|
for thread in threads:
|
||||||
|
thread.join()
|
||||||
|
the_497s = [
|
||||||
|
t for t in threads
|
||||||
|
if ''.join(t.result).startswith('Your account')]
|
||||||
|
self.assertEquals(len(the_497s), 5)
|
||||||
|
self.assertEquals(time_ticker, 0)
|
||||||
|
|
||||||
def test_ratelimit_max_rate_double(self):
|
def test_ratelimit_max_rate_double(self):
|
||||||
global time_ticker
|
global time_ticker
|
||||||
@ -443,28 +467,30 @@ class TestRateLimit(unittest.TestCase):
|
|||||||
get_container_memcache_key('a', 'c'),
|
get_container_memcache_key('a', 'c'),
|
||||||
{'container_size': 1})
|
{'container_size': 1})
|
||||||
|
|
||||||
time_override = [0, 0, 0, 0, None]
|
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
|
||||||
# simulates 4 requests coming in at same time, then sleeping
|
lambda *args, **kwargs: {}):
|
||||||
r = self.test_ratelimit(req.environ, start_response)
|
time_override = [0, 0, 0, 0, None]
|
||||||
mock_sleep(.1)
|
# simulates 4 requests coming in at same time, then sleeping
|
||||||
r = self.test_ratelimit(req.environ, start_response)
|
r = self.test_ratelimit(req.environ, start_response)
|
||||||
mock_sleep(.1)
|
mock_sleep(.1)
|
||||||
r = self.test_ratelimit(req.environ, start_response)
|
r = self.test_ratelimit(req.environ, start_response)
|
||||||
self.assertEquals(r[0], 'Slow down')
|
mock_sleep(.1)
|
||||||
mock_sleep(.1)
|
r = self.test_ratelimit(req.environ, start_response)
|
||||||
r = self.test_ratelimit(req.environ, start_response)
|
self.assertEquals(r[0], 'Slow down')
|
||||||
self.assertEquals(r[0], 'Slow down')
|
mock_sleep(.1)
|
||||||
mock_sleep(.1)
|
r = self.test_ratelimit(req.environ, start_response)
|
||||||
r = self.test_ratelimit(req.environ, start_response)
|
self.assertEquals(r[0], 'Slow down')
|
||||||
self.assertEquals(r[0], '204 No Content')
|
mock_sleep(.1)
|
||||||
mc = self.test_ratelimit.memcache_client
|
r = self.test_ratelimit(req.environ, start_response)
|
||||||
try:
|
self.assertEquals(r[0], '204 No Content')
|
||||||
self.test_ratelimit.memcache_client = None
|
mc = self.test_ratelimit.memcache_client
|
||||||
self.assertEquals(
|
try:
|
||||||
self.test_ratelimit.handle_ratelimit(req, 'n', 'c', None),
|
self.test_ratelimit.memcache_client = None
|
||||||
None)
|
self.assertEquals(
|
||||||
finally:
|
self.test_ratelimit.handle_ratelimit(req, 'n', 'c', None),
|
||||||
self.test_ratelimit.memcache_client = mc
|
None)
|
||||||
|
finally:
|
||||||
|
self.test_ratelimit.memcache_client = mc
|
||||||
|
|
||||||
def test_ratelimit_max_rate_multiple_acc(self):
|
def test_ratelimit_max_rate_multiple_acc(self):
|
||||||
num_calls = 4
|
num_calls = 4
|
||||||
|
Loading…
x
Reference in New Issue
Block a user