openstack/swift
Code Issues Proposed changes

Add a way to ratelimit all writes to an account

Browse Source 
This is in case a cluster gets a problem user who has distributed the
writes to a bunch of containers but is just taking too much of the
cluster's resources.

Change-Id: Ibd2ffd0e911463a432117b478585b9f8bc4a2495
This commit is contained in:
David Goetz
2014-01-29 08:33:27 -08:00
parent 9034558f0b
commit b89ac55c05
3 changed files with 151 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
doc/source/ratelimit.rst
View File

@@ -81,3 +81,20 @@ Container Size Rate Limit
================ ============ ================ ============
-----------------------------
Account Specific Ratelimiting
-----------------------------
The above ratelimiting is to prevent the "many writes to a single container"
bottleneck from causing a problem. There could also be a problem where a single
account is just using too much of the cluster's resources. In this case, the
container ratelimits may not help because the customer could be doing thousands
of reqs/sec to distributed containers each getting a small fraction of the
total so those limits would never trigger. If a system adminstrator notices
this, he/she can set the X-Account-Sysmeta-Global-Write-Ratelimit on an account
and that will limit the total number of write requests (PUT, POST, DELETE,
COPY) that account can do for the whole account. This limit will be in addition
to the applicable account/container limits from above. This header will be
hidden from the user, because of the gatekeeper middleware, and can only be set
using a direct client to the account nodes. It accepts a float value and will
only limit requests if the value is > 0.

29
swift/common/middleware/ratelimit.py
View File

@@ -18,7 +18,8 @@ from swift import gettext_ as _
import eventlet import eventlet
from swift.common.utils import cache_from_env, get_logger, register_swift_info from swift.common.utils import cache_from_env, get_logger, register_swift_info
from swift.proxy.controllers.base import get_container_memcache_key from swift.proxy.controllers.base import get_container_memcache_key, \
get_account_info
from swift.common.memcached import MemcacheConnectionError from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Request, Response from swift.common.swob import Request, Response
@@ -117,13 +118,13 @@ class RateLimitMiddleware(object):
'object_count', container_info.get('container_size', 0)) 'object_count', container_info.get('container_size', 0))
return rv return rv
def get_ratelimitable_key_tuples(self, req_method, account_name, def get_ratelimitable_key_tuples(self, req, account_name,
container_name=None, obj_name=None): container_name=None, obj_name=None):
""" """
Returns a list of key (used in memcache), ratelimit tuples. Keys Returns a list of key (used in memcache), ratelimit tuples. Keys
should be checked in order. should be checked in order.
:param req_method: HTTP method :param req: swob request
:param account_name: account name from path :param account_name: account name from path
:param container_name: container name from path :param container_name: container name from path
:param obj_name: object name from path :param obj_name: object name from path
@@ -132,12 +133,12 @@ class RateLimitMiddleware(object):
# COPYs are not limited # COPYs are not limited
if self.account_ratelimit and \ if self.account_ratelimit and \
account_name and container_name and not obj_name and \ account_name and container_name and not obj_name and \
req_method in ('PUT', 'DELETE'): req.method in ('PUT', 'DELETE'):
keys.append(("ratelimit/%s" % account_name, keys.append(("ratelimit/%s" % account_name,
self.account_ratelimit)) self.account_ratelimit))
if account_name and container_name and obj_name and \ if account_name and container_name and obj_name and \
req_method in ('PUT', 'DELETE', 'POST'): req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
container_size = self.get_container_size( container_size = self.get_container_size(
account_name, container_name) account_name, container_name)
container_rate = get_maxrate( container_rate = get_maxrate(
@@ -148,7 +149,7 @@ class RateLimitMiddleware(object):
container_rate)) container_rate))
if account_name and container_name and not obj_name and \ if account_name and container_name and not obj_name and \
req_method == 'GET': req.method == 'GET':
container_size = self.get_container_size( container_size = self.get_container_size(
account_name, container_name) account_name, container_name)
container_rate = get_maxrate( container_rate = get_maxrate(
@@ -158,6 +159,20 @@ class RateLimitMiddleware(object):
"ratelimit_listing/%s/%s" % (account_name, container_name), "ratelimit_listing/%s/%s" % (account_name, container_name),
container_rate)) container_rate))
if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
account_info = get_account_info(req.environ, self.app)
account_global_ratelimit = \
account_info.get('sysmeta', {}).get('global-write-ratelimit')
if account_global_ratelimit:
try:
account_global_ratelimit = float(account_global_ratelimit)
if account_global_ratelimit > 0:
keys.append((
"ratelimit/global-write/%s" % account_name,
account_global_ratelimit))
except ValueError:
pass
return keys return keys
def _get_sleep_time(self, key, max_rate): def _get_sleep_time(self, key, max_rate):
@@ -218,7 +233,7 @@ class RateLimitMiddleware(object):
if account_name in self.ratelimit_whitelist: if account_name in self.ratelimit_whitelist:
return None return None
for key, max_rate in self.get_ratelimitable_key_tuples( for key, max_rate in self.get_ratelimitable_key_tuples(
req.method, account_name, container_name=container_name, req, account_name, container_name=container_name,
obj_name=obj_name): obj_name=obj_name):
try: try:
need_to_sleep = self._get_sleep_time(key, max_rate) need_to_sleep = self._get_sleep_time(key, max_rate)

176
test/unit/common/middleware/test_ratelimit.py
View File

@@ -16,6 +16,7 @@
import unittest import unittest
import time import time
import eventlet import eventlet
import mock
from contextlib import contextmanager from contextlib import contextmanager
from threading import Thread from threading import Thread
@@ -194,16 +195,45 @@ class TestRateLimit(unittest.TestCase):
the_app = ratelimit.RateLimitMiddleware(None, conf_dict, the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
logger=FakeLogger()) logger=FakeLogger())
the_app.memcache_client = fake_memcache the_app.memcache_client = fake_memcache
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req = lambda: None
'DELETE', 'a', None, None)), 0) req.environ = {}
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( with mock.patch('swift.common.middleware.ratelimit.get_account_info',
'PUT', 'a', 'c', None)), 1) lambda *args, **kwargs: {}):
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req.method = 'DELETE'
'DELETE', 'a', 'c', None)), 1) self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( req, 'a', None, None)), 0)
'GET', 'a', 'c', 'o')), 0) req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples( self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
'PUT', 'a', 'c', 'o')), 1) req, 'a', 'c', None)), 1)
req.method = 'DELETE'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)), 1)
req.method = 'GET'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', 'o')), 0)
req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', 'o')), 1)
def get_fake_ratelimit(*args, **kwargs):
return {'sysmeta': {'global-write-ratelimit': 10}}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
get_fake_ratelimit):
req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)), 2)
self.assertEquals(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10))
def get_fake_ratelimit(*args, **kwargs):
return {'sysmeta': {'global-write-ratelimit': 'notafloat'}}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
get_fake_ratelimit):
req.method = 'PUT'
self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
req, 'a', 'c', None)), 1)
def test_memcached_container_info_dict(self): def test_memcached_container_info_dict(self):
mdict = headers_to_container_info({'x-container-object-count': '45'}) mdict = headers_to_container_info({'x-container-object-count': '45'})
@@ -219,8 +249,13 @@ class TestRateLimit(unittest.TestCase):
the_app = ratelimit.RateLimitMiddleware(None, conf_dict, the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
logger=FakeLogger()) logger=FakeLogger())
the_app.memcache_client = fake_memcache the_app.memcache_client = fake_memcache
tuples = the_app.get_ratelimitable_key_tuples('PUT', 'a', 'c', 'o') req = lambda: None
self.assertEquals(tuples, [('ratelimit/a/c', 200.0)]) req.method = 'PUT'
req.environ = {}
with mock.patch('swift.common.middleware.ratelimit.get_account_info',
lambda *args, **kwargs: {}):
tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')
self.assertEquals(tuples, [('ratelimit/a/c', 200.0)])
def test_account_ratelimit(self): def test_account_ratelimit(self):
current_rate = 5 current_rate = 5
@@ -228,18 +263,20 @@ class TestRateLimit(unittest.TestCase):
conf_dict = {'account_ratelimit': current_rate} conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp()) self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204) ratelimit.http_connect = mock_http_connect(204)
for meth, exp_time in [ with mock.patch('swift.common.middleware.ratelimit.get_account_info',
('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]: lambda *args, **kwargs: {}):
req = Request.blank('/v/a%s/c' % meth) for meth, exp_time in [
req.method = meth ('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]:
req.environ['swift.cache'] = FakeMemcache() req = Request.blank('/v/a%s/c' % meth)
make_app_call = lambda: self.test_ratelimit(req.environ, req.method = meth
start_response) req.environ['swift.cache'] = FakeMemcache()
begin = time.time() make_app_call = lambda: self.test_ratelimit(req.environ,
self._run(make_app_call, num_calls, current_rate, start_response)
check_time=bool(exp_time)) begin = time.time()
self.assertEquals(round(time.time() - begin, 1), exp_time) self._run(make_app_call, num_calls, current_rate,
self._reset_time() check_time=bool(exp_time))
self.assertEquals(round(time.time() - begin, 1), exp_time)
self._reset_time()
def test_ratelimit_set_incr(self): def test_ratelimit_set_incr(self):
current_rate = 5 current_rate = 5
@@ -254,8 +291,10 @@ class TestRateLimit(unittest.TestCase):
make_app_call = lambda: self.test_ratelimit(req.environ, make_app_call = lambda: self.test_ratelimit(req.environ,
start_response) start_response)
begin = time.time() begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False) with mock.patch('swift.common.middleware.ratelimit.get_account_info',
self.assertEquals(round(time.time() - begin, 1), 9.8) lambda *args, **kwargs: {}):
self._run(make_app_call, num_calls, current_rate, check_time=False)
self.assertEquals(round(time.time() - begin, 1), 9.8)
def test_ratelimit_whitelist(self): def test_ratelimit_whitelist(self):
global time_ticker global time_ticker
@@ -342,18 +381,20 @@ class TestRateLimit(unittest.TestCase):
time_override = [0, 0, 0, 0, None] time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping # simulates 4 requests coming in at same time, then sleeping
r = self.test_ratelimit(req.environ, start_response) with mock.patch('swift.common.middleware.ratelimit.get_account_info',
mock_sleep(.1) lambda *args, **kwargs: {}):
r = self.test_ratelimit(req.environ, start_response) r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1) mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response) r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down') mock_sleep(.1)
mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response)
r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down')
self.assertEquals(r[0], 'Slow down') mock_sleep(.1)
mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response)
r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down')
self.assertEquals(r[0], '204 No Content') mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_double_container(self): def test_ratelimit_max_rate_double_container(self):
global time_ticker global time_ticker
@@ -374,18 +415,20 @@ class TestRateLimit(unittest.TestCase):
time_override = [0, 0, 0, 0, None] time_override = [0, 0, 0, 0, None]
# simulates 4 requests coming in at same time, then sleeping # simulates 4 requests coming in at same time, then sleeping
r = self.test_ratelimit(req.environ, start_response) with mock.patch('swift.common.middleware.ratelimit.get_account_info',
mock_sleep(.1) lambda *args, **kwargs: {}):
r = self.test_ratelimit(req.environ, start_response) r = self.test_ratelimit(req.environ, start_response)
mock_sleep(.1) mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response) r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], 'Slow down') mock_sleep(.1)
mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response)
r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down')
self.assertEquals(r[0], 'Slow down') mock_sleep(.1)
mock_sleep(.1) r = self.test_ratelimit(req.environ, start_response)
r = self.test_ratelimit(req.environ, start_response) self.assertEquals(r[0], 'Slow down')
self.assertEquals(r[0], '204 No Content') mock_sleep(.1)
r = self.test_ratelimit(req.environ, start_response)
self.assertEquals(r[0], '204 No Content')
def test_ratelimit_max_rate_double_container_listing(self): def test_ratelimit_max_rate_double_container_listing(self):
global time_ticker global time_ticker
@@ -431,6 +474,7 @@ class TestRateLimit(unittest.TestCase):
the_app.memcache_client = fake_memcache the_app.memcache_client = fake_memcache
req = lambda: None req = lambda: None
req.method = 'PUT' req.method = 'PUT'
req.environ = {}
class rate_caller(Thread): class rate_caller(Thread):
@@ -443,18 +487,20 @@ class TestRateLimit(unittest.TestCase):
self.result = the_app.handle_ratelimit(req, self.myname, self.result = the_app.handle_ratelimit(req, self.myname,
'c', None) 'c', None)
nt = 15 with mock.patch('swift.common.middleware.ratelimit.get_account_info',
begin = time.time() lambda *args, **kwargs: {}):
threads = [] nt = 15
for i in range(nt): begin = time.time()
rc = rate_caller('a%s' % i) threads = []
rc.start() for i in range(nt):
threads.append(rc) rc = rate_caller('a%s' % i)
for thread in threads: rc.start()
thread.join() threads.append(rc)
for thread in threads:
thread.join()
time_took = time.time() - begin time_took = time.time() - begin
self.assertEquals(1.5, round(time_took, 1)) self.assertEquals(1.5, round(time_took, 1))
def test_call_invalid_path(self): def test_call_invalid_path(self):
env = {'REQUEST_METHOD': 'GET', env = {'REQUEST_METHOD': 'GET',
@@ -504,9 +550,11 @@ class TestRateLimit(unittest.TestCase):
make_app_call = lambda: self.test_ratelimit(req.environ, make_app_call = lambda: self.test_ratelimit(req.environ,
start_response) start_response)
begin = time.time() begin = time.time()
self._run(make_app_call, num_calls, current_rate, check_time=False) with mock.patch('swift.common.middleware.ratelimit.get_account_info',
time_took = time.time() - begin lambda *args, **kwargs: {}):
self.assertEquals(round(time_took, 1), 0) # no memcache, no limiting self._run(make_app_call, num_calls, current_rate, check_time=False)
time_took = time.time() - begin
self.assertEquals(round(time_took, 1), 0) # no memcache, no limit
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()