Add a way to ratelimit all writes to an account

This is in case a cluster gets a problem user who has distributed the
writes to a bunch of containers but is just taking too much of the
cluster's resources.

Change-Id: Ibd2ffd0e911463a432117b478585b9f8bc4a2495

doc/source/ratelimit.rst

@@ -81,3 +81,20 @@ Container Size      Rate Limit
 ================    ============
 
 
+-----------------------------
+Account Specific Ratelimiting
+-----------------------------
+
+The above ratelimiting is to prevent the "many writes to a single container"
+bottleneck from causing a problem. There could also be a problem where a single
+account is just using too much of the cluster's resources.  In this case, the
+container ratelimits may not help because the customer could be doing thousands
+of reqs/sec to distributed containers each getting a small fraction of the
+total so those limits would never trigger. If a system adminstrator notices
+this, he/she can set the X-Account-Sysmeta-Global-Write-Ratelimit on an account
+and that will limit the total number of write requests (PUT, POST, DELETE,
+COPY) that account can do for the whole account. This limit will be in addition
+to the applicable account/container limits from above. This header will be
+hidden from the user, because of the gatekeeper middleware, and can only be set
+using a direct client to the account nodes. It accepts a float value and will
+only limit requests if the value is > 0.

swift/common/middleware/ratelimit.py

@@ -18,7 +18,8 @@ from swift import gettext_ as _
 import eventlet
 
 from swift.common.utils import cache_from_env, get_logger, register_swift_info
-from swift.proxy.controllers.base import get_container_memcache_key
+from swift.proxy.controllers.base import get_container_memcache_key, \
+    get_account_info
 from swift.common.memcached import MemcacheConnectionError
 from swift.common.swob import Request, Response
 
@@ -117,13 +118,13 @@ class RateLimitMiddleware(object):
                 'object_count', container_info.get('container_size', 0))
         return rv
 
-    def get_ratelimitable_key_tuples(self, req_method, account_name,
+    def get_ratelimitable_key_tuples(self, req, account_name,
                                      container_name=None, obj_name=None):
         """
         Returns a list of key (used in memcache), ratelimit tuples. Keys
         should be checked in order.
 
-        :param req_method: HTTP method
+        :param req: swob request
         :param account_name: account name from path
         :param container_name: container name from path
         :param obj_name: object name from path
@@ -132,12 +133,12 @@ class RateLimitMiddleware(object):
         # COPYs are not limited
         if self.account_ratelimit and \
                 account_name and container_name and not obj_name and \
-                req_method in ('PUT', 'DELETE'):
+                req.method in ('PUT', 'DELETE'):
             keys.append(("ratelimit/%s" % account_name,
                          self.account_ratelimit))
 
         if account_name and container_name and obj_name and \
-                req_method in ('PUT', 'DELETE', 'POST'):
+                req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
             container_size = self.get_container_size(
                 account_name, container_name)
             container_rate = get_maxrate(
@@ -148,7 +149,7 @@ class RateLimitMiddleware(object):
                     container_rate))
 
         if account_name and container_name and not obj_name and \
-                req_method == 'GET':
+                req.method == 'GET':
             container_size = self.get_container_size(
                 account_name, container_name)
             container_rate = get_maxrate(
@@ -158,6 +159,20 @@ class RateLimitMiddleware(object):
                     "ratelimit_listing/%s/%s" % (account_name, container_name),
                     container_rate))
 
+        if account_name and req.method in ('PUT', 'DELETE', 'POST', 'COPY'):
+            account_info = get_account_info(req.environ, self.app)
+            account_global_ratelimit = \
+                account_info.get('sysmeta', {}).get('global-write-ratelimit')
+            if account_global_ratelimit:
+                try:
+                    account_global_ratelimit = float(account_global_ratelimit)
+                    if account_global_ratelimit > 0:
+                        keys.append((
+                            "ratelimit/global-write/%s" % account_name,
+                            account_global_ratelimit))
+                except ValueError:
+                    pass
+
         return keys
 
     def _get_sleep_time(self, key, max_rate):
@@ -218,7 +233,7 @@ class RateLimitMiddleware(object):
         if account_name in self.ratelimit_whitelist:
             return None
         for key, max_rate in self.get_ratelimitable_key_tuples(
-                req.method, account_name, container_name=container_name,
+                req, account_name, container_name=container_name,
                 obj_name=obj_name):
             try:
                 need_to_sleep = self._get_sleep_time(key, max_rate)

test/unit/common/middleware/test_ratelimit.py

@@ -16,6 +16,7 @@
 import unittest
 import time
 import eventlet
+import mock
 from contextlib import contextmanager
 from threading import Thread
 
@@ -194,16 +195,45 @@ class TestRateLimit(unittest.TestCase):
         the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
                                                 logger=FakeLogger())
         the_app.memcache_client = fake_memcache
+        req = lambda: None
+        req.environ = {}
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
+            req.method = 'DELETE'
             self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
-            'DELETE', 'a', None, None)), 0)
+                req, 'a', None, None)), 0)
+            req.method = 'PUT'
             self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
-            'PUT', 'a', 'c', None)), 1)
+                req, 'a', 'c', None)), 1)
+            req.method = 'DELETE'
             self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
-            'DELETE', 'a', 'c', None)), 1)
+                req, 'a', 'c', None)), 1)
+            req.method = 'GET'
             self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
-            'GET', 'a', 'c', 'o')), 0)
+                req, 'a', 'c', 'o')), 0)
+            req.method = 'PUT'
             self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
-            'PUT', 'a', 'c', 'o')), 1)
+                req, 'a', 'c', 'o')), 1)
+
+        def get_fake_ratelimit(*args, **kwargs):
+            return {'sysmeta': {'global-write-ratelimit': 10}}
+
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        get_fake_ratelimit):
+            req.method = 'PUT'
+            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
+                req, 'a', 'c', None)), 2)
+            self.assertEquals(the_app.get_ratelimitable_key_tuples(
+                req, 'a', 'c', None)[1], ('ratelimit/global-write/a', 10))
+
+        def get_fake_ratelimit(*args, **kwargs):
+            return {'sysmeta': {'global-write-ratelimit': 'notafloat'}}
+
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        get_fake_ratelimit):
+            req.method = 'PUT'
+            self.assertEquals(len(the_app.get_ratelimitable_key_tuples(
+                req, 'a', 'c', None)), 1)
 
     def test_memcached_container_info_dict(self):
         mdict = headers_to_container_info({'x-container-object-count': '45'})
@@ -219,7 +249,12 @@ class TestRateLimit(unittest.TestCase):
         the_app = ratelimit.RateLimitMiddleware(None, conf_dict,
                                                 logger=FakeLogger())
         the_app.memcache_client = fake_memcache
-        tuples = the_app.get_ratelimitable_key_tuples('PUT', 'a', 'c', 'o')
+        req = lambda: None
+        req.method = 'PUT'
+        req.environ = {}
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
+            tuples = the_app.get_ratelimitable_key_tuples(req, 'a', 'c', 'o')
             self.assertEquals(tuples, [('ratelimit/a/c', 200.0)])
 
     def test_account_ratelimit(self):
@@ -228,6 +263,8 @@ class TestRateLimit(unittest.TestCase):
         conf_dict = {'account_ratelimit': current_rate}
         self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
         ratelimit.http_connect = mock_http_connect(204)
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
             for meth, exp_time in [
                     ('DELETE', 9.8), ('GET', 0), ('POST', 0), ('PUT', 9.8)]:
                 req = Request.blank('/v/a%s/c' % meth)
@@ -254,6 +291,8 @@ class TestRateLimit(unittest.TestCase):
         make_app_call = lambda: self.test_ratelimit(req.environ,
                                                     start_response)
         begin = time.time()
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
             self._run(make_app_call, num_calls, current_rate, check_time=False)
             self.assertEquals(round(time.time() - begin, 1), 9.8)
 
@@ -342,6 +381,8 @@ class TestRateLimit(unittest.TestCase):
 
         time_override = [0, 0, 0, 0, None]
         # simulates 4 requests coming in at same time, then sleeping
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
             r = self.test_ratelimit(req.environ, start_response)
             mock_sleep(.1)
             r = self.test_ratelimit(req.environ, start_response)
@@ -374,6 +415,8 @@ class TestRateLimit(unittest.TestCase):
 
         time_override = [0, 0, 0, 0, None]
         # simulates 4 requests coming in at same time, then sleeping
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
             r = self.test_ratelimit(req.environ, start_response)
             mock_sleep(.1)
             r = self.test_ratelimit(req.environ, start_response)
@@ -431,6 +474,7 @@ class TestRateLimit(unittest.TestCase):
         the_app.memcache_client = fake_memcache
         req = lambda: None
         req.method = 'PUT'
+        req.environ = {}
 
         class rate_caller(Thread):
 
@@ -443,6 +487,8 @@ class TestRateLimit(unittest.TestCase):
                     self.result = the_app.handle_ratelimit(req, self.myname,
                                                            'c', None)
 
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
             nt = 15
             begin = time.time()
             threads = []
@@ -504,9 +550,11 @@ class TestRateLimit(unittest.TestCase):
         make_app_call = lambda: self.test_ratelimit(req.environ,
                                                     start_response)
         begin = time.time()
+        with mock.patch('swift.common.middleware.ratelimit.get_account_info',
+                        lambda *args, **kwargs: {}):
             self._run(make_app_call, num_calls, current_rate, check_time=False)
             time_took = time.time() - begin
-        self.assertEquals(round(time_took, 1), 0)  # no memcache, no limiting
+            self.assertEquals(round(time_took, 1), 0)  # no memcache, no limit
 
 if __name__ == '__main__':
     unittest.main()