From db4689689a253f359bac6d277f95f8972c22025c Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 6 Oct 2010 13:11:58 -0700 Subject: [PATCH] minor bug fixes- pre ratelimit caching of container size --- .functests | 2 +- etc/proxy-server.conf-sample | 14 ++++++++++++-- setup.py | 2 +- swift/common/middleware/ratelimit.py | 20 ++++++++++---------- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.functests b/.functests index b6bf4e0a70..86abce360a 100755 --- a/.functests +++ b/.functests @@ -1,4 +1,4 @@ #!/bin/bash -python test/functional/tests.py +nosetests test/functional --exe nosetests test/functionalnosetests --exe diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index c566590554..88001091fc 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -59,12 +59,22 @@ use = egg:swift#memcache [filter:ratelimit] use = egg:swift#ratelimit +# clock_accuracy should represent how accurate the proxy servers' system clocks +# are with each other. 1000 means that all the proxies' clock are accurate to +# each other within 1 millisecond. No ratelimit should be higher than the +# clock accuracy. +clock_accuracy = 1000 +max_sleep_time_seconds = 60 + account_ratelimit = 200 +# these are comma separated lists of account names account_whitelist = a,b # account_blacklist = # with container_limit_x = r -# for containers of size x limit requests per second to r +# for containers of size x limit requests per second to r. The container +# rate will be linearly interpolated from the values given. With the values +# below, a container of size 5 will get a rate of 75. container_limit_0 = 100 container_limit_10 = 50 -container_limit_50 = 10 +container_limit_50 = 20 diff --git a/setup.py b/setup.py index 4db007ba10..7a8898d643 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ setup( 'auth=swift.common.middleware.auth:filter_factory', 'healthcheck=swift.common.middleware.healthcheck:filter_factory', 'memcache=swift.common.middleware.memcache:filter_factory', -# 'ratelimit=swift.common.middeware.ratelimit:filter_factory', + 'ratelimit=swift.common.middleware.ratelimit:filter_factory', ], }, ) diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 11ad01dcae..90cb1f33f7 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -35,9 +35,9 @@ class RateLimitMiddleware(object): else: self.logger = logger - self.account_rate_limit = float(conf.get('account_ratelimit', 1))#200.0)) - self.max_sleep_time_seconds = int(conf.get('max_sleep_time_seconds', - 2))#60)) + self.account_rate_limit = float(conf.get('account_ratelimit', 200.0)) + self.max_sleep_time_seconds = float(conf.get('max_sleep_time_seconds', + 60)) self.clock_accuracy = int(conf.get('clock_accuracy', 1000)) self.rate_limit_whitelist = [acc.strip() for acc in @@ -101,14 +101,14 @@ class RateLimitMiddleware(object): if account_name: keys.append(("ratelimit/%s" % account_name, self.account_rate_limit)) + if account_name and container_name and not obj_name: container_size = None memcache_key = get_container_memcache_key(account_name, container_name) - container_info = self.memcache_client.get(memcache_key) + container_info = self.memcache_client.get(memcache_key) if type(container_info) == dict: - container_size = container_info.get('container_size') - + container_size = int(container_info.get('container_size', 0)) container_rate = self.get_container_maxrate(container_size) if container_rate: keys.append(("ratelimit/%s/%s" % (account_name, @@ -139,7 +139,7 @@ class RateLimitMiddleware(object): if max_sleep_m - need_to_sleep_m <= self.clock_accuracy * 0.01: # make it accurate to 1% of clock accuracy # treat as no-op decrement time - self.memcache_client.decr(key, delta=time_per_request_m) + self.memcache_client.incr(key, delta=-time_per_request_m) raise MaxSleepTimeHit("Max Sleep Time Exceeded: %s" % need_to_sleep_m) @@ -161,9 +161,8 @@ class RateLimitMiddleware(object): container_name, obj_name): try: - need_to_sleep = self._get_sleep_time(key, - max_rate) - if need_to_sleep > 0: + need_to_sleep = self._get_sleep_time(key, max_rate) + if need_to_sleep > 0: time.sleep(need_to_sleep) except MaxSleepTimeHit, e: @@ -177,6 +176,7 @@ class RateLimitMiddleware(object): def __call__(self, env, start_response, name=None): + #TODO : David- get rid of the name thing- used for debugging req = Request(env) if self.memcache_client is None: self.memcache_client = cache_from_env(env)