openstack/swift
Code Issues Proposed changes

checking for memcache incase not setup

Browse Source
This commit is contained in:
David Goetz
2010-11-08 12:39:21 -08:00
parent 8fd79870a3
commit be921452ac
3 changed files with 63 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
swift/common/middleware/ratelimit.py
View File

@@ -205,6 +205,10 @@ class RateLimitMiddleware(object):
req = Request(env) req = Request(env)
if self.memcache_client is None: if self.memcache_client is None:
self.memcache_client = cache_from_env(env) self.memcache_client = cache_from_env(env)
if not self.memcache_client:
self.logger.warning(
'Warning: Cannot ratelimit without a memcached client')
return self.app(env, start_response)
try: try:
version, account, container, obj = split_path(req.path, 1, 4, True) version, account, container, obj = split_path(req.path, 1, 4, True)
except ValueError: except ValueError:

72
swift/proxy/server.py
View File

@@ -179,7 +179,7 @@ class Controller(object):
path = '/%s' % account path = '/%s' % account
cache_key = 'account%s' % path cache_key = 'account%s' % path
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache.get(cache_key): if self.app.memcache and self.app.memcache.get(cache_key):
return partition, nodes return partition, nodes
result_code = 0 result_code = 0
attempts_left = self.app.account_ring.replica_count attempts_left = self.app.account_ring.replica_count
@@ -214,7 +214,9 @@ class Controller(object):
cache_timeout = self.app.recheck_account_existence cache_timeout = self.app.recheck_account_existence
else: else:
cache_timeout = self.app.recheck_account_existence * 0.1 cache_timeout = self.app.recheck_account_existence * 0.1
self.app.memcache.set(cache_key, result_code, timeout=cache_timeout) if self.app.memcache:
self.app.memcache.set(cache_key, result_code,
timeout=cache_timeout)
if result_code == 200: if result_code == 200:
return partition, nodes return partition, nodes
return (None, None) return (None, None)
@@ -234,14 +236,16 @@ class Controller(object):
partition, nodes = self.app.container_ring.get_nodes( partition, nodes = self.app.container_ring.get_nodes(
account, container) account, container)
path = '/%s/%s' % (account, container) path = '/%s/%s' % (account, container)
cache_key = get_container_memcache_key(account, container) cache_key = None
cache_value = self.app.memcache.get(cache_key) if self.app.memcache:
if isinstance(cache_value, dict): cache_key = get_container_memcache_key(account, container)
status = cache_value['status'] cache_value = self.app.memcache.get(cache_key)
read_acl = cache_value['read_acl'] if isinstance(cache_value, dict):
write_acl = cache_value['write_acl'] status = cache_value['status']
if status // 100 == 2: read_acl = cache_value['read_acl']
return partition, nodes, read_acl, write_acl write_acl = cache_value['write_acl']
if status // 100 == 2:
return partition, nodes, read_acl, write_acl
if not self.account_info(account)[1]: if not self.account_info(account)[1]:
return (None, None, None, None) return (None, None, None, None)
result_code = 0 result_code = 0
@@ -284,11 +288,13 @@ class Controller(object):
cache_timeout = self.app.recheck_container_existence cache_timeout = self.app.recheck_container_existence
else: else:
cache_timeout = self.app.recheck_container_existence * 0.1 cache_timeout = self.app.recheck_container_existence * 0.1
self.app.memcache.set(cache_key, {'status': result_code, if cache_key and self.app.memcache:
'read_acl': read_acl, self.app.memcache.set(cache_key,
'write_acl': write_acl, {'status': result_code,
'container_size': container_size}, 'read_acl': read_acl,
timeout=cache_timeout) 'write_acl': write_acl,
'container_size': container_size},
timeout=cache_timeout)
if result_code == 200: if result_code == 200:
return partition, nodes, read_acl, write_acl return partition, nodes, read_acl, write_acl
return (None, None, None, None) return (None, None, None, None)
@@ -886,10 +892,11 @@ class ContainerController(Controller):
resp = self.GETorHEAD_base(req, 'Container', part, nodes, resp = self.GETorHEAD_base(req, 'Container', part, nodes,
req.path_info, self.app.container_ring.replica_count) req.path_info, self.app.container_ring.replica_count)
# set the memcache container size for ratelimiting if self.app.memcache:
cache_key = get_container_memcache_key(self.account_name, # set the memcache container size for ratelimiting
self.container_name) cache_key = get_container_memcache_key(self.account_name,
self.app.memcache.set(cache_key, self.container_name)
self.app.memcache.set(cache_key,
{'status': resp.status_int, {'status': resp.status_int,
'read_acl': resp.headers.get('x-container-read'), 'read_acl': resp.headers.get('x-container-read'),
'write_acl': resp.headers.get('x-container-write'), 'write_acl': resp.headers.get('x-container-write'),
@@ -977,9 +984,10 @@ class ContainerController(Controller):
statuses.append(503) statuses.append(503)
reasons.append('') reasons.append('')
bodies.append('') bodies.append('')
cache_key = get_container_memcache_key(self.account_name, if self.app.memcache:
self.container_name) cache_key = get_container_memcache_key(self.account_name,
self.app.memcache.delete(cache_key) self.container_name)
self.app.memcache.delete(cache_key)
return self.best_response(req, statuses, reasons, bodies, return self.best_response(req, statuses, reasons, bodies,
'Container PUT') 'Container PUT')
@@ -1031,9 +1039,10 @@ class ContainerController(Controller):
statuses.append(503) statuses.append(503)
reasons.append('') reasons.append('')
bodies.append('') bodies.append('')
cache_key = get_container_memcache_key(self.account_name, if self.app.memcache:
self.container_name) cache_key = get_container_memcache_key(self.account_name,
self.app.memcache.delete(cache_key) self.container_name)
self.app.memcache.delete(cache_key)
return self.best_response(req, statuses, reasons, bodies, return self.best_response(req, statuses, reasons, bodies,
'Container POST') 'Container POST')
@@ -1087,9 +1096,10 @@ class ContainerController(Controller):
statuses.append(503) statuses.append(503)
reasons.append('') reasons.append('')
bodies.append('') bodies.append('')
cache_key = get_container_memcache_key(self.account_name, if self.app.memcache:
self.container_name) cache_key = get_container_memcache_key(self.account_name,
self.app.memcache.delete(cache_key) self.container_name)
self.app.memcache.delete(cache_key)
resp = self.best_response(req, statuses, reasons, bodies, resp = self.best_response(req, statuses, reasons, bodies,
'Container DELETE') 'Container DELETE')
if 200 <= resp.status_int <= 299: if 200 <= resp.status_int <= 299:
@@ -1169,7 +1179,8 @@ class AccountController(Controller):
statuses.append(503) statuses.append(503)
reasons.append('') reasons.append('')
bodies.append('') bodies.append('')
self.app.memcache.delete('account%s' % req.path_info.rstrip('/')) if self.app.memcache:
self.app.memcache.delete('account%s' % req.path_info.rstrip('/'))
return self.best_response(req, statuses, reasons, bodies, return self.best_response(req, statuses, reasons, bodies,
'Account PUT') 'Account PUT')
@@ -1216,7 +1227,8 @@ class AccountController(Controller):
statuses.append(503) statuses.append(503)
reasons.append('') reasons.append('')
bodies.append('') bodies.append('')
self.app.memcache.delete('account%s' % req.path_info.rstrip('/')) if self.app.memcache:
self.app.memcache.delete('account%s' % req.path_info.rstrip('/'))
return self.best_response(req, statuses, reasons, bodies, return self.best_response(req, statuses, reasons, bodies,
'Account POST') 'Account POST')

17
test/unit/common/middleware/test_ratelimit.py
View File

@@ -101,6 +101,9 @@ class FakeLogger(object):
def info(self, msg): def info(self, msg):
pass pass
def warning(self, msg):
pass
def start_response(*args): def start_response(*args):
pass pass
@@ -386,6 +389,20 @@ class TestRateLimit(unittest.TestCase):
resp = rate_mid.__call__(env, a_callable()) resp = rate_mid.__call__(env, a_callable())
self.assert_('404 Not Found' in resp[0]) self.assert_('404 Not Found' in resp[0])
def test_no_memcache(self):
current_rate = 13
num_calls = 5
conf_dict = {'account_ratelimit': current_rate}
self.test_ratelimit = ratelimit.filter_factory(conf_dict)(FakeApp())
ratelimit.http_connect = mock_http_connect(204)
req = Request.blank('/v/a')
req.environ['swift.cache'] = None
make_app_call = lambda: self.test_ratelimit(req.environ,
start_response)
self._run(make_app_call, num_calls, current_rate)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()