From 78a150751d00f3db390c00dcd0ac3bd7be23dfe4 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Mon, 29 Nov 2010 10:58:23 -0600 Subject: [PATCH 01/10] changed end_marker to not be inclusive --- swift/common/db.py | 2 +- swift/stats/log_processor.py | 10 +++++----- test/unit/common/test_db.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index c19c62910f..41854407d6 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -979,7 +979,7 @@ class ContainerBroker(DatabaseBroker): FROM object WHERE''' query_args = [] if end_marker: - query += ' name <= ? AND' + query += ' name < ? AND' query_args.append(end_marker) if marker and marker >= prefix: query += ' name > ? AND' diff --git a/swift/stats/log_processor.py b/swift/stats/log_processor.py index cae8e420b3..2511f5e8d6 100644 --- a/swift/stats/log_processor.py +++ b/swift/stats/log_processor.py @@ -137,11 +137,11 @@ class LogProcessor(object): year = '%04d' % parsed_date.tm_year month = '%02d' % parsed_date.tm_mon day = '%02d' % parsed_date.tm_mday - hour = '%02d' % parsed_date.tm_hour - # Since the end_marker filters by <=, we need to add something - # to then end_key to make sure we get all the data under the - # last hour. Adding '/\x7f' should be all inclusive. - end_key = '/'.join([year, month, day, hour]) + '/\x7f' + # Since the end_marker filters by <, we need to add something + # to make sure we get all the data under the last hour. Adding + # one to the hour should be all-inclusive. + hour = '%02d' % (parsed_date.tm_hour + 1) + end_key = '/'.join([year, month, day, hour]) container_listing = self.internal_proxy.get_container_list( swift_account, container_name, diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 8ab3a96d6e..ca1cb670f3 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -984,9 +984,9 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(listing[-1][0], '0/0099') listing = broker.list_objects_iter(100, '', '0/0050', None, '') - self.assertEquals(len(listing), 51) + self.assertEquals(len(listing), 50) self.assertEquals(listing[0][0], '0/0000') - self.assertEquals(listing[-1][0], '0/0050') + self.assertEquals(listing[-1][0], '0/0049') listing = broker.list_objects_iter(100, '0/0099', None, None, '') self.assertEquals(len(listing), 100) From 544107a4aea410d859f00a3182688b1ed6543bc6 Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 29 Nov 2010 15:19:29 -0800 Subject: [PATCH 02/10] proxy: added account DELETE method; added option to control whether account PUTs and DELETEs are even callable --- doc/source/deployment_guide.rst | 2 ++ doc/source/development_saio.rst | 1 + etc/proxy-server.conf-sample | 3 ++ swift/proxy/server.py | 52 +++++++++++++++++++++++++++++++++ test/unit/proxy/test_server.py | 27 +++++++++++++++++ 5 files changed, 85 insertions(+) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index aa7eef2b6f..68f8c9b5c8 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -462,6 +462,8 @@ error_suppression_interval 60 Time in seconds that must no longer error limited error_suppression_limit 10 Error count to consider a node error limited +allow_account_management false Whether account PUTs and DELETEs + are even callable ============================ =============== ============================= [auth] diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index a270753338..5591dda8cc 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -241,6 +241,7 @@ Sample configuration files are provided with all defaults in line-by-line commen [app:proxy-server] use = egg:swift#proxy + allow_account_management = true [filter:auth] use = egg:swift#auth diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 0c7477147c..220f003ba0 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -29,6 +29,9 @@ use = egg:swift#proxy # error_suppression_interval = 60 # How many errors can accumulate before a node is temporarily ignored. # error_suppression_limit = 10 +# If set to 'true' any authorized user may create and delete accounts; if +# 'false' no one, even authorized, can. +# allow_account_management = false [filter:auth] use = egg:swift#auth diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 38bb3966a6..f402a770d4 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1139,6 +1139,8 @@ class AccountController(Controller): @public def PUT(self, req): """HTTP PUT request handler.""" + if not self.app.allow_account_management: + return HTTPMethodNotAllowed(request=req) error_response = check_metadata(req, 'account') if error_response: return error_response @@ -1238,6 +1240,54 @@ class AccountController(Controller): return self.best_response(req, statuses, reasons, bodies, 'Account POST') + @public + def DELETE(self, req): + """HTTP DELETE request handler.""" + if not self.app.allow_account_management: + return HTTPMethodNotAllowed(request=req) + error_response = check_metadata(req, 'account') + if error_response: + return error_response + account_partition, accounts = \ + self.app.account_ring.get_nodes(self.account_name) + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'X-CF-Trans-Id': self.trans_id} + statuses = [] + reasons = [] + bodies = [] + for node in self.iter_nodes(account_partition, accounts, + self.app.account_ring): + if self.error_limited(node): + continue + try: + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], account_partition, 'DELETE', + req.path_info, headers) + with Timeout(self.app.node_timeout): + source = conn.getresponse() + body = source.read() + if 200 <= source.status < 300 \ + or 400 <= source.status < 500: + statuses.append(source.status) + reasons.append(source.reason) + bodies.append(body) + elif source.status == 507: + self.error_limit(node) + except: + self.exception_occurred(node, 'Account', + 'Trying to DELETE %s' % req.path) + if len(statuses) >= len(accounts): + break + while len(statuses) < len(accounts): + statuses.append(503) + reasons.append('') + bodies.append('') + if self.app.memcache: + self.app.memcache.delete('account%s' % req.path_info.rstrip('/')) + return self.best_response(req, statuses, reasons, bodies, + 'Account DELETE') + class BaseApplication(object): """Base WSGI application for the proxy server""" @@ -1265,6 +1315,8 @@ class BaseApplication(object): int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) + self.allow_account_management = \ + conf.get('allow_account_management', 'false').lower() == 'true' self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.object_ring = object_ring or \ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index ebf320fc71..13a7e9ddc5 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -2586,6 +2586,8 @@ class TestAccountController(unittest.TestCase): res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) + test_status_map((201, 201, 201), 405) + self.app.allow_account_management = True test_status_map((201, 201, 201), 201) test_status_map((201, 201, 500), 201) test_status_map((201, 500, 500), 503) @@ -2593,6 +2595,7 @@ class TestAccountController(unittest.TestCase): def test_PUT_max_account_name_length(self): with save_globals(): + self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, '1' * 256) self.assert_status_map(controller.PUT, (201, 201, 201), 201) controller = proxy_server.AccountController(self.app, '2' * 257) @@ -2600,6 +2603,7 @@ class TestAccountController(unittest.TestCase): def test_PUT_connect_exceptions(self): with save_globals(): + self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'account') self.assert_status_map(controller.PUT, (201, 201, -1), 201) self.assert_status_map(controller.PUT, (201, -1, -1), 503) @@ -2628,6 +2632,7 @@ class TestAccountController(unittest.TestCase): test_errors.append('%s: %s not in %s' % (test_header, test_value, headers)) with save_globals(): + self.app.allow_account_management = True controller = \ proxy_server.AccountController(self.app, 'a') proxy_server.http_connect = fake_http_connect(201, 201, 201, @@ -2646,6 +2651,7 @@ class TestAccountController(unittest.TestCase): def bad_metadata_helper(self, method): with save_globals(): + self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'a') proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}) @@ -2728,6 +2734,27 @@ class TestAccountController(unittest.TestCase): resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) + def test_DELETE(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + + def test_status_map(statuses, expected, **kwargs): + proxy_server.http_connect = \ + fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a', {'REQUEST_METHOD': 'DELETE'}) + req.content_length = 0 + self.app.update_request(req) + res = controller.DELETE(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((201, 201, 201), 405) + self.app.allow_account_management = True + test_status_map((201, 201, 201), 201) + test_status_map((201, 201, 500), 201) + test_status_map((201, 500, 500), 503) + test_status_map((204, 500, 404), 503) + if __name__ == '__main__': unittest.main() From 76edfbc94ce1acbcbf1c91eff3c04c1df0618f65 Mon Sep 17 00:00:00 2001 From: gholt Date: Mon, 29 Nov 2010 15:52:51 -0800 Subject: [PATCH 03/10] Updated howto_installmultinode.rst --- doc/source/howto_installmultinode.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst index 9f485e8df9..82a3a88099 100644 --- a/doc/source/howto_installmultinode.rst +++ b/doc/source/howto_installmultinode.rst @@ -124,6 +124,7 @@ Configure the Proxy node [app:proxy-server] use = egg:swift#proxy + allow_account_management = true [filter:auth] use = egg:swift#auth From 36935a2b5dffba99107414b37b8d152242d8eb48 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Tue, 30 Nov 2010 12:24:55 -0600 Subject: [PATCH 04/10] Adding Citrix contributions to Admin Guide --- doc/source/admin_guide.rst | 26 ++++++++++++++++++++++++++ doc/source/development_saio.rst | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/source/admin_guide.rst b/doc/source/admin_guide.rst index 99cd9c2446..c7a6b4ff03 100644 --- a/doc/source/admin_guide.rst +++ b/doc/source/admin_guide.rst @@ -6,6 +6,8 @@ Administrator's Guide Managing the Rings ------------------ +You need to build the storage rings on the proxy server node, and distribute them to all the servers in the cluster. Storage rings contain information about all the Swift storage partitions and how they are distributed between the different nodes and disks. For more information see :doc:`overview_ring`. + Removing a device from the ring:: swift-ring-builder remove / @@ -30,6 +32,30 @@ Once you are done with all changes to the ring, the changes need to be Once the new rings are built, they should be pushed out to all the servers in the cluster. +------------------------- +Scripting Ring Management +------------------------- +You can create scripts for the account and container ring management. Here's an example for the Account ring. + +1. Create a script file called make-account-ring.sh on the proxy server node with the following content:: + + #!/bin/bash + cd /etc/swift + rm -f account.builder account.ring.gz backups/account.builder backups/account.ring.gz + swift-ring-builder account.builder create 18 3 1 + swift-ring-builder account.builder add z1-:6002/sdb1 1 + swift-ring-builder account.builder add z2-:6002/sdb1 1 + swift-ring-builder account.builder rebalance + +You need to replace the values of , , etc. with the IP addresses of the account servers used in your setup. You can have as many account servers as you need. All account servers are assumed to be listening on port 6002, and have a storage device called "sdb1" (this is a directory name created under /drives when we setup the account server). The "z1", "z2", etc. designate zones, and you can choose whether you put devices in the same or different zones. + +2. Make the script file executable and run it to create the account ring file:: + + chmod +x make-account-ring.sh + sudo ./make-account-ring.sh + +3. Copy the resulting ring file /etc/swift/account.ring.gz to all the account server nodes in your Swift environment, and put them in the /etc/swift directory on these nodes. Make sure that every time you change the account ring configuration, you copy the resulting ring file to all the account nodes. + ----------------------- Handling System Updates ----------------------- diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index a270753338..7b9e0c780c 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -33,7 +33,7 @@ Installing dependencies and the core code python-xattr sqlite3 xfsprogs python-webob python-eventlet python-greenlet python-pastedeploy` #. Install anything else you want, like screen, ssh, vim, etc. - #. Next, choose either see :ref:`partition-section` or :ref:`loopback-section`. + #. Next, choose either :ref:`partition-section` or :ref:`loopback-section`. .. _partition-section: From 45c59e06539ad813a11c3080708c1ade8755fd2f Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Tue, 30 Nov 2010 14:15:41 -0600 Subject: [PATCH 05/10] Edited to reflect ring creation not management --- doc/source/admin_guide.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/admin_guide.rst b/doc/source/admin_guide.rst index c7a6b4ff03..31c5f17123 100644 --- a/doc/source/admin_guide.rst +++ b/doc/source/admin_guide.rst @@ -32,10 +32,10 @@ Once you are done with all changes to the ring, the changes need to be Once the new rings are built, they should be pushed out to all the servers in the cluster. -------------------------- -Scripting Ring Management -------------------------- -You can create scripts for the account and container ring management. Here's an example for the Account ring. +----------------------- +Scripting Ring Creation +----------------------- +You can create scripts to create the account and container rings and rebalance. Here's an example script for the Account ring. Use similar commands to create a make-container-ring.sh script on the proxy server node. 1. Create a script file called make-account-ring.sh on the proxy server node with the following content:: From 7e41c5fe4e303a92578bb2429c8a59fc912d9d71 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Tue, 30 Nov 2010 22:17:05 +0000 Subject: [PATCH 06/10] make swift-init only warn on missing configs --- bin/swift-init | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/swift-init b/bin/swift-init index cd31ea8a93..ade622a69c 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -120,13 +120,15 @@ def do_start(server, once=False): elif os.path.exists('/etc/swift/%s-server/' % server_type): # found config directory, searching for config file(s) launch_args = [] - for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' % server_type)): + for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' \ + % server_type)): pid_file = '/var/run/swift/%s/%d.pid' % (server, num) # start a server for each ini_file found launch_args.append((ini_file, pid_file)) else: # maybe there's a config file(s) out there, but I couldn't find it! - sys.exit('Unable to locate config file for %s. %s does not exist?' % (server, ini_file)) + print 'Unable to locate config file for %s. %s does not exist?' % \ + (server, ini_file) # start all servers for ini_file, pid_file in launch_args: From 784c7a85b6ea8d5c0d608fcd6fb6914938636077 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Tue, 30 Nov 2010 22:33:35 +0000 Subject: [PATCH 07/10] return from do_start early if config file is missing --- bin/swift-init | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/swift-init b/bin/swift-init index ade622a69c..bc2dea983e 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -129,6 +129,7 @@ def do_start(server, once=False): # maybe there's a config file(s) out there, but I couldn't find it! print 'Unable to locate config file for %s. %s does not exist?' % \ (server, ini_file) + return # start all servers for ini_file, pid_file in launch_args: From 27741ba25bc95320e268598a6abc5f4e72ee8639 Mon Sep 17 00:00:00 2001 From: Greg Lange Date: Tue, 30 Nov 2010 22:40:44 +0000 Subject: [PATCH 08/10] makes account and container info caching in proxy better --- swift/proxy/server.py | 46 ++++---- test/unit/proxy/test_server.py | 185 +++++++++++++++++++++++++-------- 2 files changed, 168 insertions(+), 63 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 38bb3966a6..d66612484d 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -88,10 +88,11 @@ def delay_denial(func): return func(*a, **kw) return wrapped +def get_account_memcache_key(account): + return 'account/%s' % account def get_container_memcache_key(account, container): - path = '/%s/%s' % (account, container) - return 'container%s' % path + return 'container/%s/%s' % (account, container) class Controller(object): @@ -176,13 +177,17 @@ class Controller(object): if it does not exist """ partition, nodes = self.app.account_ring.get_nodes(account) - path = '/%s' % account - cache_key = 'account%s' % path # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses - if self.app.memcache and self.app.memcache.get(cache_key) == 200: - return partition, nodes + if self.app.memcache: + cache_key = get_account_memcache_key(account) + result_code = self.app.memcache.get(cache_key) + if result_code == 200: + return partition, nodes + elif result_code == 404: + return None, None result_code = 0 attempts_left = self.app.account_ring.replica_count + path = '/%s' % account headers = {'x-cf-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.account_ring): if self.error_limited(node): @@ -213,16 +218,16 @@ class Controller(object): except: self.exception_occurred(node, 'Account', 'Trying to get account info for %s' % path) - if result_code == 200: - cache_timeout = self.app.recheck_account_existence - else: - cache_timeout = self.app.recheck_account_existence * 0.1 - if self.app.memcache: + if self.app.memcache and result_code in (200, 404): + if result_code == 200: + cache_timeout = self.app.recheck_account_existence + else: + cache_timeout = self.app.recheck_account_existence * 0.1 self.app.memcache.set(cache_key, result_code, timeout=cache_timeout) if result_code == 200: return partition, nodes - return (None, None) + return None, None def container_info(self, account, container): """ @@ -239,7 +244,6 @@ class Controller(object): partition, nodes = self.app.container_ring.get_nodes( account, container) path = '/%s/%s' % (account, container) - cache_key = None if self.app.memcache: cache_key = get_container_memcache_key(account, container) cache_value = self.app.memcache.get(cache_key) @@ -249,8 +253,10 @@ class Controller(object): write_acl = cache_value['write_acl'] if status == 200: return partition, nodes, read_acl, write_acl + elif status == 404: + return None, None, None, None if not self.account_info(account)[1]: - return (None, None, None, None) + return None, None, None, None result_code = 0 read_acl = None write_acl = None @@ -290,11 +296,11 @@ class Controller(object): except: self.exception_occurred(node, 'Container', 'Trying to get container info for %s' % path) - if result_code == 200: - cache_timeout = self.app.recheck_container_existence - else: - cache_timeout = self.app.recheck_container_existence * 0.1 - if cache_key and self.app.memcache: + if self.app.memcache and result_code in (200, 404): + if result_code == 200: + cache_timeout = self.app.recheck_container_existence + else: + cache_timeout = self.app.recheck_container_existence * 0.1 self.app.memcache.set(cache_key, {'status': result_code, 'read_acl': read_acl, @@ -303,7 +309,7 @@ class Controller(object): timeout=cache_timeout) if result_code == 200: return partition, nodes, read_acl, write_acl - return (None, None, None, None) + return None, None, None, None def iter_nodes(self, partition, nodes, ring): """ diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index ebf320fc71..393061bd8a 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -87,6 +87,8 @@ def fake_http_connect(*code_iter, **kwargs): pass if 'slow' in kwargs: headers['content-length'] = '4' + if 'headers' in kwargs: + headers.update(kwargs['headers']) return headers.items() def read(self, amt=None): @@ -163,6 +165,9 @@ class FakeMemcache(object): def get(self, key): return self.store.get(key) + def keys(self): + return self.store.keys() + def set(self, key, value, timeout=0): self.store[key] = value return True @@ -200,10 +205,12 @@ class NullLoggingHandler(logging.Handler): @contextmanager def save_globals(): orig_http_connect = getattr(proxy_server, 'http_connect', None) + orig_account_info = getattr(proxy_server.Controller, 'account_info', None) try: yield True finally: proxy_server.http_connect = orig_http_connect + proxy_server.Controller.account_info = orig_account_info # tests @@ -211,63 +218,155 @@ class TestController(unittest.TestCase): def setUp(self): self.account_ring = FakeRing() + self.container_ring = FakeRing() + self.memcache = FakeMemcache() - app = proxy_server.Application(None, FakeMemcache(), - account_ring=self.account_ring, container_ring=FakeRing(), + app = proxy_server.Application(None, self.memcache, + account_ring=self.account_ring, + container_ring=self.container_ring, object_ring=FakeRing()) self.controller = proxy_server.Controller(app) - def check_account_info_return(self, account, partition, nodes): - p, n = self.account_ring.get_nodes(account) + self.account = 'some_account' + self.container = 'some_container' + self.read_acl = 'read_acl' + self.write_acl = 'write_acl' + + def check_account_info_return(self, partition, nodes, is_none=False): + if is_none: + p, n = None, None + else: + p, n = self.account_ring.get_nodes(self.account) self.assertEqual(p, partition) self.assertEqual(n, nodes) - def test_account_info_404_200(self): - account = 'test_account_info_404_200' - - with save_globals(): - proxy_server.http_connect = fake_http_connect(404, 404, 404) - partition, nodes = self.controller.account_info(account) - self.assertEqual(partition, None) - self.assertEqual(nodes, None) - - proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) - - def test_account_info_404(self): - account = 'test_account_info_404' - - with save_globals(): - proxy_server.http_connect = fake_http_connect(404, 404, 404) - partition, nodes = self.controller.account_info(account) - self.assertEqual(partition, None) - self.assertEqual(nodes, None) - - proxy_server.http_connect = fake_http_connect(404, 404, 404) - partition, nodes = self.controller.account_info(account) - self.assertEqual(partition, None) - self.assertEqual(nodes, None) - + # tests if 200 is cached and used def test_account_info_200(self): - account = 'test_account_info_200' - with save_globals(): proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes) - def test_account_info_200_200(self): - account = 'test_account_info_200_200' + cache_key = proxy_server.get_account_memcache_key(self.account) + self.assertEquals(200, self.memcache.get(cache_key)) + + proxy_server.http_connect = fake_http_connect() + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes) + + # tests if 404 is cached and used + def test_account_info_404(self): + with save_globals(): + proxy_server.http_connect = fake_http_connect(404, 404, 404) + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes, True) + + cache_key = proxy_server.get_account_memcache_key(self.account) + self.assertEquals(404, self.memcache.get(cache_key)) + + proxy_server.http_connect = fake_http_connect() + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes, True) + + # tests if some http status codes are not cached + def test_account_info_no_cache(self): + def test(*status_list): + proxy_server.http_connect = fake_http_connect(*status_list) + partition, nodes = self.controller.account_info(self.account) + self.assertEqual(len(self.memcache.keys()), 0) + self.check_account_info_return(partition, nodes, True) with save_globals(): - proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) + test(503, 404, 404) + test(404, 404, 503) + test(404, 507, 503) + test(503, 503, 503) - proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) + def check_container_info_return(self, ret, is_none=False): + if is_none: + partition, nodes, read_acl, write_acl = None, None, None, None + else: + partition, nodes = self.container_ring.get_nodes(self.account, + self.container) + read_acl, write_acl = self.read_acl, self.write_acl + self.assertEqual(partition, ret[0]) + self.assertEqual(nodes, ret[1]) + self.assertEqual(read_acl, ret[2]) + self.assertEqual(write_acl, ret[3]) + + def test_container_info_invalid_account(self): + def account_info(self, account): + return None, None + + with save_globals(): + proxy_server.Controller.account_info = account_info + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret, True) + + # tests if 200 is cached and used + def test_container_info_200(self): + def account_info(self, account): + return True, True + + with save_globals(): + headers = {'x-container-read': self.read_acl, + 'x-container-write': self.write_acl} + proxy_server.Controller.account_info = account_info + proxy_server.http_connect = fake_http_connect(200, + headers=headers) + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret) + + cache_key = proxy_server.get_container_memcache_key(self.account, + self.container) + cache_value = self.memcache.get(cache_key) + self.assertEquals(dict, type(cache_value)) + self.assertEquals(200, cache_value.get('status')) + + proxy_server.http_connect = fake_http_connect() + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret) + + # tests if 404 is cached and used + def test_container_info_404(self): + def account_info(self, account): + return True, True + + with save_globals(): + proxy_server.Controller.account_info = account_info + proxy_server.http_connect = fake_http_connect(404, 404, 404) + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret, True) + + cache_key = proxy_server.get_container_memcache_key(self.account, + self.container) + cache_value = self.memcache.get(cache_key) + self.assertEquals(dict, type(cache_value)) + self.assertEquals(404, cache_value.get('status')) + + proxy_server.http_connect = fake_http_connect() + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret, True) + + # tests if some http status codes are not cached + def test_container_info_no_cache(self): + def test(*status_list): + proxy_server.http_connect = fake_http_connect(*status_list) + ret = self.controller.container_info(self.account, + self.container) + self.assertEqual(len(self.memcache.keys()), 0) + self.check_container_info_return(ret, True) + + with save_globals(): + test(503, 404, 404) + test(404, 404, 503) + test(404, 507, 503) + test(503, 503, 503) class TestProxyServer(unittest.TestCase): From 918ebd6bf222fde125880daeabd3c7d21e6c9769 Mon Sep 17 00:00:00 2001 From: gholt Date: Tue, 30 Nov 2010 15:00:12 -0800 Subject: [PATCH 09/10] Removed extra copy-pasted code --- swift/proxy/server.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index c71f136741..e48052a398 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1245,9 +1245,6 @@ class AccountController(Controller): """HTTP DELETE request handler.""" if not self.app.allow_account_management: return HTTPMethodNotAllowed(request=req) - error_response = check_metadata(req, 'account') - if error_response: - return error_response account_partition, accounts = \ self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), From 1f145a35cb2ac6ace7bed88374cb10d318d41ae3 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 30 Nov 2010 19:07:21 -0600 Subject: [PATCH 10/10] update common.middleware.auth tests for compat with webob 1.0 --- test/unit/common/middleware/test_auth.py | 66 +++++++++++++----------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index 800ecb4cb7..b380d4f684 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -314,12 +314,19 @@ class TestAuth(unittest.TestCase): def test_authorize_bad_path(self): req = Request.blank('/badpath') - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('401'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 401) req = Request.blank('/badpath') req.remote_user = 'act:usr,act,AUTH_cfa' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) + req = Request.blank('') + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 404) + req = Request.blank('') + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, lambda x, y: None)) + self.assert_(result.startswith('404'), result) def test_authorize_account_access(self): req = Request.blank('/v1/AUTH_cfa') @@ -327,14 +334,14 @@ class TestAuth(unittest.TestCase): self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) def test_authorize_acl_group_access(self): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act' @@ -346,27 +353,27 @@ class TestAuth(unittest.TestCase): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act2' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act:usr2' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) def test_deny_cross_reseller(self): # Tests that cross-reseller is denied, even if ACLs/group names match req = Request.blank('/v1/OTHER_cfa') req.remote_user = 'act:usr,act,AUTH_cfa' req.acl = 'act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) def test_authorize_acl_referrer_access(self): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = '.r:*' @@ -374,23 +381,23 @@ class TestAuth(unittest.TestCase): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = '.r:.example.com' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com' self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/AUTH_cfa') - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('401'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 401) req = Request.blank('/v1/AUTH_cfa') req.acl = '.r:*' self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/AUTH_cfa') req.acl = '.r:.example.com' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('401'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 401) req = Request.blank('/v1/AUTH_cfa') req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com' @@ -399,19 +406,19 @@ class TestAuth(unittest.TestCase): def test_account_put_permissions(self): req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,AUTH_other' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) # Even PUTs to your own account as account admin should fail req = Request.blank('/v1/AUTH_old', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,AUTH_old' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,.reseller_admin' @@ -423,8 +430,7 @@ class TestAuth(unittest.TestCase): req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,.super_admin' resp = self.test_auth.authorize(req) - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + self.assertEquals(resp and resp.status_int, 403) if __name__ == '__main__':