diff --git a/bin/swift-init b/bin/swift-init index cd31ea8a93..bc2dea983e 100755 --- a/bin/swift-init +++ b/bin/swift-init @@ -120,13 +120,16 @@ def do_start(server, once=False): elif os.path.exists('/etc/swift/%s-server/' % server_type): # found config directory, searching for config file(s) launch_args = [] - for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' % server_type)): + for num, ini_file in enumerate(glob.glob('/etc/swift/%s-server/*.conf' \ + % server_type)): pid_file = '/var/run/swift/%s/%d.pid' % (server, num) # start a server for each ini_file found launch_args.append((ini_file, pid_file)) else: # maybe there's a config file(s) out there, but I couldn't find it! - sys.exit('Unable to locate config file for %s. %s does not exist?' % (server, ini_file)) + print 'Unable to locate config file for %s. %s does not exist?' % \ + (server, ini_file) + return # start all servers for ini_file, pid_file in launch_args: diff --git a/doc/source/admin_guide.rst b/doc/source/admin_guide.rst index 99cd9c2446..31c5f17123 100644 --- a/doc/source/admin_guide.rst +++ b/doc/source/admin_guide.rst @@ -6,6 +6,8 @@ Administrator's Guide Managing the Rings ------------------ +You need to build the storage rings on the proxy server node, and distribute them to all the servers in the cluster. Storage rings contain information about all the Swift storage partitions and how they are distributed between the different nodes and disks. For more information see :doc:`overview_ring`. + Removing a device from the ring:: swift-ring-builder remove / @@ -30,6 +32,30 @@ Once you are done with all changes to the ring, the changes need to be Once the new rings are built, they should be pushed out to all the servers in the cluster. +----------------------- +Scripting Ring Creation +----------------------- +You can create scripts to create the account and container rings and rebalance. Here's an example script for the Account ring. Use similar commands to create a make-container-ring.sh script on the proxy server node. + +1. Create a script file called make-account-ring.sh on the proxy server node with the following content:: + + #!/bin/bash + cd /etc/swift + rm -f account.builder account.ring.gz backups/account.builder backups/account.ring.gz + swift-ring-builder account.builder create 18 3 1 + swift-ring-builder account.builder add z1-:6002/sdb1 1 + swift-ring-builder account.builder add z2-:6002/sdb1 1 + swift-ring-builder account.builder rebalance + +You need to replace the values of , , etc. with the IP addresses of the account servers used in your setup. You can have as many account servers as you need. All account servers are assumed to be listening on port 6002, and have a storage device called "sdb1" (this is a directory name created under /drives when we setup the account server). The "z1", "z2", etc. designate zones, and you can choose whether you put devices in the same or different zones. + +2. Make the script file executable and run it to create the account ring file:: + + chmod +x make-account-ring.sh + sudo ./make-account-ring.sh + +3. Copy the resulting ring file /etc/swift/account.ring.gz to all the account server nodes in your Swift environment, and put them in the /etc/swift directory on these nodes. Make sure that every time you change the account ring configuration, you copy the resulting ring file to all the account nodes. + ----------------------- Handling System Updates ----------------------- diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index aa7eef2b6f..68f8c9b5c8 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -462,6 +462,8 @@ error_suppression_interval 60 Time in seconds that must no longer error limited error_suppression_limit 10 Error count to consider a node error limited +allow_account_management false Whether account PUTs and DELETEs + are even callable ============================ =============== ============================= [auth] diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index a270753338..999d338fbf 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -33,7 +33,7 @@ Installing dependencies and the core code python-xattr sqlite3 xfsprogs python-webob python-eventlet python-greenlet python-pastedeploy` #. Install anything else you want, like screen, ssh, vim, etc. - #. Next, choose either see :ref:`partition-section` or :ref:`loopback-section`. + #. Next, choose either :ref:`partition-section` or :ref:`loopback-section`. .. _partition-section: @@ -241,6 +241,7 @@ Sample configuration files are provided with all defaults in line-by-line commen [app:proxy-server] use = egg:swift#proxy + allow_account_management = true [filter:auth] use = egg:swift#auth diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst index 9f485e8df9..82a3a88099 100644 --- a/doc/source/howto_installmultinode.rst +++ b/doc/source/howto_installmultinode.rst @@ -124,6 +124,7 @@ Configure the Proxy node [app:proxy-server] use = egg:swift#proxy + allow_account_management = true [filter:auth] use = egg:swift#auth diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 0c7477147c..220f003ba0 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -29,6 +29,9 @@ use = egg:swift#proxy # error_suppression_interval = 60 # How many errors can accumulate before a node is temporarily ignored. # error_suppression_limit = 10 +# If set to 'true' any authorized user may create and delete accounts; if +# 'false' no one, even authorized, can. +# allow_account_management = false [filter:auth] use = egg:swift#auth diff --git a/swift/common/db.py b/swift/common/db.py index c19c62910f..41854407d6 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -979,7 +979,7 @@ class ContainerBroker(DatabaseBroker): FROM object WHERE''' query_args = [] if end_marker: - query += ' name <= ? AND' + query += ' name < ? AND' query_args.append(end_marker) if marker and marker >= prefix: query += ' name > ? AND' diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 2d925fbc02..32b0543e1e 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -93,10 +93,11 @@ def delay_denial(func): return func(*a, **kw) return wrapped +def get_account_memcache_key(account): + return 'account/%s' % account def get_container_memcache_key(account, container): - path = '/%s/%s' % (account, container) - return 'container%s' % path + return 'container/%s/%s' % (account, container) class SegmentedIterable(object): @@ -329,13 +330,17 @@ class Controller(object): if it does not exist """ partition, nodes = self.app.account_ring.get_nodes(account) - path = '/%s' % account - cache_key = 'account%s' % path # 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses - if self.app.memcache and self.app.memcache.get(cache_key) == 200: - return partition, nodes + if self.app.memcache: + cache_key = get_account_memcache_key(account) + result_code = self.app.memcache.get(cache_key) + if result_code == 200: + return partition, nodes + elif result_code == 404: + return None, None result_code = 0 attempts_left = self.app.account_ring.replica_count + path = '/%s' % account headers = {'x-cf-trans-id': self.trans_id} for node in self.iter_nodes(partition, nodes, self.app.account_ring): if self.error_limited(node): @@ -366,16 +371,16 @@ class Controller(object): except: self.exception_occurred(node, 'Account', 'Trying to get account info for %s' % path) - if result_code == 200: - cache_timeout = self.app.recheck_account_existence - else: - cache_timeout = self.app.recheck_account_existence * 0.1 - if self.app.memcache: + if self.app.memcache and result_code in (200, 404): + if result_code == 200: + cache_timeout = self.app.recheck_account_existence + else: + cache_timeout = self.app.recheck_account_existence * 0.1 self.app.memcache.set(cache_key, result_code, timeout=cache_timeout) if result_code == 200: return partition, nodes - return (None, None) + return None, None def container_info(self, account, container): """ @@ -392,7 +397,6 @@ class Controller(object): partition, nodes = self.app.container_ring.get_nodes( account, container) path = '/%s/%s' % (account, container) - cache_key = None if self.app.memcache: cache_key = get_container_memcache_key(account, container) cache_value = self.app.memcache.get(cache_key) @@ -402,8 +406,10 @@ class Controller(object): write_acl = cache_value['write_acl'] if status == 200: return partition, nodes, read_acl, write_acl + elif status == 404: + return None, None, None, None if not self.account_info(account)[1]: - return (None, None, None, None) + return None, None, None, None result_code = 0 read_acl = None write_acl = None @@ -443,11 +449,11 @@ class Controller(object): except: self.exception_occurred(node, 'Container', 'Trying to get container info for %s' % path) - if result_code == 200: - cache_timeout = self.app.recheck_container_existence - else: - cache_timeout = self.app.recheck_container_existence * 0.1 - if cache_key and self.app.memcache: + if self.app.memcache and result_code in (200, 404): + if result_code == 200: + cache_timeout = self.app.recheck_container_existence + else: + cache_timeout = self.app.recheck_container_existence * 0.1 self.app.memcache.set(cache_key, {'status': result_code, 'read_acl': read_acl, @@ -456,7 +462,7 @@ class Controller(object): timeout=cache_timeout) if result_code == 200: return partition, nodes, read_acl, write_acl - return (None, None, None, None) + return None, None, None, None def iter_nodes(self, partition, nodes, ring): """ @@ -1338,6 +1344,8 @@ class AccountController(Controller): @public def PUT(self, req): """HTTP PUT request handler.""" + if not self.app.allow_account_management: + return HTTPMethodNotAllowed(request=req) error_response = check_metadata(req, 'account') if error_response: return error_response @@ -1437,6 +1445,51 @@ class AccountController(Controller): return self.best_response(req, statuses, reasons, bodies, 'Account POST') + @public + def DELETE(self, req): + """HTTP DELETE request handler.""" + if not self.app.allow_account_management: + return HTTPMethodNotAllowed(request=req) + account_partition, accounts = \ + self.app.account_ring.get_nodes(self.account_name) + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'X-CF-Trans-Id': self.trans_id} + statuses = [] + reasons = [] + bodies = [] + for node in self.iter_nodes(account_partition, accounts, + self.app.account_ring): + if self.error_limited(node): + continue + try: + with ConnectionTimeout(self.app.conn_timeout): + conn = http_connect(node['ip'], node['port'], + node['device'], account_partition, 'DELETE', + req.path_info, headers) + with Timeout(self.app.node_timeout): + source = conn.getresponse() + body = source.read() + if 200 <= source.status < 300 \ + or 400 <= source.status < 500: + statuses.append(source.status) + reasons.append(source.reason) + bodies.append(body) + elif source.status == 507: + self.error_limit(node) + except: + self.exception_occurred(node, 'Account', + 'Trying to DELETE %s' % req.path) + if len(statuses) >= len(accounts): + break + while len(statuses) < len(accounts): + statuses.append(503) + reasons.append('') + bodies.append('') + if self.app.memcache: + self.app.memcache.delete('account%s' % req.path_info.rstrip('/')) + return self.best_response(req, statuses, reasons, bodies, + 'Account DELETE') + class BaseApplication(object): """Base WSGI application for the proxy server""" @@ -1464,6 +1517,8 @@ class BaseApplication(object): int(conf.get('recheck_container_existence', 60)) self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) + self.allow_account_management = \ + conf.get('allow_account_management', 'false').lower() == 'true' self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.object_ring = object_ring or \ diff --git a/swift/stats/log_processor.py b/swift/stats/log_processor.py index cae8e420b3..2511f5e8d6 100644 --- a/swift/stats/log_processor.py +++ b/swift/stats/log_processor.py @@ -137,11 +137,11 @@ class LogProcessor(object): year = '%04d' % parsed_date.tm_year month = '%02d' % parsed_date.tm_mon day = '%02d' % parsed_date.tm_mday - hour = '%02d' % parsed_date.tm_hour - # Since the end_marker filters by <=, we need to add something - # to then end_key to make sure we get all the data under the - # last hour. Adding '/\x7f' should be all inclusive. - end_key = '/'.join([year, month, day, hour]) + '/\x7f' + # Since the end_marker filters by <, we need to add something + # to make sure we get all the data under the last hour. Adding + # one to the hour should be all-inclusive. + hour = '%02d' % (parsed_date.tm_hour + 1) + end_key = '/'.join([year, month, day, hour]) container_listing = self.internal_proxy.get_container_list( swift_account, container_name, diff --git a/test/unit/common/middleware/test_auth.py b/test/unit/common/middleware/test_auth.py index 800ecb4cb7..b380d4f684 100644 --- a/test/unit/common/middleware/test_auth.py +++ b/test/unit/common/middleware/test_auth.py @@ -314,12 +314,19 @@ class TestAuth(unittest.TestCase): def test_authorize_bad_path(self): req = Request.blank('/badpath') - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('401'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 401) req = Request.blank('/badpath') req.remote_user = 'act:usr,act,AUTH_cfa' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) + req = Request.blank('') + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 404) + req = Request.blank('') + req.environ['swift.cache'] = FakeMemcache() + result = ''.join(self.test_auth(req.environ, lambda x, y: None)) + self.assert_(result.startswith('404'), result) def test_authorize_account_access(self): req = Request.blank('/v1/AUTH_cfa') @@ -327,14 +334,14 @@ class TestAuth(unittest.TestCase): self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) def test_authorize_acl_group_access(self): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act' @@ -346,27 +353,27 @@ class TestAuth(unittest.TestCase): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act2' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = 'act:usr2' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) def test_deny_cross_reseller(self): # Tests that cross-reseller is denied, even if ACLs/group names match req = Request.blank('/v1/OTHER_cfa') req.remote_user = 'act:usr,act,AUTH_cfa' req.acl = 'act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) def test_authorize_acl_referrer_access(self): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = '.r:*' @@ -374,23 +381,23 @@ class TestAuth(unittest.TestCase): req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.acl = '.r:.example.com' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_cfa') req.remote_user = 'act:usr,act' req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com' self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/AUTH_cfa') - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('401'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 401) req = Request.blank('/v1/AUTH_cfa') req.acl = '.r:*' self.assertEquals(self.test_auth.authorize(req), None) req = Request.blank('/v1/AUTH_cfa') req.acl = '.r:.example.com' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('401'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 401) req = Request.blank('/v1/AUTH_cfa') req.referer = 'http://www.example.com/index.html' req.acl = '.r:.example.com' @@ -399,19 +406,19 @@ class TestAuth(unittest.TestCase): def test_account_put_permissions(self): req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,AUTH_other' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) # Even PUTs to your own account as account admin should fail req = Request.blank('/v1/AUTH_old', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,AUTH_old' - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + resp = self.test_auth.authorize(req) + self.assertEquals(resp and resp.status_int, 403) req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,.reseller_admin' @@ -423,8 +430,7 @@ class TestAuth(unittest.TestCase): req = Request.blank('/v1/AUTH_new', environ={'REQUEST_METHOD': 'PUT'}) req.remote_user = 'act:usr,act,.super_admin' resp = self.test_auth.authorize(req) - resp = str(self.test_auth.authorize(req)) - self.assert_(resp.startswith('403'), resp) + self.assertEquals(resp and resp.status_int, 403) if __name__ == '__main__': diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 8ab3a96d6e..ca1cb670f3 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -984,9 +984,9 @@ class TestContainerBroker(unittest.TestCase): self.assertEquals(listing[-1][0], '0/0099') listing = broker.list_objects_iter(100, '', '0/0050', None, '') - self.assertEquals(len(listing), 51) + self.assertEquals(len(listing), 50) self.assertEquals(listing[0][0], '0/0000') - self.assertEquals(listing[-1][0], '0/0050') + self.assertEquals(listing[-1][0], '0/0049') listing = broker.list_objects_iter(100, '0/0099', None, None, '') self.assertEquals(len(listing), 100) diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 0872f0adf4..d60f4c8590 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -90,6 +90,8 @@ def fake_http_connect(*code_iter, **kwargs): pass if 'slow' in kwargs: headers['content-length'] = '4' + if 'headers' in kwargs: + headers.update(kwargs['headers']) return headers.items() def read(self, amt=None): @@ -167,6 +169,9 @@ class FakeMemcache(object): def get(self, key): return self.store.get(key) + def keys(self): + return self.store.keys() + def set(self, key, value, timeout=0): self.store[key] = value return True @@ -204,10 +209,12 @@ class NullLoggingHandler(logging.Handler): @contextmanager def save_globals(): orig_http_connect = getattr(proxy_server, 'http_connect', None) + orig_account_info = getattr(proxy_server.Controller, 'account_info', None) try: yield True finally: proxy_server.http_connect = orig_http_connect + proxy_server.Controller.account_info = orig_account_info # tests @@ -215,63 +222,155 @@ class TestController(unittest.TestCase): def setUp(self): self.account_ring = FakeRing() + self.container_ring = FakeRing() + self.memcache = FakeMemcache() - app = proxy_server.Application(None, FakeMemcache(), - account_ring=self.account_ring, container_ring=FakeRing(), + app = proxy_server.Application(None, self.memcache, + account_ring=self.account_ring, + container_ring=self.container_ring, object_ring=FakeRing()) self.controller = proxy_server.Controller(app) - def check_account_info_return(self, account, partition, nodes): - p, n = self.account_ring.get_nodes(account) + self.account = 'some_account' + self.container = 'some_container' + self.read_acl = 'read_acl' + self.write_acl = 'write_acl' + + def check_account_info_return(self, partition, nodes, is_none=False): + if is_none: + p, n = None, None + else: + p, n = self.account_ring.get_nodes(self.account) self.assertEqual(p, partition) self.assertEqual(n, nodes) - def test_account_info_404_200(self): - account = 'test_account_info_404_200' - - with save_globals(): - proxy_server.http_connect = fake_http_connect(404, 404, 404) - partition, nodes = self.controller.account_info(account) - self.assertEqual(partition, None) - self.assertEqual(nodes, None) - - proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) - - def test_account_info_404(self): - account = 'test_account_info_404' - - with save_globals(): - proxy_server.http_connect = fake_http_connect(404, 404, 404) - partition, nodes = self.controller.account_info(account) - self.assertEqual(partition, None) - self.assertEqual(nodes, None) - - proxy_server.http_connect = fake_http_connect(404, 404, 404) - partition, nodes = self.controller.account_info(account) - self.assertEqual(partition, None) - self.assertEqual(nodes, None) - + # tests if 200 is cached and used def test_account_info_200(self): - account = 'test_account_info_200' - with save_globals(): proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes) - def test_account_info_200_200(self): - account = 'test_account_info_200_200' + cache_key = proxy_server.get_account_memcache_key(self.account) + self.assertEquals(200, self.memcache.get(cache_key)) + + proxy_server.http_connect = fake_http_connect() + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes) + + # tests if 404 is cached and used + def test_account_info_404(self): + with save_globals(): + proxy_server.http_connect = fake_http_connect(404, 404, 404) + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes, True) + + cache_key = proxy_server.get_account_memcache_key(self.account) + self.assertEquals(404, self.memcache.get(cache_key)) + + proxy_server.http_connect = fake_http_connect() + partition, nodes = self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes, True) + + # tests if some http status codes are not cached + def test_account_info_no_cache(self): + def test(*status_list): + proxy_server.http_connect = fake_http_connect(*status_list) + partition, nodes = self.controller.account_info(self.account) + self.assertEqual(len(self.memcache.keys()), 0) + self.check_account_info_return(partition, nodes, True) with save_globals(): - proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) + test(503, 404, 404) + test(404, 404, 503) + test(404, 507, 503) + test(503, 503, 503) - proxy_server.http_connect = fake_http_connect(200) - partition, nodes = self.controller.account_info(account) - self.check_account_info_return(account, partition, nodes) + def check_container_info_return(self, ret, is_none=False): + if is_none: + partition, nodes, read_acl, write_acl = None, None, None, None + else: + partition, nodes = self.container_ring.get_nodes(self.account, + self.container) + read_acl, write_acl = self.read_acl, self.write_acl + self.assertEqual(partition, ret[0]) + self.assertEqual(nodes, ret[1]) + self.assertEqual(read_acl, ret[2]) + self.assertEqual(write_acl, ret[3]) + + def test_container_info_invalid_account(self): + def account_info(self, account): + return None, None + + with save_globals(): + proxy_server.Controller.account_info = account_info + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret, True) + + # tests if 200 is cached and used + def test_container_info_200(self): + def account_info(self, account): + return True, True + + with save_globals(): + headers = {'x-container-read': self.read_acl, + 'x-container-write': self.write_acl} + proxy_server.Controller.account_info = account_info + proxy_server.http_connect = fake_http_connect(200, + headers=headers) + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret) + + cache_key = proxy_server.get_container_memcache_key(self.account, + self.container) + cache_value = self.memcache.get(cache_key) + self.assertEquals(dict, type(cache_value)) + self.assertEquals(200, cache_value.get('status')) + + proxy_server.http_connect = fake_http_connect() + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret) + + # tests if 404 is cached and used + def test_container_info_404(self): + def account_info(self, account): + return True, True + + with save_globals(): + proxy_server.Controller.account_info = account_info + proxy_server.http_connect = fake_http_connect(404, 404, 404) + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret, True) + + cache_key = proxy_server.get_container_memcache_key(self.account, + self.container) + cache_value = self.memcache.get(cache_key) + self.assertEquals(dict, type(cache_value)) + self.assertEquals(404, cache_value.get('status')) + + proxy_server.http_connect = fake_http_connect() + ret = self.controller.container_info(self.account, + self.container) + self.check_container_info_return(ret, True) + + # tests if some http status codes are not cached + def test_container_info_no_cache(self): + def test(*status_list): + proxy_server.http_connect = fake_http_connect(*status_list) + ret = self.controller.container_info(self.account, + self.container) + self.assertEqual(len(self.memcache.keys()), 0) + self.check_container_info_return(ret, True) + + with save_globals(): + test(503, 404, 404) + test(404, 404, 503) + test(404, 507, 503) + test(503, 503, 503) class TestProxyServer(unittest.TestCase): @@ -2681,6 +2780,8 @@ class TestAccountController(unittest.TestCase): res = controller.PUT(req) expected = str(expected) self.assertEquals(res.status[:len(expected)], expected) + test_status_map((201, 201, 201), 405) + self.app.allow_account_management = True test_status_map((201, 201, 201), 201) test_status_map((201, 201, 500), 201) test_status_map((201, 500, 500), 503) @@ -2688,6 +2789,7 @@ class TestAccountController(unittest.TestCase): def test_PUT_max_account_name_length(self): with save_globals(): + self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, '1' * 256) self.assert_status_map(controller.PUT, (201, 201, 201), 201) controller = proxy_server.AccountController(self.app, '2' * 257) @@ -2695,6 +2797,7 @@ class TestAccountController(unittest.TestCase): def test_PUT_connect_exceptions(self): with save_globals(): + self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'account') self.assert_status_map(controller.PUT, (201, 201, -1), 201) self.assert_status_map(controller.PUT, (201, -1, -1), 503) @@ -2723,6 +2826,7 @@ class TestAccountController(unittest.TestCase): test_errors.append('%s: %s not in %s' % (test_header, test_value, headers)) with save_globals(): + self.app.allow_account_management = True controller = \ proxy_server.AccountController(self.app, 'a') proxy_server.http_connect = fake_http_connect(201, 201, 201, @@ -2741,6 +2845,7 @@ class TestAccountController(unittest.TestCase): def bad_metadata_helper(self, method): with save_globals(): + self.app.allow_account_management = True controller = proxy_server.AccountController(self.app, 'a') proxy_server.http_connect = fake_http_connect(200, 201, 201, 201) req = Request.blank('/a/c', environ={'REQUEST_METHOD': method}) @@ -2823,6 +2928,27 @@ class TestAccountController(unittest.TestCase): resp = getattr(controller, method)(req) self.assertEquals(resp.status_int, 400) + def test_DELETE(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + + def test_status_map(statuses, expected, **kwargs): + proxy_server.http_connect = \ + fake_http_connect(*statuses, **kwargs) + self.app.memcache.store = {} + req = Request.blank('/a', {'REQUEST_METHOD': 'DELETE'}) + req.content_length = 0 + self.app.update_request(req) + res = controller.DELETE(req) + expected = str(expected) + self.assertEquals(res.status[:len(expected)], expected) + test_status_map((201, 201, 201), 405) + self.app.allow_account_management = True + test_status_map((201, 201, 201), 201) + test_status_map((201, 201, 500), 201) + test_status_map((201, 500, 500), 503) + test_status_map((204, 500, 404), 503) + class FakeObjectController(object):