diff --git a/swift/container/server.py b/swift/container/server.py index 77b91ce07c..e36bba64d0 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -768,16 +768,22 @@ class ContainerController(BaseStorageServer): include_deleted=include_deleted, fill_gaps=fill_gaps, include_own=include_own) else: + requested_policy_index = self.get_and_validate_policy_index(req) resp_headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=resp_headers) resp_headers['X-Backend-Record-Type'] = 'object' + storage_policy_index = ( + requested_policy_index if requested_policy_index is not None + else info['storage_policy_index']) + resp_headers['X-Backend-Record-Storage-Policy-Index'] = \ + storage_policy_index # Use the retired db while container is in process of sharding, # otherwise use current db src_broker = broker.get_brokers()[0] container_list = src_broker.list_objects_iter( limit, marker, end_marker, prefix, delimiter, path, - storage_policy_index=info['storage_policy_index'], + storage_policy_index=storage_policy_index, reverse=reverse, allow_reserved=req.allow_reserved_names) return self.create_listing(req, out_content_type, info, resp_headers, broker.metadata, container_list, container) diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index 0ed01c313d..500cc96237 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -2285,7 +2285,9 @@ class Controller(object): :param req: original Request instance. :param account: account in which `container` is stored. :param container: container from which listing should be fetched. - :param headers: headers to be included with the request + :param headers: extra headers to be included with the listing + sub-request; these update the headers copied from the original + request. :param params: query string parameters to be used. :return: a tuple of (deserialized json data structure, swob Response) """ diff --git a/swift/proxy/controllers/container.py b/swift/proxy/controllers/container.py index 5144313be5..81b5856fee 100644 --- a/swift/proxy/controllers/container.py +++ b/swift/proxy/controllers/container.py @@ -318,6 +318,17 @@ class ContainerController(Controller): # shard may return the root's shard range. shard_listing_history = req.environ.setdefault( 'swift.shard_listing_history', []) + policy_key = 'X-Backend-Storage-Policy-Index' + if not (shard_listing_history or policy_key in req.headers): + # We're handling the original request to the root container: set + # the root policy index in the request, unless it is already set, + # so that shards will return listings for that policy index. + # Note: we only get here if the root responded with shard ranges, + # or if the shard ranges were cached and the cached root container + # info has sharding_state==sharded; in both cases we can assume + # that the response is "modern enough" to include + # 'X-Backend-Storage-Policy-Index'. + req.headers[policy_key] = resp.headers[policy_key] shard_listing_history.append((self.account_name, self.container_name)) shard_ranges = [ShardRange.from_dict(data) for data in json.loads(resp.body)] diff --git a/test/probe/test_sharder.py b/test/probe/test_sharder.py index f702a73b23..adcf6dae1c 100644 --- a/test/probe/test_sharder.py +++ b/test/probe/test_sharder.py @@ -16,6 +16,7 @@ import json import os import shutil import subprocess +import unittest import uuid from nose import SkipTest @@ -23,6 +24,7 @@ import six from six.moves.urllib.parse import quote from swift.common import direct_client, utils +from swift.common.internal_client import UnexpectedResponse from swift.common.manager import Manager from swift.common.memcached import MemcacheRing from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \ @@ -39,7 +41,7 @@ from test import annotate_failure from test.probe import PROXY_BASE_URL from test.probe.brain import BrainSplitter from test.probe.common import ReplProbeTest, get_server_number, \ - wait_for_server_to_hangup + wait_for_server_to_hangup, ENABLED_POLICIES import mock @@ -2215,6 +2217,72 @@ class TestContainerSharding(BaseTestContainerSharding): self.assert_container_delete_fails() self.assert_container_post_ok('revived') + def _do_test_sharded_can_get_objects_different_policy(self, + policy_idx, + new_policy_idx): + # create sharded container + client.delete_container(self.url, self.token, self.container_name) + self.brain.put_container(policy_index=int(policy_idx)) + all_obj_names = self._make_object_names(self.max_shard_size) + self.put_objects(all_obj_names) + client.post_container(self.url, self.admin_token, self.container_name, + headers={'X-Container-Sharding': 'on'}) + for n in self.brain.node_numbers: + self.sharders.once( + number=n, additional_args='--partitions=%s' % self.brain.part) + # empty and delete + self.delete_objects(all_obj_names) + shard_ranges = self.get_container_shard_ranges() + self.run_sharders(shard_ranges) + client.delete_container(self.url, self.token, self.container_name) + + # re-create with new_policy_idx + self.brain.put_container(policy_index=int(new_policy_idx)) + + # we re-use shard ranges + new_shard_ranges = self.get_container_shard_ranges() + self.assertEqual(shard_ranges, new_shard_ranges) + self.put_objects(all_obj_names) + + # The shard is still on the old policy index, but the root spi + # is passed to shard container server and is used to pull objects + # of that index out. + self.assert_container_listing(all_obj_names) + # although a head request is getting object count for the shard spi + self.assert_container_object_count(0) + + # we can force the listing to use the old policy index in which case we + # expect no objects to be listed + try: + resp = self.internal_client.make_request( + 'GET', + path=self.internal_client.make_path( + self.account, self.container_name), + headers={'X-Backend-Storage-Policy-Index': str(policy_idx)}, + acceptable_statuses=(2,), + params={'format': 'json'} + ) + except UnexpectedResponse as exc: + self.fail('Listing failed with %s' % exc.resp.status) + + self.assertEqual([], json.loads(b''.join(resp.app_iter))) + + @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy") + def test_sharded_can_get_objects_different_policy(self): + policy_idx = self.policy.idx + new_policy_idx = [pol.idx for pol in ENABLED_POLICIES + if pol != self.policy.idx][0] + self._do_test_sharded_can_get_objects_different_policy( + policy_idx, new_policy_idx) + + @unittest.skipIf(len(ENABLED_POLICIES) < 2, "Need more than one policy") + def test_sharded_can_get_objects_different_policy_reversed(self): + policy_idx = [pol.idx for pol in ENABLED_POLICIES + if pol != self.policy][0] + new_policy_idx = self.policy.idx + self._do_test_sharded_can_get_objects_different_policy( + policy_idx, new_policy_idx) + def test_object_update_redirection(self): all_obj_names = self._make_object_names(self.max_shard_size) self.put_objects(all_obj_names) diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index fb7a8a568b..97c98931a8 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -3415,6 +3415,12 @@ class TestContainerController(unittest.TestCase): self.assertIn('X-Backend-Record-Type', resp.headers) self.assertEqual( 'object', resp.headers.pop('X-Backend-Record-Type')) + self.assertEqual( + str(POLICIES.default.idx), + resp.headers.pop('X-Backend-Storage-Policy-Index')) + self.assertEqual( + str(POLICIES.default.idx), + resp.headers.pop('X-Backend-Record-Storage-Policy-Index')) resp.headers.pop('Content-Length') return resp @@ -3430,6 +3436,11 @@ class TestContainerController(unittest.TestCase): self.assertIn('X-Backend-Record-Type', resp.headers) self.assertEqual( 'shard', resp.headers.pop('X-Backend-Record-Type')) + self.assertEqual( + str(POLICIES.default.idx), + resp.headers.pop('X-Backend-Storage-Policy-Index')) + self.assertNotIn('X-Backend-Record-Storage-Policy-Index', + resp.headers) resp.headers.pop('Content-Length') return resp @@ -4239,6 +4250,119 @@ class TestContainerController(unittest.TestCase): resp = req.get_response(self.controller) self.assertEqual(resp.status_int, 406) + @patch_policies([ + StoragePolicy(0, name='nulo', is_default=True), + StoragePolicy(1, name='unu'), + StoragePolicy(2, name='du'), + ]) + def test_GET_objects_of_different_policies(self): + # make a container + req = Request.blank( + '/sda1/p/a/c', environ={'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '0'}) + resp = req.get_response(self.controller) + resp_policy_idx = resp.headers['X-Backend-Storage-Policy-Index'] + self.assertEqual(resp_policy_idx, str(POLICIES.default.idx)) + + pol_def_objs = ['obj_default_%d' % i for i in range(11)] + pol_1_objs = ['obj_1_%d' % i for i in range(10)] + + # fill the container + for obj in pol_def_objs: + req = Request.blank( + '/sda1/p/a/c/%s' % obj, + environ={ + 'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0}) + self._update_object_put_headers(req) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + + for obj in pol_1_objs: + req = Request.blank( + '/sda1/p/a/c/%s' % obj, + environ={ + 'REQUEST_METHOD': 'PUT', + 'HTTP_X_TIMESTAMP': '1', + 'HTTP_X_CONTENT_TYPE': 'text/plain', + 'HTTP_X_ETAG': 'x', + 'HTTP_X_SIZE': 0, + 'HTTP_X_BACKEND_STORAGE_POLICY_INDEX': 1}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 201) + + expected_pol_def_objs = [o.encode('utf8') for o in pol_def_objs] + expected_pol_1_objs = [o.encode('utf8') for o in pol_1_objs] + + # By default the container server will return objects belonging to + # the brokers storage policy + req = Request.blank( + '/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + result = [o for o in resp.body.split(b'\n') if o] + self.assertEqual(len(result), 11) + self.assertEqual(sorted(result), sorted(expected_pol_def_objs)) + self.assertIn('X-Backend-Storage-Policy-Index', resp.headers) + self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index']) + self.assertEqual('0', + resp.headers['X-Backend-Record-Storage-Policy-Index']) + + # If we specify the policy 0 idx we should get the same + req = Request.blank( + '/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.headers['X-Backend-Storage-Policy-Index'] = POLICIES.default.idx + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + result = [o for o in resp.body.split(b'\n') if o] + self.assertEqual(len(result), 11) + self.assertEqual(sorted(result), sorted(expected_pol_def_objs)) + self.assertIn('X-Backend-Storage-Policy-Index', resp.headers) + self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index']) + self.assertEqual('0', + resp.headers['X-Backend-Record-Storage-Policy-Index']) + + # And if we specify a different idx we'll get objects for that policy + # and the X-Backend-Record-Storage-Policy-Index letting us know the + # policy for which these objects came from, if it differs from the + # policy stored in the DB. + req = Request.blank( + '/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.headers['X-Backend-Storage-Policy-Index'] = 1 + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 200) + result = [o for o in resp.body.split(b'\n') if o] + self.assertEqual(len(result), 10) + self.assertEqual(sorted(result), sorted(expected_pol_1_objs)) + self.assertIn('X-Backend-Storage-Policy-Index', resp.headers) + self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index']) + self.assertEqual('1', + resp.headers['X-Backend-Record-Storage-Policy-Index']) + + # And an index that the broker doesn't have any objects for + req = Request.blank( + '/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.headers['X-Backend-Storage-Policy-Index'] = 2 + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 204) + result = [o for o in resp.body.split(b'\n') if o] + self.assertEqual(len(result), 0) + self.assertFalse(result) + self.assertIn('X-Backend-Storage-Policy-Index', resp.headers) + self.assertEqual('0', resp.headers['X-Backend-Storage-Policy-Index']) + self.assertEqual('2', + resp.headers['X-Backend-Record-Storage-Policy-Index']) + + # And an index that doesn't exist in POLICIES + req = Request.blank( + '/sda1/p/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.headers['X-Backend-Storage-Policy-Index'] = 3 + resp = req.get_response(self.controller) + self.assertEqual(resp.status_int, 400) + def test_GET_limit(self): # make a container req = Request.blank( diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py index e7eb8cee3a..3091cf1fad 100644 --- a/test/unit/proxy/controllers/test_container.py +++ b/test/unit/proxy/controllers/test_container.py @@ -610,25 +610,30 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', limit=str(limit), states='listing')), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xd1\xb0', end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2])))), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xe2\xa8\x83', end_marker='', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2] + sr_objs[3])))), # 200 @@ -653,13 +658,19 @@ class TestContainerController(TestRingBase): # path, headers, params ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (shard_ranges[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', limit=str(limit), states='listing')), # 200 - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (shard_ranges[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # 200 - (root_range.name, {'X-Backend-Record-Type': 'object'}, + (root_range.name, + {'X-Backend-Record-Type': 'object', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='', limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 ] @@ -685,27 +696,32 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing', reverse='true', limit='')), (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='\xf0\x9f\x8c\xb4', states='listing', reverse='true', limit=str(limit))), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xf0\x9f\x8c\xb5', end_marker='\xe2\x98\x83', states='listing', reverse='true', limit=str(limit - len(sr_objs[4])))), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='\xe2\x98\x84', end_marker='pie', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3])))), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='q', end_marker='ham', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2])))), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), - {'X-Backend-Record-Type': 'auto'}, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='i', end_marker='', states='listing', reverse='true', limit=str(limit - len(sr_objs[4] + sr_objs[3] + sr_objs[2] + sr_objs[1])))), # 200 @@ -735,15 +751,18 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(limit=str(limit), states='listing')), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), ] @@ -771,15 +790,18 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(marker=marker, states='listing')), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit))), (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker='\xf0\x9f\x8c\xb4\x00', states='listing', limit=str(limit))), (wsgi_quote(str_to_wsgi(shard_ranges[4].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='\xe2\xa8\x83', end_marker='', states='listing', limit=str(limit - len(sr_objs[3][2:])))), ] @@ -809,28 +831,34 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(end_marker=end_marker, states='listing')), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[0].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), - {'X-Backend-Record-Type': 'auto'}, # 404 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), (wsgi_quote(str_to_wsgi(shard_ranges[2].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='p', end_marker='\xe2\x98\x83\x00', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))), (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, # 404 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(marker='\xd1\xb0', end_marker=end_marker, states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2])))), (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker='\xd1\xb0', end_marker=end_marker, states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1] + sr_objs[2])))), @@ -856,7 +884,8 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(prefix=prefix, states='listing')), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[1].name)), - {'X-Backend-Record-Type': 'auto'}, # 404 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 404 dict(prefix=prefix, marker='', end_marker='pie\x00', states='listing', limit=str(limit))), ] @@ -877,7 +906,8 @@ class TestContainerController(TestRingBase): dict(states='listing', limit=str(limit), marker=marker, end_marker=end_marker)), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=marker, end_marker=end_marker, states='listing', limit=str(limit))), ] @@ -898,7 +928,8 @@ class TestContainerController(TestRingBase): dict(marker=end_marker, reverse='true', end_marker=marker, limit=str(limit), states='listing',)), # 200 (wsgi_quote(str_to_wsgi(shard_ranges[3].name)), - {'X-Backend-Record-Type': 'auto'}, # 200 + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, # 200 dict(marker=end_marker, end_marker=marker, states='listing', limit=str(limit), reverse='true')), ] @@ -1720,28 +1751,160 @@ class TestContainerController(TestRingBase): ('a/c', {'X-Backend-Record-Type': 'auto'}, dict(states='listing')), # 200 # get first shard objects - (shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (shard_ranges[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='', end_marker='ham\x00', states='listing', limit=str(limit))), # 200 # get second shard sub-shard ranges - (shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (shard_ranges[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='pie\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # get first sub-shard objects - (sub_shard_ranges[0].name, {'X-Backend-Record-Type': 'auto'}, + (sub_shard_ranges[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='h', end_marker='juice\x00', states='listing', limit=str(limit - len(sr_objs[0])))), # get second sub-shard objects - (sub_shard_ranges[1].name, {'X-Backend-Record-Type': 'auto'}, + (sub_shard_ranges[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='j', end_marker='lemon\x00', states='listing', limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))), # get remainder of first shard objects - (shard_ranges[1].name, {'X-Backend-Record-Type': 'object'}, + (shard_ranges[1].name, + {'X-Backend-Record-Type': 'object', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='l', end_marker='pie\x00', limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1])))), # 200 # get third shard objects - (shard_ranges[2].name, {'X-Backend-Record-Type': 'auto'}, + (shard_ranges[2].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='p', end_marker='', states='listing', + limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 + ] + expected_objects = ( + sr_objs[0] + sub_sr_objs[0] + sub_sr_objs[1] + + sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):] + sr_objs[2]) + resp = self._check_GET_shard_listing( + mock_responses, expected_objects, expected_requests) + # root object count will overridden by actual length of listing + self.check_response(resp, root_resp_hdrs) + + @patch_policies([ + StoragePolicy(0, 'zero', True, object_ring=FakeRing()), + StoragePolicy(1, 'one', False, object_ring=FakeRing()) + ]) + def test_GET_sharded_container_sharding_shard_mixed_policies(self): + # scenario: one shard is in process of sharding, shards have different + # policy than root, expect listing to always request root policy index + shard_bounds = (('', 'ham'), ('ham', 'pie'), ('pie', '')) + shard_ranges = [ + ShardRange('.shards_a/c_' + upper, Timestamp.now(), lower, upper) + for lower, upper in shard_bounds] + sr_dicts = [dict(sr) for sr in shard_ranges] + sr_objs = [self._make_shard_objects(sr) for sr in shard_ranges] + shard_resp_hdrs = [ + {'X-Backend-Sharding-State': 'unsharded', + 'X-Container-Object-Count': len(sr_objs[i]), + 'X-Container-Bytes-Used': + sum([obj['bytes'] for obj in sr_objs[i]]), + 'X-Container-Meta-Flavour': 'flavour%d' % i, + 'X-Backend-Storage-Policy-Index': 1, + 'X-Backend-Record-Storage-Policy-Index': 0} + for i in range(3)] + shard_1_shard_resp_hdrs = dict(shard_resp_hdrs[1]) + shard_1_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + + # second shard is sharding and has cleaved two out of three sub shards + shard_resp_hdrs[1]['X-Backend-Sharding-State'] = 'sharding' + sub_shard_bounds = (('ham', 'juice'), ('juice', 'lemon')) + sub_shard_ranges = [ + ShardRange('a/c_sub_' + upper, Timestamp.now(), lower, upper) + for lower, upper in sub_shard_bounds] + sub_sr_dicts = [dict(sr) for sr in sub_shard_ranges] + sub_sr_objs = [self._make_shard_objects(sr) for sr in sub_shard_ranges] + sub_shard_resp_hdrs = [ + {'X-Backend-Sharding-State': 'unsharded', + 'X-Container-Object-Count': len(sub_sr_objs[i]), + 'X-Container-Bytes-Used': + sum([obj['bytes'] for obj in sub_sr_objs[i]]), + 'X-Container-Meta-Flavour': 'flavour%d' % i, + 'X-Backend-Storage-Policy-Index': 1, + 'X-Backend-Record-Storage-Policy-Index': 0} + for i in range(2)] + + all_objects = [] + for objects in sr_objs: + all_objects.extend(objects) + size_all_objects = sum([obj['bytes'] for obj in all_objects]) + num_all_objects = len(all_objects) + limit = CONTAINER_LISTING_LIMIT + root_resp_hdrs = {'X-Backend-Sharding-State': 'sharded', + 'X-Backend-Timestamp': '99', + 'X-Container-Object-Count': num_all_objects, + 'X-Container-Bytes-Used': size_all_objects, + 'X-Container-Meta-Flavour': 'peach', + 'X-Backend-Storage-Policy-Index': 0} + root_shard_resp_hdrs = dict(root_resp_hdrs) + root_shard_resp_hdrs['X-Backend-Record-Type'] = 'shard' + + mock_responses = [ + # status, body, headers + (200, sr_dicts, root_shard_resp_hdrs), + (200, sr_objs[0], shard_resp_hdrs[0]), + (200, sub_sr_dicts + [sr_dicts[1]], shard_1_shard_resp_hdrs), + (200, sub_sr_objs[0], sub_shard_resp_hdrs[0]), + (200, sub_sr_objs[1], sub_shard_resp_hdrs[1]), + (200, sr_objs[1][len(sub_sr_objs[0] + sub_sr_objs[1]):], + shard_resp_hdrs[1]), + (200, sr_objs[2], shard_resp_hdrs[2]) + ] + # NB marker always advances to last object name + expected_requests = [ + # get root shard ranges + ('a/c', {'X-Backend-Record-Type': 'auto'}, + dict(states='listing')), # 200 + # get first shard objects + (shard_ranges[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='', end_marker='ham\x00', states='listing', + limit=str(limit))), # 200 + # get second shard sub-shard ranges + (shard_ranges[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='h', end_marker='pie\x00', states='listing', + limit=str(limit - len(sr_objs[0])))), + # get first sub-shard objects + (sub_shard_ranges[0].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='h', end_marker='juice\x00', states='listing', + limit=str(limit - len(sr_objs[0])))), + # get second sub-shard objects + (sub_shard_ranges[1].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='j', end_marker='lemon\x00', states='listing', + limit=str(limit - len(sr_objs[0] + sub_sr_objs[0])))), + # get remainder of second shard objects + (shard_ranges[1].name, + {'X-Backend-Record-Type': 'object', + 'X-Backend-Storage-Policy-Index': '0'}, + dict(marker='l', end_marker='pie\x00', + limit=str(limit - len(sr_objs[0] + sub_sr_objs[0] + + sub_sr_objs[1])))), # 200 + # get third shard objects + (shard_ranges[2].name, + {'X-Backend-Record-Type': 'auto', + 'X-Backend-Storage-Policy-Index': '0'}, dict(marker='p', end_marker='', states='listing', limit=str(limit - len(sr_objs[0] + sr_objs[1])))) # 200 ] @@ -1927,6 +2090,88 @@ class TestContainerController(TestRingBase): ('delete', 'shard-listing/a/c', None, None)], self.memcache.calls) + def test_get_from_shards_add_root_spi(self): + self._setup_shard_range_stubs() + shard_resp = mock.MagicMock(status_int=204, headers={}) + + def mock_get_container_listing(self_, req, *args, **kargs): + captured_hdrs.update(req.headers) + return None, shard_resp + + # header in response -> header added to request + captured_hdrs = {} + req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) + resp = mock.MagicMock(body=self._stub_shards_dump, + headers=self.root_resp_hdrs, + request=req) + resp.headers['X-Backend-Storage-Policy-Index'] = '0' + with mock.patch('swift.proxy.controllers.container.' + 'ContainerController._get_container_listing', + mock_get_container_listing): + controller_cls, d = self.app.get_controller(req) + controller = controller_cls(self.app, **d) + controller._get_from_shards(req, resp) + + self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs) + self.assertEqual( + captured_hdrs['X-Backend-Storage-Policy-Index'], '0') + + captured_hdrs = {} + req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) + resp = mock.MagicMock(body=self._stub_shards_dump, + headers=self.root_resp_hdrs, + request=req) + resp.headers['X-Backend-Storage-Policy-Index'] = '1' + with mock.patch('swift.proxy.controllers.container.' + 'ContainerController._get_container_listing', + mock_get_container_listing): + controller_cls, d = self.app.get_controller(req) + controller = controller_cls(self.app, **d) + controller._get_from_shards(req, resp) + + self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs) + self.assertEqual( + captured_hdrs['X-Backend-Storage-Policy-Index'], '1') + + # header not added to request if not root request + captured_hdrs = {} + req = Request.blank('/v1/a/c', + environ={ + 'REQUEST_METHOD': 'GET', + 'swift.shard_listing_history': [('a', 'c')]} + ) + resp = mock.MagicMock(body=self._stub_shards_dump, + headers=self.root_resp_hdrs, + request=req) + resp.headers['X-Backend-Storage-Policy-Index'] = '0' + with mock.patch('swift.proxy.controllers.container.' + 'ContainerController._get_container_listing', + mock_get_container_listing): + controller_cls, d = self.app.get_controller(req) + controller = controller_cls(self.app, **d) + controller._get_from_shards(req, resp) + + self.assertNotIn('X-Backend-Storage-Policy-Index', captured_hdrs) + + # existing X-Backend-Storage-Policy-Index in request is respected + captured_hdrs = {} + req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'}) + req.headers['X-Backend-Storage-Policy-Index'] = '0' + resp = mock.MagicMock(body=self._stub_shards_dump, + headers=self.root_resp_hdrs, + request=req) + resp.headers['X-Backend-Storage-Policy-Index'] = '1' + with mock.patch('swift.proxy.controllers.container.' + 'ContainerController._get_container_listing', + mock_get_container_listing): + controller_cls, d = self.app.get_controller(req) + controller = controller_cls(self.app, **d) + controller._get_from_shards(req, resp) + + self.assertIn('X-Backend-Storage-Policy-Index', captured_hdrs) + self.assertEqual( + captured_hdrs['X-Backend-Storage-Policy-Index'], '0') + def test_GET_shard_ranges(self): self._setup_shard_range_stubs() # expect shard ranges cache time to be default value of 600