proxy: remove x-backend-record-type=shard in object listing

When constructing an object listing from container shards, the proxy
would previously return the X-Backend-Record-Type header with the
value 'shard' that is returned with the initial GET response from the
root container. It didn't break anything but was plainly wrong.

This patch removes the header from object listing responses to request
that did not have the header. The header value is not set to 'object'
because in a request that value specifically means 'do not recurse
into shards'.

Change-Id: I94c68e5d5625bc8b3d9cd9baa17a33bb35a7f82f

swift/proxy/controllers/container.py

@@ -342,10 +342,15 @@ class ContainerController(Controller):
         # the setter must be called for new params to update the query string.
         params = req.params
         params['format'] = 'json'
-        # x-backend-record-type may be sent via internal client e.g. from
-        # the sharder or in probe tests
-        record_type = req.headers.get('X-Backend-Record-Type', '').lower()
-        if not record_type:
+        # x-backend-record-type may be sent via internal client e.g. from the
+        # sharder, or by the proxy itself when making a recursive request, or
+        # in probe tests. If the header is present then the only values that
+        # the proxy respects are 'object' or 'shard'. However, the proxy may
+        # use the value 'auto' when making requests to container server.
+        orig_record_type = req.headers.get('X-Backend-Record-Type', '').lower()
+        if orig_record_type in ('object', 'shard'):
+            record_type = orig_record_type
+        else:
             record_type = 'auto'
             req.headers['X-Backend-Record-Type'] = 'auto'
             params['states'] = 'listing'
@@ -391,6 +396,9 @@ class ContainerController(Controller):
                resp_record_type.lower() == 'shard')):
             resp = self._get_from_shards(req, resp)
 
+        if orig_record_type not in ('object', 'shard'):
+            resp.headers.pop('X-Backend-Record-Type', None)
+
         if not config_true_value(
                 resp.headers.get('X-Backend-Cached-Results')):
             # Cache container metadata. We just made a request to a storage

test/probe/test_sharder.py

@@ -24,11 +24,12 @@ import six
 from six.moves.urllib.parse import quote
 
 from swift.common import direct_client, utils
+from swift.common.header_key_dict import HeaderKeyDict
 from swift.common.internal_client import UnexpectedResponse
 from swift.common.manager import Manager
 from swift.common.memcached import MemcacheRing
 from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \
-    quorum_size, config_true_value, Timestamp, md5
+    quorum_size, config_true_value, Timestamp, md5, Namespace
 from swift.container.backend import ContainerBroker, UNSHARDED, SHARDING, \
     SHARDED
 from swift.container.sharder import CleavingContext, ContainerSharder
@@ -173,16 +174,35 @@ class BaseTestContainerSharding(ReplProbeTest):
             else:
                 conn.delete_object(self.container_name, obj)
 
-    def get_container_shard_ranges(self, account=None, container=None,
-                                   include_deleted=False):
+    def get_container_listing(self, account=None, container=None,
+                              headers=None, params=None):
         account = account if account else self.account
         container = container if container else self.container_to_shard
         path = self.internal_client.make_path(account, container)
-        headers = {'X-Backend-Record-Type': 'shard'}
-        if include_deleted:
-            headers['X-Backend-Include-Deleted'] = 'true'
-        resp = self.internal_client.make_request(
-            'GET', path + '?format=json', headers, [200])
+        headers = headers or {}
+        return self.internal_client.make_request(
+            'GET', path + '?format=json', headers, [200], params=params)
+
+    def get_container_objects(self, account=None, container=None,
+                              headers=None, params=None):
+        headers = HeaderKeyDict(headers) if headers else {}
+        resp = self.get_container_listing(account, container, headers,
+                                          params=params)
+        req_record_type = headers.get('X-Backend-Record-Type')
+        resp_record_type = resp.headers.get('X-Backend-Record-Type')
+        if req_record_type and req_record_type.lower() == 'object':
+            self.assertEqual('object', resp_record_type)
+        else:
+            self.assertIsNone(resp_record_type)
+        return json.loads(resp.body)
+
+    def get_container_shard_ranges(self, account=None, container=None,
+                                   headers=None, params=None):
+        headers = dict(headers) if headers else {}
+        headers.update({'X-Backend-Record-Type': 'shard'})
+        resp = self.get_container_listing(account, container, headers,
+                                          params=params)
+        self.assertEqual('shard', resp.headers.get('X-Backend-Record-Type'))
         return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)]
 
     def direct_get_container_shard_ranges(self, account=None, container=None,
@@ -386,6 +406,10 @@ class BaseTestContainerSharding(ReplProbeTest):
             expected_state, headers['X-Backend-Sharding-State'])
         return [ShardRange.from_dict(sr) for sr in shard_ranges]
 
+    def assert_container_states(self, expected_state, num_shard_ranges):
+        for node in self.brain.nodes:
+            self.assert_container_state(node, expected_state, num_shard_ranges)
+
     def assert_subprocess_success(self, cmd_args):
         try:
             return subprocess.check_output(cmd_args, stderr=subprocess.STDOUT)
@@ -434,6 +458,15 @@ class BaseTestContainerSharding(ReplProbeTest):
         return self.run_custom_daemon(ContainerSharder, 'container-sharder',
                                       conf_index, custom_conf, **kwargs)
 
+    def sharders_once_non_auto(self, **kwargs):
+        # inhibit auto_sharding regardless of the config setting
+        additional_args = kwargs.get('additional_args', [])
+        if not isinstance(additional_args, list):
+            additional_args = [additional_args]
+        additional_args.append('--no-auto-shard')
+        kwargs['additional_args'] = additional_args
+        self.sharders.once(**kwargs)
+
 
 class BaseAutoContainerSharding(BaseTestContainerSharding):
 
@@ -553,8 +586,7 @@ class TestContainerShardingNonUTF8(BaseAutoContainerSharding):
                 number=n, additional_args='--partitions=%s' % self.brain.part)
 
         # sanity check shard range states
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharding', 4)
+        self.assert_container_states('sharding', 4)
         shard_ranges = self.get_container_shard_ranges()
         self.assertLengthEqual(shard_ranges, 4)
         self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2])
@@ -584,8 +616,7 @@ class TestContainerShardingNonUTF8(BaseAutoContainerSharding):
 
         # run all the sharders again and the last two shard ranges get cleaved
         self.sharders.once(additional_args='--partitions=%s' % self.brain.part)
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 4)
+        self.assert_container_states('sharded', 4)
         shard_ranges = self.get_container_shard_ranges()
         self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges)
 
@@ -839,8 +870,7 @@ class TestContainerShardingObjectVersioning(BaseAutoContainerSharding):
                 number=n, additional_args='--partitions=%s' % self.brain.part)
 
         # sanity check shard range states
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharding', 4)
+        self.assert_container_states('sharding', 4)
         shard_ranges = self.get_container_shard_ranges()
         self.assertLengthEqual(shard_ranges, 4)
         self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2])
@@ -869,8 +899,7 @@ class TestContainerShardingObjectVersioning(BaseAutoContainerSharding):
 
         # run all the sharders again and the last two shard ranges get cleaved
         self.sharders.once(additional_args='--partitions=%s' % self.brain.part)
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 4)
+        self.assert_container_states('sharded', 4)
         shard_ranges = self.get_container_shard_ranges()
         self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges)
 
@@ -1918,8 +1947,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
             self.sharders.once(
                 number=n, additional_args='--partitions=%s' % self.brain.part)
         # sanity checks
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 2)
+        self.assert_container_states('sharded', 2)
         self.assert_container_delete_fails()
         self.assert_container_has_shard_sysmeta()
         self.assert_container_post_ok('sharded')
@@ -2255,8 +2283,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
             self.sharders.once(
                 number=n, additional_args='--partitions=%s' % self.brain.part)
         # sanity checks
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 2)
+        self.assert_container_states('sharded', 2)
         self.assert_container_delete_fails()
         self.assert_container_has_shard_sysmeta()
         self.assert_container_post_ok('sharded')
@@ -2373,8 +2400,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
             self.sharders.once(
                 number=n, additional_args='--partitions=%s' % self.brain.part)
         # sanity checks
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 2)
+        self.assert_container_states('sharded', 2)
         self.assert_container_delete_fails()
         self.assert_container_has_shard_sysmeta()
         self.assert_container_post_ok('sharded')
@@ -2506,8 +2532,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
             self.sharders.once(
                 number=n, additional_args='--partitions=%s' % self.brain.part)
         # sanity checks
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 2)
+        self.assert_container_states('sharded', 2)
         self.assert_container_delete_fails()
         self.assert_container_has_shard_sysmeta()
         self.assert_container_post_ok('sharded')
@@ -2620,8 +2645,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
             self.sharders.once(
                 number=n, additional_args='--partitions=%s' % self.brain.part)
         # sanity checks
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 2)
+        self.assert_container_states('sharded', 2)
         self.assert_container_delete_fails()
         self.assert_container_has_shard_sysmeta()
         self.assert_container_post_ok('sharded')
@@ -2860,8 +2884,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
         for node in self.brain.nodes[1:]:
             self.assert_container_state(node, 'sharding', 3)
         self.sharders.once()
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 3)
+        self.assert_container_states('sharded', 3)
 
         self.assert_container_listing(obj_names)
 
@@ -2900,6 +2923,137 @@ class TestContainerSharding(BaseAutoContainerSharding):
         self.assertEqual(0, int(metadata.get('x-account-bytes-used')))
 
 
+class TestShardedAPI(BaseTestContainerSharding):
+    def _assert_namespace_equivalence(
+            self, namespaces_list, other_namespaces_list):
+        # verify given lists are equivalent when cast to Namespaces
+        self.assertEqual(len(namespaces_list), len(other_namespaces_list))
+        self.assertEqual(
+            [Namespace(sr.name, sr.lower, sr.upper)
+             for sr in namespaces_list],
+            [Namespace(sr.name, sr.lower, sr.upper)
+             for sr in other_namespaces_list])
+
+    def test_GET(self):
+        all_obj_names = self._make_object_names(10)
+        self.put_objects(all_obj_names)
+
+        # unsharded container
+        objs = self.get_container_objects()
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        objs = self.get_container_objects(
+            headers={'X-Backend-Record-Type': 'auto'})
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        objs = self.get_container_objects(
+            headers={'X-Backend-Record-Type': 'object'})
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        objs = self.get_container_objects(
+            headers={'X-Backend-Record-Type': 'banana'})
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        shard_ranges = self.get_container_shard_ranges()
+        self.assertFalse(shard_ranges)
+
+        # Shard the container
+        client.post_container(self.url, self.admin_token, self.container_name,
+                              headers={'X-Container-Sharding': 'on'})
+        self.assert_subprocess_success([
+            'swift-manage-shard-ranges',
+            self.get_db_file(self.brain.part, self.brain.nodes[0]),
+            'find_and_replace', '5', '--enable', '--minimum-shard-size', '5'])
+        self.replicators.once()
+        # "Run container-sharder on all nodes to shard the container."
+        # first pass cleaves 2 shards
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
+        # sanity check
+        self.assert_container_states('sharded', 2)
+
+        orig_shard_ranges = self.get_container_shard_ranges()
+        self.assertEqual(2, len(orig_shard_ranges))
+
+        # the container is sharded so *all* shard ranges should satisfy
+        # updating and listing state aliases
+        shard_ranges = self.get_container_shard_ranges(
+            params={'states': 'updating'})
+        self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
+
+        # XXX the states=listing param provokes the proxy to cache the backend
+        # values and then respond to the client with the cached *namespaces* !!
+        # shard_ranges = self.get_container_shard_ranges(
+        #     params={'states': 'listing'})
+        # self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
+
+        # XXX ditto...
+        # shard_ranges = self.get_container_shard_ranges(
+        #     headers={'X-Newest': 'true'},
+        #     params={'states': 'listing'})
+        # self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
+
+        # this is what the sharder requests...
+        shard_ranges = self.get_container_shard_ranges(
+            headers={'X-Newest': 'true'},
+            params={'states': 'auditing'})
+        own_ns = Namespace('%s/%s' % (self.account, self.container_name),
+                           lower='', upper='')
+        self._assert_namespace_equivalence(orig_shard_ranges + [own_ns],
+                                           shard_ranges)
+
+        shard_ranges = self.get_container_shard_ranges(
+            params={'includes': all_obj_names[1]})
+        self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges)
+
+        shard_ranges = self.get_container_shard_ranges(
+            # override 'includes'
+            headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'},
+            params={'includes': all_obj_names[1]})
+        self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
+
+        shard_ranges = self.get_container_shard_ranges(
+            params={'end_marker': all_obj_names[1]})
+        self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges)
+
+        shard_ranges = self.get_container_shard_ranges(
+            # override 'end_marker'
+            headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'},
+            params={'end_marker': all_obj_names[1]})
+        self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
+
+        shard_ranges = self.get_container_shard_ranges(
+            params={'reverse': 'true'})
+        self._assert_namespace_equivalence(list(reversed(orig_shard_ranges)),
+                                           shard_ranges)
+
+        shard_ranges = self.get_container_shard_ranges(
+            # override 'reverse'
+            headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'},
+            params={'reverse': 'true'})
+        self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
+
+        objs = self.get_container_objects()
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        objs = self.get_container_objects(
+            headers={'X-Newest': 'true'})
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        objs = self.get_container_objects(
+            headers={'X-Backend-Record-Type': 'auto'})
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        objs = self.get_container_objects(
+            headers={'X-Backend-Record-Type': 'banana'})
+        self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
+
+        # note: explicitly asking for the root object rows, but it has None
+        objs = self.get_container_objects(
+            headers={'X-Backend-Record-Type': 'object'})
+        self.assertEqual([], objs)
+
+
 class TestContainerShardingMoreUTF8(TestContainerSharding):
     def _make_object_names(self, number):
         # override default with names that include non-ascii chars
@@ -2925,16 +3079,7 @@ class TestContainerShardingMoreUTF8(TestContainerSharding):
 
 
 class TestManagedContainerSharding(BaseTestContainerSharding):
-    '''Test sharding using swift-manage-shard-ranges'''
-
-    def sharders_once(self, **kwargs):
-        # inhibit auto_sharding regardless of the config setting
-        additional_args = kwargs.get('additional_args', [])
-        if not isinstance(additional_args, list):
-            additional_args = [additional_args]
-        additional_args.append('--no-auto-shard')
-        kwargs['additional_args'] = additional_args
-        self.sharders.once(**kwargs)
+    """Test sharding using swift-manage-shard-ranges"""
 
     def test_manage_shard_ranges(self):
         obj_names = self._make_object_names(10)
@@ -2948,8 +3093,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
 
         # sanity check: we don't have nearly enough objects for this to shard
         # automatically
-        self.sharders_once(number=self.brain.node_numbers[0],
-                           additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            number=self.brain.node_numbers[0],
+            additional_args='--partitions=%s' % self.brain.part)
         self.assert_container_state(self.brain.nodes[0], 'unsharded', 0)
 
         self.assert_subprocess_success([
@@ -2962,7 +3108,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         self.replicators.once()
         # "Run container-sharder on all nodes to shard the container."
         # first pass cleaves 2 shards
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
         self.assert_container_state(self.brain.nodes[0], 'sharding', 3)
         self.assert_container_state(self.brain.nodes[1], 'sharding', 3)
         shard_ranges = self.assert_container_state(
@@ -2972,7 +3119,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # make the un-cleaved shard update the root container...
         self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges])
         shard_part, nodes = self.get_part_and_node_numbers(shard_ranges[2])
-        self.sharders_once(additional_args='--partitions=%s' % shard_part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % shard_part)
         shard_ranges = self.assert_container_state(
             self.brain.nodes[2], 'sharding', 3)
         # ...it does not report zero-stats despite being empty, because it has
@@ -2980,7 +3128,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges])
 
         # second pass cleaves final shard
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
 
         # Everybody's settled
         self.assert_container_state(self.brain.nodes[0], 'sharded', 3)
@@ -3006,11 +3155,11 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         self.assert_container_state(self.brain.nodes[0], 'unsharded', 4)
         self.replicators.once()
         # run sharders twice to cleave all 4 shard ranges
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
-        self.assert_container_state(self.brain.nodes[0], 'sharded', 4)
-        self.assert_container_state(self.brain.nodes[1], 'sharded', 4)
-        self.assert_container_state(self.brain.nodes[2], 'sharded', 4)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
+        self.assert_container_states('sharded', 4)
         self.assert_container_listing(obj_names)
 
         # now compact some ranges; use --max-shrinking to allow 2 shrinking
@@ -3025,7 +3174,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         self.assertEqual([ShardRange.SHRINKING] * 2 + [ShardRange.ACTIVE] * 2,
                          [sr.state for sr in shard_ranges])
         self.replicators.once()
-        self.sharders_once()
+        self.sharders_once_non_auto()
         # check there's now just 2 remaining shard ranges
         shard_ranges = self.assert_container_state(
             self.brain.nodes[0], 'sharded', 2)
@@ -3051,7 +3200,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         self.assertEqual([ShardRange.SHRINKING] * 2,
                          [sr.state for sr in shard_ranges])
         self.replicators.once()
-        self.sharders_once()
+        self.sharders_once_non_auto()
         self.assert_container_state(self.brain.nodes[0], 'collapsed', 0)
         self.assert_container_listing(obj_names, req_hdrs={'X-Newest': 'True'})
 
@@ -3100,8 +3249,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # proceeds to cleave shard 0.0, but after 0.0 cleaving stalls because
         # next in iteration is shard range 1.0 in FOUND state from the other
         # replica that it cannot yet cleave.
-        self.sharders_once(number=self.brain.node_numbers[0],
-                           additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            number=self.brain.node_numbers[0],
+            additional_args='--partitions=%s' % self.brain.part)
 
         # On first pass the second replica passes audit (it has its own found
         # ranges and the first replica's created shard ranges but none in the
@@ -3109,8 +3259,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # with the other replicas. All of the 7 shard ranges on this replica
         # are now in CREATED state so it proceeds to cleave the first two shard
         # ranges, 0.1 and 1.0.
-        self.sharders_once(number=self.brain.node_numbers[1],
-                           additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            number=self.brain.node_numbers[1],
+            additional_args='--partitions=%s' % self.brain.part)
         self.replicators.once()
 
         # Uh-oh
@@ -3119,9 +3270,11 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # There's a race: the third replica may be sharding, may be unsharded
 
         # Try it again a few times
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
         self.replicators.once()
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
 
         # It's not really fixing itself... the sharder audit will detect
         # overlapping ranges which prevents cleaving proceeding; expect the
@@ -3182,8 +3335,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # and the third replica may have cleaved no shards. We therefore need
         # two more passes of the sharder to get to a predictable state where
         # all replicas have cleaved all three 0.* shards.
-        self.sharders_once()
-        self.sharders_once()
+        self.sharders_once_non_auto()
+        self.sharders_once_non_auto()
 
         # now we expect all replicas to have just the three 1.* shards, with
         # the 0.* shards all deleted
@@ -3230,7 +3383,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # because of the older epoch_0 in its db filename will now start to
         # shard again with a newer epoch_1 db, and will start to re-cleave the
         # 3 active shards, albeit with zero objects to cleave.
-        self.sharders_once()
+        self.sharders_once_non_auto()
         for node in (0, 1, 2):
             with annotate_failure('node %s' % node):
                 broker = self.get_broker(self.brain.part,
@@ -3274,7 +3427,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # to exist on the same node as the root because the roots cleaving
         # process doesn't think that it created the shard db and will therefore
         # replicate it as per a normal cleave.
-        self.sharders_once()
+        self.sharders_once_non_auto()
         for node in (0, 1, 2):
             with annotate_failure('node %s' % node):
                 broker = self.get_broker(self.brain.part,
@@ -3311,11 +3464,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
             'find_and_replace', '4', '--enable'])
         self.replicators.once()
         # cleave first two shards
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
         # cleave third shard
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
         # ensure all shards learn their ACTIVE state from root
-        self.sharders_once()
+        self.sharders_once_non_auto()
         for node in (0, 1, 2):
             with annotate_failure('node %d' % node):
                 shard_ranges = self.assert_container_state(
@@ -3368,9 +3523,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         self.replicators.once()
 
         # try hard to shard the shard...
-        self.sharders_once(additional_args='--partitions=%s' % shard_1_part)
-        self.sharders_once(additional_args='--partitions=%s' % shard_1_part)
-        self.sharders_once(additional_args='--partitions=%s' % shard_1_part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % shard_1_part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % shard_1_part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % shard_1_part)
         # sharding hasn't completed and there's overlaps in the shard and root:
         # the sub-shards will have been cleaved in the order listed above, but
         # sub-shards (10 -12) and one of (12 - 14) will be overlooked because
@@ -3403,8 +3561,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
             ['swift-manage-shard-ranges', db_file, 'repair', '--yes',
              '--min-shard-age', '0'])
         self.replicators.once()
-        self.sharders_once()
-        self.sharders_once()
+        self.sharders_once_non_auto()
+        self.sharders_once_non_auto()
 
         # check root now has just 5 shard ranges
         root_shard_ranges = self.get_container_shard_ranges()
@@ -3416,7 +3574,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # because the sub-shards report their state to the root; we cannot make
         # assertions about shrunk states in shard_1's shard range table)
         root_shard_ranges = self.get_container_shard_ranges(
-            include_deleted=True)
+            headers={'X-Backend-Include-Deleted': 'true'})
         self.assertEqual(10, len(root_shard_ranges), root_shard_ranges)
         shrunk_shard_ranges = [sr for sr in root_shard_ranges
                                if sr.state == ShardRange.SHRUNK]
@@ -3452,13 +3610,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
             'find_and_replace', '2', '--enable'])
         self.container_replicators.once(
             additional_args='--partitions=%s' % self.brain.part)
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'unsharded', 2)
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.assert_container_states('unsharded', 2)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
         # get shards to update state from parent...
-        self.sharders_once()
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 2)
+        self.sharders_once_non_auto()
+        self.assert_container_states('sharded', 2)
 
         # sanity check, all is well
         msg = self.assert_subprocess_success([
@@ -3483,9 +3640,10 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
             self.assert_container_state(
                 node, 'unsharded', 2, account=shard_ranges[0].account,
                 container=shard_ranges[0].container, part=shard_part)
-        self.sharders_once(additional_args='--partitions=%s' % shard_part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % shard_part)
         # get shards to update state from parent...
-        self.sharders_once()
+        self.sharders_once_non_auto()
         for node in exclude_nodes(shard_nodes, self.brain.nodes[0]):
             self.assert_container_state(
                 node, 'sharded', 2, account=shard_ranges[0].account,
@@ -3528,7 +3686,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # TODO: how can we work around this?
         self.assertNotEqual(sub_shard_part, shard_part,
                             'You were unlucky, try again')
-        self.sharders_once(additional_args='--partitions=%s' % sub_shard_part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % sub_shard_part)
 
         # now root node 0 has the original shards plus one of the sub-shards
         # but all are active :(
@@ -3558,8 +3717,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
              for sr in root_brokers[0].get_shard_ranges(include_deleted=True)])
 
         # the transient overlap is 'fixed' in subsequent sharder cycles...
-        self.sharders_once()
-        self.sharders_once()
+        self.sharders_once_non_auto()
+        self.sharders_once_non_auto()
         self.container_replicators.once()
 
         for broker in root_brokers:
@@ -3593,13 +3752,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
             'find_and_replace', '2', '--enable'])
         self.container_replicators.once(
             additional_args='--partitions=%s' % self.brain.part)
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'unsharded', 4)
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.assert_container_states('unsharded', 4)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
         # get shards to update state from parent...
-        self.sharders_once()
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 4)
+        self.sharders_once_non_auto()
+        self.assert_container_states('sharded', 4)
 
         # sanity check, all is well
         msg = self.assert_subprocess_success([
@@ -3635,8 +3793,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
             '--yes'])
         self.assertIn(b'Repairs necessary to fill gaps.', msg)
 
-        self.sharders_once()
-        self.sharders_once()
+        self.sharders_once_non_auto()
+        self.sharders_once_non_auto()
         self.container_replicators.once()
 
         # yay! we fixed the gap (without creating an overlap)
@@ -3662,7 +3820,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         new_objs = [obj_names[4] + 'a']
         self.put_objects(new_objs)
         # get root stats up to date
-        self.sharders_once()
+        self.sharders_once_non_auto()
         # new object is in listing but old objects in the gap have been lost -
         # don't delete shard ranges!
         self.assert_container_listing(obj_names[:4] + new_objs + obj_names[6:])
@@ -3702,7 +3860,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # Run container-sharder to shard the 2 primary replicas that did
         # receive the object PUTs
         for num in self.brain.primary_numbers:
-            self.sharders_once(
+            self.sharders_once_non_auto(
                 number=num,
                 additional_args='--partitions=%s' % self.brain.part)
 
@@ -3711,7 +3869,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         self.delete_objects(obj_names)
         # deal with DELETE's being misplaced in root db's...
         for num in self.brain.primary_numbers:
-            self.sharders_once(
+            self.sharders_once_non_auto(
                 number=num,
                 additional_args='--partitions=%s' % self.brain.part)
 
@@ -3738,7 +3896,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
 
         # now shard the final DB
         for num in self.brain.handoff_numbers:
-            self.sharders_once(
+            self.sharders_once_non_auto(
                 number=num,
                 additional_args='--partitions=%s' % self.brain.part)
 
@@ -3806,7 +3964,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # Run container-sharder to shard the 2 primary replicas that did
         # receive the object PUTs
         for num in self.brain.primary_numbers:
-            self.sharders_once(
+            self.sharders_once_non_auto(
                 number=num,
                 additional_args='--partitions=%s' % self.brain.part)
 
@@ -3828,7 +3986,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
 
         # now shard the final DB
         for num in self.brain.handoff_numbers:
-            self.sharders_once(
+            self.sharders_once_non_auto(
                 number=num,
                 additional_args='--partitions=%s' % self.brain.part)
         shard_ranges = self.assert_container_state(
@@ -3876,14 +4034,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # Run container-replicator to replicate them to other nodes.
         self.container_replicators.once(
             additional_args='--partitions=%s' % self.brain.part)
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'unsharded', 2)
+        self.assert_container_states('unsharded', 2)
         # Run container-sharder on all nodes to shard the container.
-        self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
+        self.sharders_once_non_auto(
+            additional_args='--partitions=%s' % self.brain.part)
         # get shards to update state from parent...
-        self.sharders_once()
-        for node in self.brain.nodes:
-            self.assert_container_state(node, 'sharded', 2)
+        self.sharders_once_non_auto()
+        self.assert_container_states('sharded', 2)
 
         # shard first child shard into 2 grand-child-shards.
         c_shard_ranges = self.get_container_shard_ranges()
@@ -3912,14 +4069,14 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         c_shard_dir = os.path.dirname(c_shard_brokers[2].db_file)
         c_shard_tmp_dir = c_shard_dir + ".tmp"
         os.rename(c_shard_dir, c_shard_tmp_dir)
-        self.sharders_once(additional_args='--partitions=%s' %
-                           child_shard_part)
+        self.sharders_once_non_auto(additional_args='--partitions=%s' %
+                                                    child_shard_part)
         for node in c_shard_nodes[:2]:
             self.assert_container_state(
                 node, 'sharded', 2, account=c_shard_ranges[0].account,
                 container=c_shard_ranges[0].container, part=child_shard_part)
         # get updates done...
-        self.sharders_once()
+        self.sharders_once_non_auto()
 
         # shard first grand-child shard into 2 grand-grand-child-shards.
         gc_shard_ranges = self.get_container_shard_ranges(
@@ -3936,12 +4093,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
             gc_shard_ranges[0].account, gc_shard_ranges[0].container)
         self.container_replicators.once(
             additional_args='--partitions=%s' % grandchild_shard_part)
-        self.sharders_once(additional_args='--partitions=%s' %
-                           grandchild_shard_part)
+        self.sharders_once_non_auto(additional_args='--partitions=%s' %
+                                                    grandchild_shard_part)
 
         # get shards to update state from parent...
-        self.sharders_once()
-        self.sharders_once()
+        self.sharders_once_non_auto()
+        self.sharders_once_non_auto()
         self.container_replicators.once(
             additional_args='--partitions=%s' % child_shard_part)
 
@@ -3966,13 +4123,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
         # now, finally, run the sharder on the child that is still waiting to
         # shard. It will get 2 great-grandchild ranges from root to replace
         # deleted grandchild.
-        self.sharders_once(
+        self.sharders_once_non_auto(
             additional_args=['--partitions=%s' %
                              child_shard_part, '--devices=%s' %
                              c_shard_nodes[2]['device']])
         # batch size is 2 but this replicas has 3 shard ranges so we need two
         # runs of the sharder
-        self.sharders_once(
+        self.sharders_once_non_auto(
             additional_args=['--partitions=%s' %
                              child_shard_part, '--devices=%s' %
                              c_shard_nodes[2]['device']])

test/unit/container/test_sharder.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 import json
 import random
-from argparse import Namespace
+import argparse
 
 import eventlet
 import os
@@ -47,7 +47,7 @@ from swift.container.sharder import ContainerSharder, sharding_enabled, \
     update_own_shard_range_stats
 from swift.common.utils import ShardRange, Timestamp, hash_path, \
     encode_timestamps, parse_db_filename, quorum_size, Everything, md5, \
-    ShardName
+    ShardName, Namespace
 from test import annotate_failure
 
 from test.debug_logger import debug_logger
@@ -1261,6 +1261,10 @@ class TestSharder(BaseTestSharder):
         do_test('')
         do_test(json.dumps({}))
         do_test(json.dumps([{'account': 'a', 'container': 'c'}]))
+        do_test(json.dumps([dict(Namespace('a/c', 'l', 'u'))]))
+        sr_dict = dict(ShardRange('a/c', next(self.ts_iter), 'l', 'u'))
+        sr_dict.pop('object_count')
+        do_test(json.dumps([sr_dict]))
 
     def test_fetch_shard_ranges_ok(self):
         def do_test(mock_resp_body, params):
@@ -9696,11 +9700,11 @@ class TestContainerSharderConf(unittest.TestCase):
         # given namespace
         def assert_bad(conf):
             with self.assertRaises(ValueError):
-                ContainerSharderConf.validate_conf(Namespace(**conf))
+                ContainerSharderConf.validate_conf(argparse.Namespace(**conf))
 
         def assert_ok(conf):
             try:
-                ContainerSharderConf.validate_conf(Namespace(**conf))
+                ContainerSharderConf.validate_conf(argparse.Namespace(**conf))
             except ValueError as err:
                 self.fail('Unexpected ValueError: %s' % err)
 

test/unit/proxy/controllers/test_container.py

@@ -486,7 +486,7 @@ class TestContainerController(TestRingBase):
     def _check_GET_shard_listing(self, mock_responses, expected_objects,
                                  expected_requests, query_string='',
                                  reverse=False, expected_status=200,
-                                 memcache=False):
+                                 memcache=False, req_headers=None):
         # mock_responses is a list of tuples (status, json body, headers)
         # expected objects is a list of dicts
         # expected_requests is a list of tuples (path, hdrs dict, params dict)
@@ -506,6 +506,8 @@ class TestContainerController(TestRingBase):
                        for resp in mock_responses])
         exp_headers = [resp[2] for resp in mock_responses]
         request = Request.blank(container_path)
+        if req_headers:
+            request.headers.update(req_headers)
         if memcache:
             # memcache exists, which causes backend to ignore constraints and
             # reverse params for shard range GETs
@@ -566,6 +568,7 @@ class TestContainerController(TestRingBase):
                          int(resp.headers['X-Container-Object-Count']))
         self.assertEqual(exp_sharding_state,
                          resp.headers['X-Backend-Sharding-State'])
+        self.assertNotIn('X-Backend-Record-Type', resp.headers)
         for k, v in root_resp_hdrs.items():
             if k.lower().startswith('x-container-meta'):
                 self.assertEqual(v, resp.headers[k])
@@ -662,6 +665,18 @@ class TestContainerController(TestRingBase):
         self.check_response(resp, root_resp_hdrs,
                             expected_objects=expected_objects)
 
+        resp = self._check_GET_shard_listing(
+            mock_responses, expected_objects, expected_requests,
+            req_headers={'X-Backend-Record-Type': 'auto'})
+        self.check_response(resp, root_resp_hdrs,
+                            expected_objects=expected_objects)
+
+        resp = self._check_GET_shard_listing(
+            mock_responses, expected_objects, expected_requests,
+            req_headers={'X-Backend-Record-Type': 'banana'})
+        self.check_response(resp, root_resp_hdrs,
+                            expected_objects=expected_objects)
+
         # GET all objects - sharding, final shard range points back to root
         root_range = ShardRange('a/c', Timestamp.now(), 'pie', '')
         mock_responses = [
@@ -1049,6 +1064,18 @@ class TestContainerController(TestRingBase):
         self.check_response(resp, root_resp_hdrs,
                             expected_objects=expected_objects)
 
+        resp = self._check_GET_shard_listing(
+            mock_responses, expected_objects, expected_requests, memcache=True,
+            req_headers={'X-Backend-Record-Type': 'auto'})
+        self.check_response(resp, root_resp_hdrs,
+                            expected_objects=expected_objects)
+
+        resp = self._check_GET_shard_listing(
+            mock_responses, expected_objects, expected_requests, memcache=True,
+            req_headers={'X-Backend-Record-Type': 'banana'})
+        self.check_response(resp, root_resp_hdrs,
+                            expected_objects=expected_objects)
+
         # GET all objects - sharding, final shard range points back to root
         root_range = ShardRange('a/c', Timestamp.now(), 'pie', '')
         mock_responses = [
@@ -2730,9 +2757,20 @@ class TestContainerController(TestRingBase):
             'X-Container-Bytes-Used': '12',
             'X-Backend-Storage-Policy-Index': '0'}
 
-    def _do_test_caching(self, record_type, exp_recheck_listing):
+    def _do_test_caching(self, record_type, exp_recheck_listing,
+                         exp_record_type):
         # this test gets shard ranges into cache and then reads from cache
         sharding_state = 'sharded'
+        exp_resp_headers = {
+            'X-Backend-Recheck-Container-Existence': '60',
+            'X-Backend-Sharding-State': sharding_state}
+        if exp_record_type:
+            exp_resp_headers['X-Backend-Record-Type'] = exp_record_type
+        exp_cache_resp_headers = {
+            'X-Backend-Cached-Results': 'true',
+            'X-Backend-Sharding-State': sharding_state}
+        if exp_record_type:
+            exp_cache_resp_headers['X-Backend-Record-Type'] = exp_record_type
         self.memcache.delete_all()
         # container is sharded but proxy does not have that state cached;
         # expect a backend request and expect shard ranges to be cached
@@ -2749,10 +2787,7 @@ class TestContainerController(TestRingBase):
             req, backend_req,
             extra_hdrs={'X-Backend-Record-Type': record_type,
                         'X-Backend-Override-Shard-Name-Filter': 'sharded'})
-        self._check_response(resp, self.ns_dicts, {
-            'X-Backend-Recheck-Container-Existence': '60',
-            'X-Backend-Record-Type': 'shard',
-            'X-Backend-Sharding-State': sharding_state})
+        self._check_response(resp, self.ns_dicts, exp_resp_headers)
 
         cache_key = 'shard-listing-v2/a/c'
         self.assertEqual(
@@ -2789,10 +2824,7 @@ class TestContainerController(TestRingBase):
             req, backend_req,
             extra_hdrs={'X-Backend-Record-Type': record_type,
                         'X-Backend-Override-Shard-Name-Filter': 'sharded'})
-        self._check_response(resp, self.ns_dicts, {
-            'X-Backend-Recheck-Container-Existence': '60',
-            'X-Backend-Record-Type': 'shard',
-            'X-Backend-Sharding-State': sharding_state})
+        self._check_response(resp, self.ns_dicts, exp_resp_headers)
         self.assertEqual(
             [mock.call.get('container/a/c'),
              mock.call.get(cache_key, raise_on_error=True),
@@ -2819,10 +2851,7 @@ class TestContainerController(TestRingBase):
         req = self._build_request({'X-Backend-Record-Type': record_type},
                                   {'states': 'listing'}, {})
         resp = req.get_response(self.app)
-        self._check_response(resp, self.ns_dicts, {
-            'X-Backend-Cached-Results': 'true',
-            'X-Backend-Record-Type': 'shard',
-            'X-Backend-Sharding-State': sharding_state})
+        self._check_response(resp, self.ns_dicts, exp_cache_resp_headers)
         self.assertEqual(
             [mock.call.get('container/a/c'),
              mock.call.get(cache_key, raise_on_error=True)],
@@ -2853,10 +2882,7 @@ class TestContainerController(TestRingBase):
             req, backend_req,
             extra_hdrs={'X-Backend-Record-Type': record_type,
                         'X-Backend-Override-Shard-Name-Filter': 'sharded'})
-        self._check_response(resp, self.ns_dicts, {
-            'X-Backend-Recheck-Container-Existence': '60',
-            'X-Backend-Record-Type': 'shard',
-            'X-Backend-Sharding-State': sharding_state})
+        self._check_response(resp, self.ns_dicts, exp_resp_headers)
         self.assertEqual(
             [mock.call.get('container/a/c'),
              mock.call.set(cache_key, self.ns_bound_list.bounds,
@@ -2882,10 +2908,7 @@ class TestContainerController(TestRingBase):
                                   {'states': 'listing'}, {})
         with mock.patch('random.random', return_value=0.11):
             resp = req.get_response(self.app)
-        self._check_response(resp, self.ns_dicts, {
-            'X-Backend-Cached-Results': 'true',
-            'X-Backend-Record-Type': 'shard',
-            'X-Backend-Sharding-State': sharding_state})
+        self._check_response(resp, self.ns_dicts, exp_cache_resp_headers)
         self.assertEqual(
             [mock.call.get('container/a/c'),
              mock.call.get(cache_key, raise_on_error=True)],
@@ -2909,10 +2932,7 @@ class TestContainerController(TestRingBase):
             infocache=req.environ['swift.infocache'])
         with mock.patch('random.random', return_value=0.11):
             resp = req.get_response(self.app)
-        self._check_response(resp, self.ns_dicts, {
-            'X-Backend-Cached-Results': 'true',
-            'X-Backend-Record-Type': 'shard',
-            'X-Backend-Sharding-State': sharding_state})
+        self._check_response(resp, self.ns_dicts, exp_cache_resp_headers)
         self.assertEqual([], self.memcache.calls)
         self.assertIn('swift.infocache', req.environ)
         self.assertIn(cache_key, req.environ['swift.infocache'])
@@ -3025,10 +3045,10 @@ class TestContainerController(TestRingBase):
     def test_GET_shard_ranges(self):
         self._setup_shard_range_stubs()
         # expect shard ranges cache time to be default value of 600
-        self._do_test_caching('shard', 600)
+        self._do_test_caching('shard', 600, 'shard')
         # expect shard ranges cache time to be configured value of 120
         self.app.recheck_listing_shard_ranges = 120
-        self._do_test_caching('shard', 120)
+        self._do_test_caching('shard', 120, 'shard')
 
         def mock_get_from_shards(self, req, resp):
             # for the purposes of these tests we override _get_from_shards so
@@ -3042,7 +3062,7 @@ class TestContainerController(TestRingBase):
                         'ContainerController._get_from_shards',
                         mock_get_from_shards):
             self.app.recheck_listing_shard_ranges = 600
-            self._do_test_caching('auto', 600)
+            self._do_test_caching('auto', 600, exp_record_type=None)
 
     def test_GET_shard_ranges_404_response(self):
         # pre-warm cache with container info but not shard ranges so that the
@@ -3169,6 +3189,8 @@ class TestContainerController(TestRingBase):
         def mock_get_from_shards(self, req, resp):
             return resp
 
+        # request with record-type=auto does not expect record-type in response
+        del exp_hdrs['X-Backend-Record-Type']
         with mock.patch('swift.proxy.controllers.container.'
                         'ContainerController._get_from_shards',
                         mock_get_from_shards):
@@ -3264,6 +3286,8 @@ class TestContainerController(TestRingBase):
         def mock_get_from_shards(self, req, resp):
             return resp
 
+        # request with record-type=auto does not expect record-type in response
+        del exp_hdrs['X-Backend-Record-Type']
         with mock.patch('swift.proxy.controllers.container.'
                         'ContainerController._get_from_shards',
                         mock_get_from_shards):