Merge "proxy: remove x-backend-record-type=shard in object listing"
This commit is contained in:
commit
7a3124d82d
@ -342,10 +342,15 @@ class ContainerController(Controller):
|
|||||||
# the setter must be called for new params to update the query string.
|
# the setter must be called for new params to update the query string.
|
||||||
params = req.params
|
params = req.params
|
||||||
params['format'] = 'json'
|
params['format'] = 'json'
|
||||||
# x-backend-record-type may be sent via internal client e.g. from
|
# x-backend-record-type may be sent via internal client e.g. from the
|
||||||
# the sharder or in probe tests
|
# sharder, or by the proxy itself when making a recursive request, or
|
||||||
record_type = req.headers.get('X-Backend-Record-Type', '').lower()
|
# in probe tests. If the header is present then the only values that
|
||||||
if not record_type:
|
# the proxy respects are 'object' or 'shard'. However, the proxy may
|
||||||
|
# use the value 'auto' when making requests to container server.
|
||||||
|
orig_record_type = req.headers.get('X-Backend-Record-Type', '').lower()
|
||||||
|
if orig_record_type in ('object', 'shard'):
|
||||||
|
record_type = orig_record_type
|
||||||
|
else:
|
||||||
record_type = 'auto'
|
record_type = 'auto'
|
||||||
req.headers['X-Backend-Record-Type'] = 'auto'
|
req.headers['X-Backend-Record-Type'] = 'auto'
|
||||||
params['states'] = 'listing'
|
params['states'] = 'listing'
|
||||||
@ -391,6 +396,9 @@ class ContainerController(Controller):
|
|||||||
resp_record_type.lower() == 'shard')):
|
resp_record_type.lower() == 'shard')):
|
||||||
resp = self._get_from_shards(req, resp)
|
resp = self._get_from_shards(req, resp)
|
||||||
|
|
||||||
|
if orig_record_type not in ('object', 'shard'):
|
||||||
|
resp.headers.pop('X-Backend-Record-Type', None)
|
||||||
|
|
||||||
if not config_true_value(
|
if not config_true_value(
|
||||||
resp.headers.get('X-Backend-Cached-Results')):
|
resp.headers.get('X-Backend-Cached-Results')):
|
||||||
# Cache container metadata. We just made a request to a storage
|
# Cache container metadata. We just made a request to a storage
|
||||||
|
@ -24,11 +24,12 @@ import six
|
|||||||
from six.moves.urllib.parse import quote
|
from six.moves.urllib.parse import quote
|
||||||
|
|
||||||
from swift.common import direct_client, utils
|
from swift.common import direct_client, utils
|
||||||
|
from swift.common.header_key_dict import HeaderKeyDict
|
||||||
from swift.common.internal_client import UnexpectedResponse
|
from swift.common.internal_client import UnexpectedResponse
|
||||||
from swift.common.manager import Manager
|
from swift.common.manager import Manager
|
||||||
from swift.common.memcached import MemcacheRing
|
from swift.common.memcached import MemcacheRing
|
||||||
from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \
|
from swift.common.utils import ShardRange, parse_db_filename, get_db_files, \
|
||||||
quorum_size, config_true_value, Timestamp, md5
|
quorum_size, config_true_value, Timestamp, md5, Namespace
|
||||||
from swift.container.backend import ContainerBroker, UNSHARDED, SHARDING, \
|
from swift.container.backend import ContainerBroker, UNSHARDED, SHARDING, \
|
||||||
SHARDED
|
SHARDED
|
||||||
from swift.container.sharder import CleavingContext, ContainerSharder
|
from swift.container.sharder import CleavingContext, ContainerSharder
|
||||||
@ -173,16 +174,35 @@ class BaseTestContainerSharding(ReplProbeTest):
|
|||||||
else:
|
else:
|
||||||
conn.delete_object(self.container_name, obj)
|
conn.delete_object(self.container_name, obj)
|
||||||
|
|
||||||
def get_container_shard_ranges(self, account=None, container=None,
|
def get_container_listing(self, account=None, container=None,
|
||||||
include_deleted=False):
|
headers=None, params=None):
|
||||||
account = account if account else self.account
|
account = account if account else self.account
|
||||||
container = container if container else self.container_to_shard
|
container = container if container else self.container_to_shard
|
||||||
path = self.internal_client.make_path(account, container)
|
path = self.internal_client.make_path(account, container)
|
||||||
headers = {'X-Backend-Record-Type': 'shard'}
|
headers = headers or {}
|
||||||
if include_deleted:
|
return self.internal_client.make_request(
|
||||||
headers['X-Backend-Include-Deleted'] = 'true'
|
'GET', path + '?format=json', headers, [200], params=params)
|
||||||
resp = self.internal_client.make_request(
|
|
||||||
'GET', path + '?format=json', headers, [200])
|
def get_container_objects(self, account=None, container=None,
|
||||||
|
headers=None, params=None):
|
||||||
|
headers = HeaderKeyDict(headers) if headers else {}
|
||||||
|
resp = self.get_container_listing(account, container, headers,
|
||||||
|
params=params)
|
||||||
|
req_record_type = headers.get('X-Backend-Record-Type')
|
||||||
|
resp_record_type = resp.headers.get('X-Backend-Record-Type')
|
||||||
|
if req_record_type and req_record_type.lower() == 'object':
|
||||||
|
self.assertEqual('object', resp_record_type)
|
||||||
|
else:
|
||||||
|
self.assertIsNone(resp_record_type)
|
||||||
|
return json.loads(resp.body)
|
||||||
|
|
||||||
|
def get_container_shard_ranges(self, account=None, container=None,
|
||||||
|
headers=None, params=None):
|
||||||
|
headers = dict(headers) if headers else {}
|
||||||
|
headers.update({'X-Backend-Record-Type': 'shard'})
|
||||||
|
resp = self.get_container_listing(account, container, headers,
|
||||||
|
params=params)
|
||||||
|
self.assertEqual('shard', resp.headers.get('X-Backend-Record-Type'))
|
||||||
return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)]
|
return [ShardRange.from_dict(sr) for sr in json.loads(resp.body)]
|
||||||
|
|
||||||
def direct_get_container_shard_ranges(self, account=None, container=None,
|
def direct_get_container_shard_ranges(self, account=None, container=None,
|
||||||
@ -386,6 +406,10 @@ class BaseTestContainerSharding(ReplProbeTest):
|
|||||||
expected_state, headers['X-Backend-Sharding-State'])
|
expected_state, headers['X-Backend-Sharding-State'])
|
||||||
return [ShardRange.from_dict(sr) for sr in shard_ranges]
|
return [ShardRange.from_dict(sr) for sr in shard_ranges]
|
||||||
|
|
||||||
|
def assert_container_states(self, expected_state, num_shard_ranges):
|
||||||
|
for node in self.brain.nodes:
|
||||||
|
self.assert_container_state(node, expected_state, num_shard_ranges)
|
||||||
|
|
||||||
def assert_subprocess_success(self, cmd_args):
|
def assert_subprocess_success(self, cmd_args):
|
||||||
try:
|
try:
|
||||||
return subprocess.check_output(cmd_args, stderr=subprocess.STDOUT)
|
return subprocess.check_output(cmd_args, stderr=subprocess.STDOUT)
|
||||||
@ -434,6 +458,15 @@ class BaseTestContainerSharding(ReplProbeTest):
|
|||||||
return self.run_custom_daemon(ContainerSharder, 'container-sharder',
|
return self.run_custom_daemon(ContainerSharder, 'container-sharder',
|
||||||
conf_index, custom_conf, **kwargs)
|
conf_index, custom_conf, **kwargs)
|
||||||
|
|
||||||
|
def sharders_once_non_auto(self, **kwargs):
|
||||||
|
# inhibit auto_sharding regardless of the config setting
|
||||||
|
additional_args = kwargs.get('additional_args', [])
|
||||||
|
if not isinstance(additional_args, list):
|
||||||
|
additional_args = [additional_args]
|
||||||
|
additional_args.append('--no-auto-shard')
|
||||||
|
kwargs['additional_args'] = additional_args
|
||||||
|
self.sharders.once(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
class BaseAutoContainerSharding(BaseTestContainerSharding):
|
class BaseAutoContainerSharding(BaseTestContainerSharding):
|
||||||
|
|
||||||
@ -553,8 +586,7 @@ class TestContainerShardingNonUTF8(BaseAutoContainerSharding):
|
|||||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
# sanity check shard range states
|
# sanity check shard range states
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharding', 4)
|
||||||
self.assert_container_state(node, 'sharding', 4)
|
|
||||||
shard_ranges = self.get_container_shard_ranges()
|
shard_ranges = self.get_container_shard_ranges()
|
||||||
self.assertLengthEqual(shard_ranges, 4)
|
self.assertLengthEqual(shard_ranges, 4)
|
||||||
self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2])
|
self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2])
|
||||||
@ -584,8 +616,7 @@ class TestContainerShardingNonUTF8(BaseAutoContainerSharding):
|
|||||||
|
|
||||||
# run all the sharders again and the last two shard ranges get cleaved
|
# run all the sharders again and the last two shard ranges get cleaved
|
||||||
self.sharders.once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders.once(additional_args='--partitions=%s' % self.brain.part)
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 4)
|
||||||
self.assert_container_state(node, 'sharded', 4)
|
|
||||||
shard_ranges = self.get_container_shard_ranges()
|
shard_ranges = self.get_container_shard_ranges()
|
||||||
self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges)
|
self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges)
|
||||||
|
|
||||||
@ -839,8 +870,7 @@ class TestContainerShardingObjectVersioning(BaseAutoContainerSharding):
|
|||||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
# sanity check shard range states
|
# sanity check shard range states
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharding', 4)
|
||||||
self.assert_container_state(node, 'sharding', 4)
|
|
||||||
shard_ranges = self.get_container_shard_ranges()
|
shard_ranges = self.get_container_shard_ranges()
|
||||||
self.assertLengthEqual(shard_ranges, 4)
|
self.assertLengthEqual(shard_ranges, 4)
|
||||||
self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2])
|
self.assert_shard_range_state(ShardRange.CLEAVED, shard_ranges[:2])
|
||||||
@ -869,8 +899,7 @@ class TestContainerShardingObjectVersioning(BaseAutoContainerSharding):
|
|||||||
|
|
||||||
# run all the sharders again and the last two shard ranges get cleaved
|
# run all the sharders again and the last two shard ranges get cleaved
|
||||||
self.sharders.once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders.once(additional_args='--partitions=%s' % self.brain.part)
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 4)
|
||||||
self.assert_container_state(node, 'sharded', 4)
|
|
||||||
shard_ranges = self.get_container_shard_ranges()
|
shard_ranges = self.get_container_shard_ranges()
|
||||||
self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges)
|
self.assert_shard_range_state(ShardRange.ACTIVE, shard_ranges)
|
||||||
|
|
||||||
@ -1918,8 +1947,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
|
|||||||
self.sharders.once(
|
self.sharders.once(
|
||||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||||
# sanity checks
|
# sanity checks
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 2)
|
||||||
self.assert_container_state(node, 'sharded', 2)
|
|
||||||
self.assert_container_delete_fails()
|
self.assert_container_delete_fails()
|
||||||
self.assert_container_has_shard_sysmeta()
|
self.assert_container_has_shard_sysmeta()
|
||||||
self.assert_container_post_ok('sharded')
|
self.assert_container_post_ok('sharded')
|
||||||
@ -2255,8 +2283,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
|
|||||||
self.sharders.once(
|
self.sharders.once(
|
||||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||||
# sanity checks
|
# sanity checks
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 2)
|
||||||
self.assert_container_state(node, 'sharded', 2)
|
|
||||||
self.assert_container_delete_fails()
|
self.assert_container_delete_fails()
|
||||||
self.assert_container_has_shard_sysmeta()
|
self.assert_container_has_shard_sysmeta()
|
||||||
self.assert_container_post_ok('sharded')
|
self.assert_container_post_ok('sharded')
|
||||||
@ -2373,8 +2400,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
|
|||||||
self.sharders.once(
|
self.sharders.once(
|
||||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||||
# sanity checks
|
# sanity checks
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 2)
|
||||||
self.assert_container_state(node, 'sharded', 2)
|
|
||||||
self.assert_container_delete_fails()
|
self.assert_container_delete_fails()
|
||||||
self.assert_container_has_shard_sysmeta()
|
self.assert_container_has_shard_sysmeta()
|
||||||
self.assert_container_post_ok('sharded')
|
self.assert_container_post_ok('sharded')
|
||||||
@ -2506,8 +2532,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
|
|||||||
self.sharders.once(
|
self.sharders.once(
|
||||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||||
# sanity checks
|
# sanity checks
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 2)
|
||||||
self.assert_container_state(node, 'sharded', 2)
|
|
||||||
self.assert_container_delete_fails()
|
self.assert_container_delete_fails()
|
||||||
self.assert_container_has_shard_sysmeta()
|
self.assert_container_has_shard_sysmeta()
|
||||||
self.assert_container_post_ok('sharded')
|
self.assert_container_post_ok('sharded')
|
||||||
@ -2620,8 +2645,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
|
|||||||
self.sharders.once(
|
self.sharders.once(
|
||||||
number=n, additional_args='--partitions=%s' % self.brain.part)
|
number=n, additional_args='--partitions=%s' % self.brain.part)
|
||||||
# sanity checks
|
# sanity checks
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 2)
|
||||||
self.assert_container_state(node, 'sharded', 2)
|
|
||||||
self.assert_container_delete_fails()
|
self.assert_container_delete_fails()
|
||||||
self.assert_container_has_shard_sysmeta()
|
self.assert_container_has_shard_sysmeta()
|
||||||
self.assert_container_post_ok('sharded')
|
self.assert_container_post_ok('sharded')
|
||||||
@ -2860,8 +2884,7 @@ class TestContainerSharding(BaseAutoContainerSharding):
|
|||||||
for node in self.brain.nodes[1:]:
|
for node in self.brain.nodes[1:]:
|
||||||
self.assert_container_state(node, 'sharding', 3)
|
self.assert_container_state(node, 'sharding', 3)
|
||||||
self.sharders.once()
|
self.sharders.once()
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 3)
|
||||||
self.assert_container_state(node, 'sharded', 3)
|
|
||||||
|
|
||||||
self.assert_container_listing(obj_names)
|
self.assert_container_listing(obj_names)
|
||||||
|
|
||||||
@ -2900,6 +2923,137 @@ class TestContainerSharding(BaseAutoContainerSharding):
|
|||||||
self.assertEqual(0, int(metadata.get('x-account-bytes-used')))
|
self.assertEqual(0, int(metadata.get('x-account-bytes-used')))
|
||||||
|
|
||||||
|
|
||||||
|
class TestShardedAPI(BaseTestContainerSharding):
|
||||||
|
def _assert_namespace_equivalence(
|
||||||
|
self, namespaces_list, other_namespaces_list):
|
||||||
|
# verify given lists are equivalent when cast to Namespaces
|
||||||
|
self.assertEqual(len(namespaces_list), len(other_namespaces_list))
|
||||||
|
self.assertEqual(
|
||||||
|
[Namespace(sr.name, sr.lower, sr.upper)
|
||||||
|
for sr in namespaces_list],
|
||||||
|
[Namespace(sr.name, sr.lower, sr.upper)
|
||||||
|
for sr in other_namespaces_list])
|
||||||
|
|
||||||
|
def test_GET(self):
|
||||||
|
all_obj_names = self._make_object_names(10)
|
||||||
|
self.put_objects(all_obj_names)
|
||||||
|
|
||||||
|
# unsharded container
|
||||||
|
objs = self.get_container_objects()
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
objs = self.get_container_objects(
|
||||||
|
headers={'X-Backend-Record-Type': 'auto'})
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
objs = self.get_container_objects(
|
||||||
|
headers={'X-Backend-Record-Type': 'object'})
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
objs = self.get_container_objects(
|
||||||
|
headers={'X-Backend-Record-Type': 'banana'})
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
shard_ranges = self.get_container_shard_ranges()
|
||||||
|
self.assertFalse(shard_ranges)
|
||||||
|
|
||||||
|
# Shard the container
|
||||||
|
client.post_container(self.url, self.admin_token, self.container_name,
|
||||||
|
headers={'X-Container-Sharding': 'on'})
|
||||||
|
self.assert_subprocess_success([
|
||||||
|
'swift-manage-shard-ranges',
|
||||||
|
self.get_db_file(self.brain.part, self.brain.nodes[0]),
|
||||||
|
'find_and_replace', '5', '--enable', '--minimum-shard-size', '5'])
|
||||||
|
self.replicators.once()
|
||||||
|
# "Run container-sharder on all nodes to shard the container."
|
||||||
|
# first pass cleaves 2 shards
|
||||||
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
# sanity check
|
||||||
|
self.assert_container_states('sharded', 2)
|
||||||
|
|
||||||
|
orig_shard_ranges = self.get_container_shard_ranges()
|
||||||
|
self.assertEqual(2, len(orig_shard_ranges))
|
||||||
|
|
||||||
|
# the container is sharded so *all* shard ranges should satisfy
|
||||||
|
# updating and listing state aliases
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
params={'states': 'updating'})
|
||||||
|
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||||
|
|
||||||
|
# XXX the states=listing param provokes the proxy to cache the backend
|
||||||
|
# values and then respond to the client with the cached *namespaces* !!
|
||||||
|
# shard_ranges = self.get_container_shard_ranges(
|
||||||
|
# params={'states': 'listing'})
|
||||||
|
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||||
|
|
||||||
|
# XXX ditto...
|
||||||
|
# shard_ranges = self.get_container_shard_ranges(
|
||||||
|
# headers={'X-Newest': 'true'},
|
||||||
|
# params={'states': 'listing'})
|
||||||
|
# self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||||
|
|
||||||
|
# this is what the sharder requests...
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
headers={'X-Newest': 'true'},
|
||||||
|
params={'states': 'auditing'})
|
||||||
|
own_ns = Namespace('%s/%s' % (self.account, self.container_name),
|
||||||
|
lower='', upper='')
|
||||||
|
self._assert_namespace_equivalence(orig_shard_ranges + [own_ns],
|
||||||
|
shard_ranges)
|
||||||
|
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
params={'includes': all_obj_names[1]})
|
||||||
|
self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges)
|
||||||
|
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
# override 'includes'
|
||||||
|
headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'},
|
||||||
|
params={'includes': all_obj_names[1]})
|
||||||
|
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||||
|
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
params={'end_marker': all_obj_names[1]})
|
||||||
|
self._assert_namespace_equivalence(orig_shard_ranges[:1], shard_ranges)
|
||||||
|
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
# override 'end_marker'
|
||||||
|
headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'},
|
||||||
|
params={'end_marker': all_obj_names[1]})
|
||||||
|
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||||
|
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
params={'reverse': 'true'})
|
||||||
|
self._assert_namespace_equivalence(list(reversed(orig_shard_ranges)),
|
||||||
|
shard_ranges)
|
||||||
|
|
||||||
|
shard_ranges = self.get_container_shard_ranges(
|
||||||
|
# override 'reverse'
|
||||||
|
headers={'X-Backend-Override-Shard-Name-Filter': 'sharded'},
|
||||||
|
params={'reverse': 'true'})
|
||||||
|
self._assert_namespace_equivalence(orig_shard_ranges, shard_ranges)
|
||||||
|
|
||||||
|
objs = self.get_container_objects()
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
objs = self.get_container_objects(
|
||||||
|
headers={'X-Newest': 'true'})
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
objs = self.get_container_objects(
|
||||||
|
headers={'X-Backend-Record-Type': 'auto'})
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
objs = self.get_container_objects(
|
||||||
|
headers={'X-Backend-Record-Type': 'banana'})
|
||||||
|
self.assertEqual(all_obj_names, [obj['name'] for obj in objs])
|
||||||
|
|
||||||
|
# note: explicitly asking for the root object rows, but it has None
|
||||||
|
objs = self.get_container_objects(
|
||||||
|
headers={'X-Backend-Record-Type': 'object'})
|
||||||
|
self.assertEqual([], objs)
|
||||||
|
|
||||||
|
|
||||||
class TestContainerShardingMoreUTF8(TestContainerSharding):
|
class TestContainerShardingMoreUTF8(TestContainerSharding):
|
||||||
def _make_object_names(self, number):
|
def _make_object_names(self, number):
|
||||||
# override default with names that include non-ascii chars
|
# override default with names that include non-ascii chars
|
||||||
@ -2925,16 +3079,7 @@ class TestContainerShardingMoreUTF8(TestContainerSharding):
|
|||||||
|
|
||||||
|
|
||||||
class TestManagedContainerSharding(BaseTestContainerSharding):
|
class TestManagedContainerSharding(BaseTestContainerSharding):
|
||||||
'''Test sharding using swift-manage-shard-ranges'''
|
"""Test sharding using swift-manage-shard-ranges"""
|
||||||
|
|
||||||
def sharders_once(self, **kwargs):
|
|
||||||
# inhibit auto_sharding regardless of the config setting
|
|
||||||
additional_args = kwargs.get('additional_args', [])
|
|
||||||
if not isinstance(additional_args, list):
|
|
||||||
additional_args = [additional_args]
|
|
||||||
additional_args.append('--no-auto-shard')
|
|
||||||
kwargs['additional_args'] = additional_args
|
|
||||||
self.sharders.once(**kwargs)
|
|
||||||
|
|
||||||
def test_manage_shard_ranges(self):
|
def test_manage_shard_ranges(self):
|
||||||
obj_names = self._make_object_names(10)
|
obj_names = self._make_object_names(10)
|
||||||
@ -2948,8 +3093,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
|
|
||||||
# sanity check: we don't have nearly enough objects for this to shard
|
# sanity check: we don't have nearly enough objects for this to shard
|
||||||
# automatically
|
# automatically
|
||||||
self.sharders_once(number=self.brain.node_numbers[0],
|
self.sharders_once_non_auto(
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
number=self.brain.node_numbers[0],
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
self.assert_container_state(self.brain.nodes[0], 'unsharded', 0)
|
self.assert_container_state(self.brain.nodes[0], 'unsharded', 0)
|
||||||
|
|
||||||
self.assert_subprocess_success([
|
self.assert_subprocess_success([
|
||||||
@ -2962,7 +3108,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
# "Run container-sharder on all nodes to shard the container."
|
# "Run container-sharder on all nodes to shard the container."
|
||||||
# first pass cleaves 2 shards
|
# first pass cleaves 2 shards
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
self.assert_container_state(self.brain.nodes[0], 'sharding', 3)
|
self.assert_container_state(self.brain.nodes[0], 'sharding', 3)
|
||||||
self.assert_container_state(self.brain.nodes[1], 'sharding', 3)
|
self.assert_container_state(self.brain.nodes[1], 'sharding', 3)
|
||||||
shard_ranges = self.assert_container_state(
|
shard_ranges = self.assert_container_state(
|
||||||
@ -2972,7 +3119,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# make the un-cleaved shard update the root container...
|
# make the un-cleaved shard update the root container...
|
||||||
self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges])
|
self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges])
|
||||||
shard_part, nodes = self.get_part_and_node_numbers(shard_ranges[2])
|
shard_part, nodes = self.get_part_and_node_numbers(shard_ranges[2])
|
||||||
self.sharders_once(additional_args='--partitions=%s' % shard_part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % shard_part)
|
||||||
shard_ranges = self.assert_container_state(
|
shard_ranges = self.assert_container_state(
|
||||||
self.brain.nodes[2], 'sharding', 3)
|
self.brain.nodes[2], 'sharding', 3)
|
||||||
# ...it does not report zero-stats despite being empty, because it has
|
# ...it does not report zero-stats despite being empty, because it has
|
||||||
@ -2980,7 +3128,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges])
|
self.assertEqual([3, 3, 4], [sr.object_count for sr in shard_ranges])
|
||||||
|
|
||||||
# second pass cleaves final shard
|
# second pass cleaves final shard
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
# Everybody's settled
|
# Everybody's settled
|
||||||
self.assert_container_state(self.brain.nodes[0], 'sharded', 3)
|
self.assert_container_state(self.brain.nodes[0], 'sharded', 3)
|
||||||
@ -3006,11 +3155,11 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.assert_container_state(self.brain.nodes[0], 'unsharded', 4)
|
self.assert_container_state(self.brain.nodes[0], 'unsharded', 4)
|
||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
# run sharders twice to cleave all 4 shard ranges
|
# run sharders twice to cleave all 4 shard ranges
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
self.assert_container_state(self.brain.nodes[0], 'sharded', 4)
|
self.sharders_once_non_auto(
|
||||||
self.assert_container_state(self.brain.nodes[1], 'sharded', 4)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
self.assert_container_state(self.brain.nodes[2], 'sharded', 4)
|
self.assert_container_states('sharded', 4)
|
||||||
self.assert_container_listing(obj_names)
|
self.assert_container_listing(obj_names)
|
||||||
|
|
||||||
# now compact some ranges; use --max-shrinking to allow 2 shrinking
|
# now compact some ranges; use --max-shrinking to allow 2 shrinking
|
||||||
@ -3025,7 +3174,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.assertEqual([ShardRange.SHRINKING] * 2 + [ShardRange.ACTIVE] * 2,
|
self.assertEqual([ShardRange.SHRINKING] * 2 + [ShardRange.ACTIVE] * 2,
|
||||||
[sr.state for sr in shard_ranges])
|
[sr.state for sr in shard_ranges])
|
||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
# check there's now just 2 remaining shard ranges
|
# check there's now just 2 remaining shard ranges
|
||||||
shard_ranges = self.assert_container_state(
|
shard_ranges = self.assert_container_state(
|
||||||
self.brain.nodes[0], 'sharded', 2)
|
self.brain.nodes[0], 'sharded', 2)
|
||||||
@ -3051,7 +3200,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.assertEqual([ShardRange.SHRINKING] * 2,
|
self.assertEqual([ShardRange.SHRINKING] * 2,
|
||||||
[sr.state for sr in shard_ranges])
|
[sr.state for sr in shard_ranges])
|
||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.assert_container_state(self.brain.nodes[0], 'collapsed', 0)
|
self.assert_container_state(self.brain.nodes[0], 'collapsed', 0)
|
||||||
self.assert_container_listing(obj_names, req_hdrs={'X-Newest': 'True'})
|
self.assert_container_listing(obj_names, req_hdrs={'X-Newest': 'True'})
|
||||||
|
|
||||||
@ -3100,8 +3249,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# proceeds to cleave shard 0.0, but after 0.0 cleaving stalls because
|
# proceeds to cleave shard 0.0, but after 0.0 cleaving stalls because
|
||||||
# next in iteration is shard range 1.0 in FOUND state from the other
|
# next in iteration is shard range 1.0 in FOUND state from the other
|
||||||
# replica that it cannot yet cleave.
|
# replica that it cannot yet cleave.
|
||||||
self.sharders_once(number=self.brain.node_numbers[0],
|
self.sharders_once_non_auto(
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
number=self.brain.node_numbers[0],
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
# On first pass the second replica passes audit (it has its own found
|
# On first pass the second replica passes audit (it has its own found
|
||||||
# ranges and the first replica's created shard ranges but none in the
|
# ranges and the first replica's created shard ranges but none in the
|
||||||
@ -3109,8 +3259,9 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# with the other replicas. All of the 7 shard ranges on this replica
|
# with the other replicas. All of the 7 shard ranges on this replica
|
||||||
# are now in CREATED state so it proceeds to cleave the first two shard
|
# are now in CREATED state so it proceeds to cleave the first two shard
|
||||||
# ranges, 0.1 and 1.0.
|
# ranges, 0.1 and 1.0.
|
||||||
self.sharders_once(number=self.brain.node_numbers[1],
|
self.sharders_once_non_auto(
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
number=self.brain.node_numbers[1],
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
|
|
||||||
# Uh-oh
|
# Uh-oh
|
||||||
@ -3119,9 +3270,11 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# There's a race: the third replica may be sharding, may be unsharded
|
# There's a race: the third replica may be sharding, may be unsharded
|
||||||
|
|
||||||
# Try it again a few times
|
# Try it again a few times
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
# It's not really fixing itself... the sharder audit will detect
|
# It's not really fixing itself... the sharder audit will detect
|
||||||
# overlapping ranges which prevents cleaving proceeding; expect the
|
# overlapping ranges which prevents cleaving proceeding; expect the
|
||||||
@ -3182,8 +3335,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# and the third replica may have cleaved no shards. We therefore need
|
# and the third replica may have cleaved no shards. We therefore need
|
||||||
# two more passes of the sharder to get to a predictable state where
|
# two more passes of the sharder to get to a predictable state where
|
||||||
# all replicas have cleaved all three 0.* shards.
|
# all replicas have cleaved all three 0.* shards.
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
|
|
||||||
# now we expect all replicas to have just the three 1.* shards, with
|
# now we expect all replicas to have just the three 1.* shards, with
|
||||||
# the 0.* shards all deleted
|
# the 0.* shards all deleted
|
||||||
@ -3230,7 +3383,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# because of the older epoch_0 in its db filename will now start to
|
# because of the older epoch_0 in its db filename will now start to
|
||||||
# shard again with a newer epoch_1 db, and will start to re-cleave the
|
# shard again with a newer epoch_1 db, and will start to re-cleave the
|
||||||
# 3 active shards, albeit with zero objects to cleave.
|
# 3 active shards, albeit with zero objects to cleave.
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
for node in (0, 1, 2):
|
for node in (0, 1, 2):
|
||||||
with annotate_failure('node %s' % node):
|
with annotate_failure('node %s' % node):
|
||||||
broker = self.get_broker(self.brain.part,
|
broker = self.get_broker(self.brain.part,
|
||||||
@ -3274,7 +3427,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# to exist on the same node as the root because the roots cleaving
|
# to exist on the same node as the root because the roots cleaving
|
||||||
# process doesn't think that it created the shard db and will therefore
|
# process doesn't think that it created the shard db and will therefore
|
||||||
# replicate it as per a normal cleave.
|
# replicate it as per a normal cleave.
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
for node in (0, 1, 2):
|
for node in (0, 1, 2):
|
||||||
with annotate_failure('node %s' % node):
|
with annotate_failure('node %s' % node):
|
||||||
broker = self.get_broker(self.brain.part,
|
broker = self.get_broker(self.brain.part,
|
||||||
@ -3311,11 +3464,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
'find_and_replace', '4', '--enable'])
|
'find_and_replace', '4', '--enable'])
|
||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
# cleave first two shards
|
# cleave first two shards
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
# cleave third shard
|
# cleave third shard
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
# ensure all shards learn their ACTIVE state from root
|
# ensure all shards learn their ACTIVE state from root
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
for node in (0, 1, 2):
|
for node in (0, 1, 2):
|
||||||
with annotate_failure('node %d' % node):
|
with annotate_failure('node %d' % node):
|
||||||
shard_ranges = self.assert_container_state(
|
shard_ranges = self.assert_container_state(
|
||||||
@ -3368,9 +3523,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
|
|
||||||
# try hard to shard the shard...
|
# try hard to shard the shard...
|
||||||
self.sharders_once(additional_args='--partitions=%s' % shard_1_part)
|
self.sharders_once_non_auto(
|
||||||
self.sharders_once(additional_args='--partitions=%s' % shard_1_part)
|
additional_args='--partitions=%s' % shard_1_part)
|
||||||
self.sharders_once(additional_args='--partitions=%s' % shard_1_part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % shard_1_part)
|
||||||
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % shard_1_part)
|
||||||
# sharding hasn't completed and there's overlaps in the shard and root:
|
# sharding hasn't completed and there's overlaps in the shard and root:
|
||||||
# the sub-shards will have been cleaved in the order listed above, but
|
# the sub-shards will have been cleaved in the order listed above, but
|
||||||
# sub-shards (10 -12) and one of (12 - 14) will be overlooked because
|
# sub-shards (10 -12) and one of (12 - 14) will be overlooked because
|
||||||
@ -3403,8 +3561,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
['swift-manage-shard-ranges', db_file, 'repair', '--yes',
|
['swift-manage-shard-ranges', db_file, 'repair', '--yes',
|
||||||
'--min-shard-age', '0'])
|
'--min-shard-age', '0'])
|
||||||
self.replicators.once()
|
self.replicators.once()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
|
|
||||||
# check root now has just 5 shard ranges
|
# check root now has just 5 shard ranges
|
||||||
root_shard_ranges = self.get_container_shard_ranges()
|
root_shard_ranges = self.get_container_shard_ranges()
|
||||||
@ -3416,7 +3574,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# because the sub-shards report their state to the root; we cannot make
|
# because the sub-shards report their state to the root; we cannot make
|
||||||
# assertions about shrunk states in shard_1's shard range table)
|
# assertions about shrunk states in shard_1's shard range table)
|
||||||
root_shard_ranges = self.get_container_shard_ranges(
|
root_shard_ranges = self.get_container_shard_ranges(
|
||||||
include_deleted=True)
|
headers={'X-Backend-Include-Deleted': 'true'})
|
||||||
self.assertEqual(10, len(root_shard_ranges), root_shard_ranges)
|
self.assertEqual(10, len(root_shard_ranges), root_shard_ranges)
|
||||||
shrunk_shard_ranges = [sr for sr in root_shard_ranges
|
shrunk_shard_ranges = [sr for sr in root_shard_ranges
|
||||||
if sr.state == ShardRange.SHRUNK]
|
if sr.state == ShardRange.SHRUNK]
|
||||||
@ -3452,13 +3610,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
'find_and_replace', '2', '--enable'])
|
'find_and_replace', '2', '--enable'])
|
||||||
self.container_replicators.once(
|
self.container_replicators.once(
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('unsharded', 2)
|
||||||
self.assert_container_state(node, 'unsharded', 2)
|
self.sharders_once_non_auto(
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
# get shards to update state from parent...
|
# get shards to update state from parent...
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 2)
|
||||||
self.assert_container_state(node, 'sharded', 2)
|
|
||||||
|
|
||||||
# sanity check, all is well
|
# sanity check, all is well
|
||||||
msg = self.assert_subprocess_success([
|
msg = self.assert_subprocess_success([
|
||||||
@ -3483,9 +3640,10 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.assert_container_state(
|
self.assert_container_state(
|
||||||
node, 'unsharded', 2, account=shard_ranges[0].account,
|
node, 'unsharded', 2, account=shard_ranges[0].account,
|
||||||
container=shard_ranges[0].container, part=shard_part)
|
container=shard_ranges[0].container, part=shard_part)
|
||||||
self.sharders_once(additional_args='--partitions=%s' % shard_part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % shard_part)
|
||||||
# get shards to update state from parent...
|
# get shards to update state from parent...
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
for node in exclude_nodes(shard_nodes, self.brain.nodes[0]):
|
for node in exclude_nodes(shard_nodes, self.brain.nodes[0]):
|
||||||
self.assert_container_state(
|
self.assert_container_state(
|
||||||
node, 'sharded', 2, account=shard_ranges[0].account,
|
node, 'sharded', 2, account=shard_ranges[0].account,
|
||||||
@ -3528,7 +3686,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# TODO: how can we work around this?
|
# TODO: how can we work around this?
|
||||||
self.assertNotEqual(sub_shard_part, shard_part,
|
self.assertNotEqual(sub_shard_part, shard_part,
|
||||||
'You were unlucky, try again')
|
'You were unlucky, try again')
|
||||||
self.sharders_once(additional_args='--partitions=%s' % sub_shard_part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % sub_shard_part)
|
||||||
|
|
||||||
# now root node 0 has the original shards plus one of the sub-shards
|
# now root node 0 has the original shards plus one of the sub-shards
|
||||||
# but all are active :(
|
# but all are active :(
|
||||||
@ -3558,8 +3717,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
for sr in root_brokers[0].get_shard_ranges(include_deleted=True)])
|
for sr in root_brokers[0].get_shard_ranges(include_deleted=True)])
|
||||||
|
|
||||||
# the transient overlap is 'fixed' in subsequent sharder cycles...
|
# the transient overlap is 'fixed' in subsequent sharder cycles...
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.container_replicators.once()
|
self.container_replicators.once()
|
||||||
|
|
||||||
for broker in root_brokers:
|
for broker in root_brokers:
|
||||||
@ -3593,13 +3752,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
'find_and_replace', '2', '--enable'])
|
'find_and_replace', '2', '--enable'])
|
||||||
self.container_replicators.once(
|
self.container_replicators.once(
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('unsharded', 4)
|
||||||
self.assert_container_state(node, 'unsharded', 4)
|
self.sharders_once_non_auto(
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
# get shards to update state from parent...
|
# get shards to update state from parent...
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 4)
|
||||||
self.assert_container_state(node, 'sharded', 4)
|
|
||||||
|
|
||||||
# sanity check, all is well
|
# sanity check, all is well
|
||||||
msg = self.assert_subprocess_success([
|
msg = self.assert_subprocess_success([
|
||||||
@ -3635,8 +3793,8 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
'--yes'])
|
'--yes'])
|
||||||
self.assertIn(b'Repairs necessary to fill gaps.', msg)
|
self.assertIn(b'Repairs necessary to fill gaps.', msg)
|
||||||
|
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.container_replicators.once()
|
self.container_replicators.once()
|
||||||
|
|
||||||
# yay! we fixed the gap (without creating an overlap)
|
# yay! we fixed the gap (without creating an overlap)
|
||||||
@ -3662,7 +3820,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
new_objs = [obj_names[4] + 'a']
|
new_objs = [obj_names[4] + 'a']
|
||||||
self.put_objects(new_objs)
|
self.put_objects(new_objs)
|
||||||
# get root stats up to date
|
# get root stats up to date
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
# new object is in listing but old objects in the gap have been lost -
|
# new object is in listing but old objects in the gap have been lost -
|
||||||
# don't delete shard ranges!
|
# don't delete shard ranges!
|
||||||
self.assert_container_listing(obj_names[:4] + new_objs + obj_names[6:])
|
self.assert_container_listing(obj_names[:4] + new_objs + obj_names[6:])
|
||||||
@ -3702,7 +3860,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# Run container-sharder to shard the 2 primary replicas that did
|
# Run container-sharder to shard the 2 primary replicas that did
|
||||||
# receive the object PUTs
|
# receive the object PUTs
|
||||||
for num in self.brain.primary_numbers:
|
for num in self.brain.primary_numbers:
|
||||||
self.sharders_once(
|
self.sharders_once_non_auto(
|
||||||
number=num,
|
number=num,
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
@ -3711,7 +3869,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
self.delete_objects(obj_names)
|
self.delete_objects(obj_names)
|
||||||
# deal with DELETE's being misplaced in root db's...
|
# deal with DELETE's being misplaced in root db's...
|
||||||
for num in self.brain.primary_numbers:
|
for num in self.brain.primary_numbers:
|
||||||
self.sharders_once(
|
self.sharders_once_non_auto(
|
||||||
number=num,
|
number=num,
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
@ -3738,7 +3896,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
|
|
||||||
# now shard the final DB
|
# now shard the final DB
|
||||||
for num in self.brain.handoff_numbers:
|
for num in self.brain.handoff_numbers:
|
||||||
self.sharders_once(
|
self.sharders_once_non_auto(
|
||||||
number=num,
|
number=num,
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
@ -3806,7 +3964,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# Run container-sharder to shard the 2 primary replicas that did
|
# Run container-sharder to shard the 2 primary replicas that did
|
||||||
# receive the object PUTs
|
# receive the object PUTs
|
||||||
for num in self.brain.primary_numbers:
|
for num in self.brain.primary_numbers:
|
||||||
self.sharders_once(
|
self.sharders_once_non_auto(
|
||||||
number=num,
|
number=num,
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
|
|
||||||
@ -3828,7 +3986,7 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
|
|
||||||
# now shard the final DB
|
# now shard the final DB
|
||||||
for num in self.brain.handoff_numbers:
|
for num in self.brain.handoff_numbers:
|
||||||
self.sharders_once(
|
self.sharders_once_non_auto(
|
||||||
number=num,
|
number=num,
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
shard_ranges = self.assert_container_state(
|
shard_ranges = self.assert_container_state(
|
||||||
@ -3876,14 +4034,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# Run container-replicator to replicate them to other nodes.
|
# Run container-replicator to replicate them to other nodes.
|
||||||
self.container_replicators.once(
|
self.container_replicators.once(
|
||||||
additional_args='--partitions=%s' % self.brain.part)
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('unsharded', 2)
|
||||||
self.assert_container_state(node, 'unsharded', 2)
|
|
||||||
# Run container-sharder on all nodes to shard the container.
|
# Run container-sharder on all nodes to shard the container.
|
||||||
self.sharders_once(additional_args='--partitions=%s' % self.brain.part)
|
self.sharders_once_non_auto(
|
||||||
|
additional_args='--partitions=%s' % self.brain.part)
|
||||||
# get shards to update state from parent...
|
# get shards to update state from parent...
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
for node in self.brain.nodes:
|
self.assert_container_states('sharded', 2)
|
||||||
self.assert_container_state(node, 'sharded', 2)
|
|
||||||
|
|
||||||
# shard first child shard into 2 grand-child-shards.
|
# shard first child shard into 2 grand-child-shards.
|
||||||
c_shard_ranges = self.get_container_shard_ranges()
|
c_shard_ranges = self.get_container_shard_ranges()
|
||||||
@ -3912,14 +4069,14 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
c_shard_dir = os.path.dirname(c_shard_brokers[2].db_file)
|
c_shard_dir = os.path.dirname(c_shard_brokers[2].db_file)
|
||||||
c_shard_tmp_dir = c_shard_dir + ".tmp"
|
c_shard_tmp_dir = c_shard_dir + ".tmp"
|
||||||
os.rename(c_shard_dir, c_shard_tmp_dir)
|
os.rename(c_shard_dir, c_shard_tmp_dir)
|
||||||
self.sharders_once(additional_args='--partitions=%s' %
|
self.sharders_once_non_auto(additional_args='--partitions=%s' %
|
||||||
child_shard_part)
|
child_shard_part)
|
||||||
for node in c_shard_nodes[:2]:
|
for node in c_shard_nodes[:2]:
|
||||||
self.assert_container_state(
|
self.assert_container_state(
|
||||||
node, 'sharded', 2, account=c_shard_ranges[0].account,
|
node, 'sharded', 2, account=c_shard_ranges[0].account,
|
||||||
container=c_shard_ranges[0].container, part=child_shard_part)
|
container=c_shard_ranges[0].container, part=child_shard_part)
|
||||||
# get updates done...
|
# get updates done...
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
|
|
||||||
# shard first grand-child shard into 2 grand-grand-child-shards.
|
# shard first grand-child shard into 2 grand-grand-child-shards.
|
||||||
gc_shard_ranges = self.get_container_shard_ranges(
|
gc_shard_ranges = self.get_container_shard_ranges(
|
||||||
@ -3936,12 +4093,12 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
gc_shard_ranges[0].account, gc_shard_ranges[0].container)
|
gc_shard_ranges[0].account, gc_shard_ranges[0].container)
|
||||||
self.container_replicators.once(
|
self.container_replicators.once(
|
||||||
additional_args='--partitions=%s' % grandchild_shard_part)
|
additional_args='--partitions=%s' % grandchild_shard_part)
|
||||||
self.sharders_once(additional_args='--partitions=%s' %
|
self.sharders_once_non_auto(additional_args='--partitions=%s' %
|
||||||
grandchild_shard_part)
|
grandchild_shard_part)
|
||||||
|
|
||||||
# get shards to update state from parent...
|
# get shards to update state from parent...
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.sharders_once()
|
self.sharders_once_non_auto()
|
||||||
self.container_replicators.once(
|
self.container_replicators.once(
|
||||||
additional_args='--partitions=%s' % child_shard_part)
|
additional_args='--partitions=%s' % child_shard_part)
|
||||||
|
|
||||||
@ -3966,13 +4123,13 @@ class TestManagedContainerSharding(BaseTestContainerSharding):
|
|||||||
# now, finally, run the sharder on the child that is still waiting to
|
# now, finally, run the sharder on the child that is still waiting to
|
||||||
# shard. It will get 2 great-grandchild ranges from root to replace
|
# shard. It will get 2 great-grandchild ranges from root to replace
|
||||||
# deleted grandchild.
|
# deleted grandchild.
|
||||||
self.sharders_once(
|
self.sharders_once_non_auto(
|
||||||
additional_args=['--partitions=%s' %
|
additional_args=['--partitions=%s' %
|
||||||
child_shard_part, '--devices=%s' %
|
child_shard_part, '--devices=%s' %
|
||||||
c_shard_nodes[2]['device']])
|
c_shard_nodes[2]['device']])
|
||||||
# batch size is 2 but this replicas has 3 shard ranges so we need two
|
# batch size is 2 but this replicas has 3 shard ranges so we need two
|
||||||
# runs of the sharder
|
# runs of the sharder
|
||||||
self.sharders_once(
|
self.sharders_once_non_auto(
|
||||||
additional_args=['--partitions=%s' %
|
additional_args=['--partitions=%s' %
|
||||||
child_shard_part, '--devices=%s' %
|
child_shard_part, '--devices=%s' %
|
||||||
c_shard_nodes[2]['device']])
|
c_shard_nodes[2]['device']])
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
import json
|
import json
|
||||||
import random
|
import random
|
||||||
from argparse import Namespace
|
import argparse
|
||||||
|
|
||||||
import eventlet
|
import eventlet
|
||||||
import os
|
import os
|
||||||
@ -47,7 +47,7 @@ from swift.container.sharder import ContainerSharder, sharding_enabled, \
|
|||||||
update_own_shard_range_stats
|
update_own_shard_range_stats
|
||||||
from swift.common.utils import ShardRange, Timestamp, hash_path, \
|
from swift.common.utils import ShardRange, Timestamp, hash_path, \
|
||||||
encode_timestamps, parse_db_filename, quorum_size, Everything, md5, \
|
encode_timestamps, parse_db_filename, quorum_size, Everything, md5, \
|
||||||
ShardName
|
ShardName, Namespace
|
||||||
from test import annotate_failure
|
from test import annotate_failure
|
||||||
|
|
||||||
from test.debug_logger import debug_logger
|
from test.debug_logger import debug_logger
|
||||||
@ -1261,6 +1261,10 @@ class TestSharder(BaseTestSharder):
|
|||||||
do_test('')
|
do_test('')
|
||||||
do_test(json.dumps({}))
|
do_test(json.dumps({}))
|
||||||
do_test(json.dumps([{'account': 'a', 'container': 'c'}]))
|
do_test(json.dumps([{'account': 'a', 'container': 'c'}]))
|
||||||
|
do_test(json.dumps([dict(Namespace('a/c', 'l', 'u'))]))
|
||||||
|
sr_dict = dict(ShardRange('a/c', next(self.ts_iter), 'l', 'u'))
|
||||||
|
sr_dict.pop('object_count')
|
||||||
|
do_test(json.dumps([sr_dict]))
|
||||||
|
|
||||||
def test_fetch_shard_ranges_ok(self):
|
def test_fetch_shard_ranges_ok(self):
|
||||||
def do_test(mock_resp_body, params):
|
def do_test(mock_resp_body, params):
|
||||||
@ -9696,11 +9700,11 @@ class TestContainerSharderConf(unittest.TestCase):
|
|||||||
# given namespace
|
# given namespace
|
||||||
def assert_bad(conf):
|
def assert_bad(conf):
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
ContainerSharderConf.validate_conf(Namespace(**conf))
|
ContainerSharderConf.validate_conf(argparse.Namespace(**conf))
|
||||||
|
|
||||||
def assert_ok(conf):
|
def assert_ok(conf):
|
||||||
try:
|
try:
|
||||||
ContainerSharderConf.validate_conf(Namespace(**conf))
|
ContainerSharderConf.validate_conf(argparse.Namespace(**conf))
|
||||||
except ValueError as err:
|
except ValueError as err:
|
||||||
self.fail('Unexpected ValueError: %s' % err)
|
self.fail('Unexpected ValueError: %s' % err)
|
||||||
|
|
||||||
|
@ -486,7 +486,7 @@ class TestContainerController(TestRingBase):
|
|||||||
def _check_GET_shard_listing(self, mock_responses, expected_objects,
|
def _check_GET_shard_listing(self, mock_responses, expected_objects,
|
||||||
expected_requests, query_string='',
|
expected_requests, query_string='',
|
||||||
reverse=False, expected_status=200,
|
reverse=False, expected_status=200,
|
||||||
memcache=False):
|
memcache=False, req_headers=None):
|
||||||
# mock_responses is a list of tuples (status, json body, headers)
|
# mock_responses is a list of tuples (status, json body, headers)
|
||||||
# expected objects is a list of dicts
|
# expected objects is a list of dicts
|
||||||
# expected_requests is a list of tuples (path, hdrs dict, params dict)
|
# expected_requests is a list of tuples (path, hdrs dict, params dict)
|
||||||
@ -506,6 +506,8 @@ class TestContainerController(TestRingBase):
|
|||||||
for resp in mock_responses])
|
for resp in mock_responses])
|
||||||
exp_headers = [resp[2] for resp in mock_responses]
|
exp_headers = [resp[2] for resp in mock_responses]
|
||||||
request = Request.blank(container_path)
|
request = Request.blank(container_path)
|
||||||
|
if req_headers:
|
||||||
|
request.headers.update(req_headers)
|
||||||
if memcache:
|
if memcache:
|
||||||
# memcache exists, which causes backend to ignore constraints and
|
# memcache exists, which causes backend to ignore constraints and
|
||||||
# reverse params for shard range GETs
|
# reverse params for shard range GETs
|
||||||
@ -566,6 +568,7 @@ class TestContainerController(TestRingBase):
|
|||||||
int(resp.headers['X-Container-Object-Count']))
|
int(resp.headers['X-Container-Object-Count']))
|
||||||
self.assertEqual(exp_sharding_state,
|
self.assertEqual(exp_sharding_state,
|
||||||
resp.headers['X-Backend-Sharding-State'])
|
resp.headers['X-Backend-Sharding-State'])
|
||||||
|
self.assertNotIn('X-Backend-Record-Type', resp.headers)
|
||||||
for k, v in root_resp_hdrs.items():
|
for k, v in root_resp_hdrs.items():
|
||||||
if k.lower().startswith('x-container-meta'):
|
if k.lower().startswith('x-container-meta'):
|
||||||
self.assertEqual(v, resp.headers[k])
|
self.assertEqual(v, resp.headers[k])
|
||||||
@ -662,6 +665,18 @@ class TestContainerController(TestRingBase):
|
|||||||
self.check_response(resp, root_resp_hdrs,
|
self.check_response(resp, root_resp_hdrs,
|
||||||
expected_objects=expected_objects)
|
expected_objects=expected_objects)
|
||||||
|
|
||||||
|
resp = self._check_GET_shard_listing(
|
||||||
|
mock_responses, expected_objects, expected_requests,
|
||||||
|
req_headers={'X-Backend-Record-Type': 'auto'})
|
||||||
|
self.check_response(resp, root_resp_hdrs,
|
||||||
|
expected_objects=expected_objects)
|
||||||
|
|
||||||
|
resp = self._check_GET_shard_listing(
|
||||||
|
mock_responses, expected_objects, expected_requests,
|
||||||
|
req_headers={'X-Backend-Record-Type': 'banana'})
|
||||||
|
self.check_response(resp, root_resp_hdrs,
|
||||||
|
expected_objects=expected_objects)
|
||||||
|
|
||||||
# GET all objects - sharding, final shard range points back to root
|
# GET all objects - sharding, final shard range points back to root
|
||||||
root_range = ShardRange('a/c', Timestamp.now(), 'pie', '')
|
root_range = ShardRange('a/c', Timestamp.now(), 'pie', '')
|
||||||
mock_responses = [
|
mock_responses = [
|
||||||
@ -1049,6 +1064,18 @@ class TestContainerController(TestRingBase):
|
|||||||
self.check_response(resp, root_resp_hdrs,
|
self.check_response(resp, root_resp_hdrs,
|
||||||
expected_objects=expected_objects)
|
expected_objects=expected_objects)
|
||||||
|
|
||||||
|
resp = self._check_GET_shard_listing(
|
||||||
|
mock_responses, expected_objects, expected_requests, memcache=True,
|
||||||
|
req_headers={'X-Backend-Record-Type': 'auto'})
|
||||||
|
self.check_response(resp, root_resp_hdrs,
|
||||||
|
expected_objects=expected_objects)
|
||||||
|
|
||||||
|
resp = self._check_GET_shard_listing(
|
||||||
|
mock_responses, expected_objects, expected_requests, memcache=True,
|
||||||
|
req_headers={'X-Backend-Record-Type': 'banana'})
|
||||||
|
self.check_response(resp, root_resp_hdrs,
|
||||||
|
expected_objects=expected_objects)
|
||||||
|
|
||||||
# GET all objects - sharding, final shard range points back to root
|
# GET all objects - sharding, final shard range points back to root
|
||||||
root_range = ShardRange('a/c', Timestamp.now(), 'pie', '')
|
root_range = ShardRange('a/c', Timestamp.now(), 'pie', '')
|
||||||
mock_responses = [
|
mock_responses = [
|
||||||
@ -2730,9 +2757,20 @@ class TestContainerController(TestRingBase):
|
|||||||
'X-Container-Bytes-Used': '12',
|
'X-Container-Bytes-Used': '12',
|
||||||
'X-Backend-Storage-Policy-Index': '0'}
|
'X-Backend-Storage-Policy-Index': '0'}
|
||||||
|
|
||||||
def _do_test_caching(self, record_type, exp_recheck_listing):
|
def _do_test_caching(self, record_type, exp_recheck_listing,
|
||||||
|
exp_record_type):
|
||||||
# this test gets shard ranges into cache and then reads from cache
|
# this test gets shard ranges into cache and then reads from cache
|
||||||
sharding_state = 'sharded'
|
sharding_state = 'sharded'
|
||||||
|
exp_resp_headers = {
|
||||||
|
'X-Backend-Recheck-Container-Existence': '60',
|
||||||
|
'X-Backend-Sharding-State': sharding_state}
|
||||||
|
if exp_record_type:
|
||||||
|
exp_resp_headers['X-Backend-Record-Type'] = exp_record_type
|
||||||
|
exp_cache_resp_headers = {
|
||||||
|
'X-Backend-Cached-Results': 'true',
|
||||||
|
'X-Backend-Sharding-State': sharding_state}
|
||||||
|
if exp_record_type:
|
||||||
|
exp_cache_resp_headers['X-Backend-Record-Type'] = exp_record_type
|
||||||
self.memcache.delete_all()
|
self.memcache.delete_all()
|
||||||
# container is sharded but proxy does not have that state cached;
|
# container is sharded but proxy does not have that state cached;
|
||||||
# expect a backend request and expect shard ranges to be cached
|
# expect a backend request and expect shard ranges to be cached
|
||||||
@ -2749,10 +2787,7 @@ class TestContainerController(TestRingBase):
|
|||||||
req, backend_req,
|
req, backend_req,
|
||||||
extra_hdrs={'X-Backend-Record-Type': record_type,
|
extra_hdrs={'X-Backend-Record-Type': record_type,
|
||||||
'X-Backend-Override-Shard-Name-Filter': 'sharded'})
|
'X-Backend-Override-Shard-Name-Filter': 'sharded'})
|
||||||
self._check_response(resp, self.ns_dicts, {
|
self._check_response(resp, self.ns_dicts, exp_resp_headers)
|
||||||
'X-Backend-Recheck-Container-Existence': '60',
|
|
||||||
'X-Backend-Record-Type': 'shard',
|
|
||||||
'X-Backend-Sharding-State': sharding_state})
|
|
||||||
|
|
||||||
cache_key = 'shard-listing-v2/a/c'
|
cache_key = 'shard-listing-v2/a/c'
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@ -2789,10 +2824,7 @@ class TestContainerController(TestRingBase):
|
|||||||
req, backend_req,
|
req, backend_req,
|
||||||
extra_hdrs={'X-Backend-Record-Type': record_type,
|
extra_hdrs={'X-Backend-Record-Type': record_type,
|
||||||
'X-Backend-Override-Shard-Name-Filter': 'sharded'})
|
'X-Backend-Override-Shard-Name-Filter': 'sharded'})
|
||||||
self._check_response(resp, self.ns_dicts, {
|
self._check_response(resp, self.ns_dicts, exp_resp_headers)
|
||||||
'X-Backend-Recheck-Container-Existence': '60',
|
|
||||||
'X-Backend-Record-Type': 'shard',
|
|
||||||
'X-Backend-Sharding-State': sharding_state})
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
[mock.call.get('container/a/c'),
|
[mock.call.get('container/a/c'),
|
||||||
mock.call.get(cache_key, raise_on_error=True),
|
mock.call.get(cache_key, raise_on_error=True),
|
||||||
@ -2819,10 +2851,7 @@ class TestContainerController(TestRingBase):
|
|||||||
req = self._build_request({'X-Backend-Record-Type': record_type},
|
req = self._build_request({'X-Backend-Record-Type': record_type},
|
||||||
{'states': 'listing'}, {})
|
{'states': 'listing'}, {})
|
||||||
resp = req.get_response(self.app)
|
resp = req.get_response(self.app)
|
||||||
self._check_response(resp, self.ns_dicts, {
|
self._check_response(resp, self.ns_dicts, exp_cache_resp_headers)
|
||||||
'X-Backend-Cached-Results': 'true',
|
|
||||||
'X-Backend-Record-Type': 'shard',
|
|
||||||
'X-Backend-Sharding-State': sharding_state})
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
[mock.call.get('container/a/c'),
|
[mock.call.get('container/a/c'),
|
||||||
mock.call.get(cache_key, raise_on_error=True)],
|
mock.call.get(cache_key, raise_on_error=True)],
|
||||||
@ -2853,10 +2882,7 @@ class TestContainerController(TestRingBase):
|
|||||||
req, backend_req,
|
req, backend_req,
|
||||||
extra_hdrs={'X-Backend-Record-Type': record_type,
|
extra_hdrs={'X-Backend-Record-Type': record_type,
|
||||||
'X-Backend-Override-Shard-Name-Filter': 'sharded'})
|
'X-Backend-Override-Shard-Name-Filter': 'sharded'})
|
||||||
self._check_response(resp, self.ns_dicts, {
|
self._check_response(resp, self.ns_dicts, exp_resp_headers)
|
||||||
'X-Backend-Recheck-Container-Existence': '60',
|
|
||||||
'X-Backend-Record-Type': 'shard',
|
|
||||||
'X-Backend-Sharding-State': sharding_state})
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
[mock.call.get('container/a/c'),
|
[mock.call.get('container/a/c'),
|
||||||
mock.call.set(cache_key, self.ns_bound_list.bounds,
|
mock.call.set(cache_key, self.ns_bound_list.bounds,
|
||||||
@ -2882,10 +2908,7 @@ class TestContainerController(TestRingBase):
|
|||||||
{'states': 'listing'}, {})
|
{'states': 'listing'}, {})
|
||||||
with mock.patch('random.random', return_value=0.11):
|
with mock.patch('random.random', return_value=0.11):
|
||||||
resp = req.get_response(self.app)
|
resp = req.get_response(self.app)
|
||||||
self._check_response(resp, self.ns_dicts, {
|
self._check_response(resp, self.ns_dicts, exp_cache_resp_headers)
|
||||||
'X-Backend-Cached-Results': 'true',
|
|
||||||
'X-Backend-Record-Type': 'shard',
|
|
||||||
'X-Backend-Sharding-State': sharding_state})
|
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
[mock.call.get('container/a/c'),
|
[mock.call.get('container/a/c'),
|
||||||
mock.call.get(cache_key, raise_on_error=True)],
|
mock.call.get(cache_key, raise_on_error=True)],
|
||||||
@ -2909,10 +2932,7 @@ class TestContainerController(TestRingBase):
|
|||||||
infocache=req.environ['swift.infocache'])
|
infocache=req.environ['swift.infocache'])
|
||||||
with mock.patch('random.random', return_value=0.11):
|
with mock.patch('random.random', return_value=0.11):
|
||||||
resp = req.get_response(self.app)
|
resp = req.get_response(self.app)
|
||||||
self._check_response(resp, self.ns_dicts, {
|
self._check_response(resp, self.ns_dicts, exp_cache_resp_headers)
|
||||||
'X-Backend-Cached-Results': 'true',
|
|
||||||
'X-Backend-Record-Type': 'shard',
|
|
||||||
'X-Backend-Sharding-State': sharding_state})
|
|
||||||
self.assertEqual([], self.memcache.calls)
|
self.assertEqual([], self.memcache.calls)
|
||||||
self.assertIn('swift.infocache', req.environ)
|
self.assertIn('swift.infocache', req.environ)
|
||||||
self.assertIn(cache_key, req.environ['swift.infocache'])
|
self.assertIn(cache_key, req.environ['swift.infocache'])
|
||||||
@ -3025,10 +3045,10 @@ class TestContainerController(TestRingBase):
|
|||||||
def test_GET_shard_ranges(self):
|
def test_GET_shard_ranges(self):
|
||||||
self._setup_shard_range_stubs()
|
self._setup_shard_range_stubs()
|
||||||
# expect shard ranges cache time to be default value of 600
|
# expect shard ranges cache time to be default value of 600
|
||||||
self._do_test_caching('shard', 600)
|
self._do_test_caching('shard', 600, 'shard')
|
||||||
# expect shard ranges cache time to be configured value of 120
|
# expect shard ranges cache time to be configured value of 120
|
||||||
self.app.recheck_listing_shard_ranges = 120
|
self.app.recheck_listing_shard_ranges = 120
|
||||||
self._do_test_caching('shard', 120)
|
self._do_test_caching('shard', 120, 'shard')
|
||||||
|
|
||||||
def mock_get_from_shards(self, req, resp):
|
def mock_get_from_shards(self, req, resp):
|
||||||
# for the purposes of these tests we override _get_from_shards so
|
# for the purposes of these tests we override _get_from_shards so
|
||||||
@ -3042,7 +3062,7 @@ class TestContainerController(TestRingBase):
|
|||||||
'ContainerController._get_from_shards',
|
'ContainerController._get_from_shards',
|
||||||
mock_get_from_shards):
|
mock_get_from_shards):
|
||||||
self.app.recheck_listing_shard_ranges = 600
|
self.app.recheck_listing_shard_ranges = 600
|
||||||
self._do_test_caching('auto', 600)
|
self._do_test_caching('auto', 600, exp_record_type=None)
|
||||||
|
|
||||||
def test_GET_shard_ranges_404_response(self):
|
def test_GET_shard_ranges_404_response(self):
|
||||||
# pre-warm cache with container info but not shard ranges so that the
|
# pre-warm cache with container info but not shard ranges so that the
|
||||||
@ -3169,6 +3189,8 @@ class TestContainerController(TestRingBase):
|
|||||||
def mock_get_from_shards(self, req, resp):
|
def mock_get_from_shards(self, req, resp):
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
|
# request with record-type=auto does not expect record-type in response
|
||||||
|
del exp_hdrs['X-Backend-Record-Type']
|
||||||
with mock.patch('swift.proxy.controllers.container.'
|
with mock.patch('swift.proxy.controllers.container.'
|
||||||
'ContainerController._get_from_shards',
|
'ContainerController._get_from_shards',
|
||||||
mock_get_from_shards):
|
mock_get_from_shards):
|
||||||
@ -3264,6 +3286,8 @@ class TestContainerController(TestRingBase):
|
|||||||
def mock_get_from_shards(self, req, resp):
|
def mock_get_from_shards(self, req, resp):
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
|
# request with record-type=auto does not expect record-type in response
|
||||||
|
del exp_hdrs['X-Backend-Record-Type']
|
||||||
with mock.patch('swift.proxy.controllers.container.'
|
with mock.patch('swift.proxy.controllers.container.'
|
||||||
'ContainerController._get_from_shards',
|
'ContainerController._get_from_shards',
|
||||||
mock_get_from_shards):
|
mock_get_from_shards):
|
||||||
|
Loading…
Reference in New Issue
Block a user