openstack/swift
Code Issues Proposed changes

sharder: merge shard shard_ranges from root while sharding

Browse Source 
We've seen shards become stuck while sharding because they had
incomplete or stale deleted shard ranges. The root container had more
complete and useful shard ranges into which objects could have been
cleaved, but the shard never merged the root's shard ranges.

While the sharder is auditing shard container DBs it would previously
only merge shard ranges fetched from root into the shard DB if the
shard was shrinking or the shard ranges were known to be children of
the shard. With this patch the sharder will now merge other shard
ranges from root during sharding as well as shrinking.

Shard ranges from root are only merged if they would not result in
overlaps or gaps in the set of shard ranges in the shard DB. Shard
ranges that are known to be ancestors of the shard are never merged,
except the root shard range which may be merged into a shrinking
shard. These checks were not previously applied when merging
shard ranges into a shrinking shard.

The two substantive changes with this patch are therefore:

  - shard ranges from root are now merged during sharding,
    subject to checks.
  - shard ranges from root are still merged during shrinking,
    but are now subjected to checks.

Change-Id: I066cfbd9062c43cd9638710882ae9bd85a5b4c37
This commit is contained in:
Alistair Coles 2022-08-12 15:33:46 +01:00
parent c82c73122b
commit 2bcf3d1a8e
8 changed files with 918 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
swift/cli/manage_shard_ranges.py
View File

@ -168,13 +168,12 @@ from six.moves import input
from swift.common.utils import Timestamp, get_logger, ShardRange, readconf, \
ShardRangeList, non_negative_int, config_positive_int_value
from swift.container.backend import ContainerBroker, UNSHARDED, \
sift_shard_ranges
from swift.container.backend import ContainerBroker, UNSHARDED
from swift.container.sharder import make_shard_ranges, sharding_enabled, \
CleavingContext, process_compactible_shard_sequences, \
find_compactible_shard_sequences, find_overlapping_ranges, \
find_paths, rank_paths, finalize_shrinking, DEFAULT_SHARDER_CONF, \
ContainerSharderConf, find_paths_with_gaps
ContainerSharderConf, find_paths_with_gaps, combine_shard_ranges
EXIT_SUCCESS = 0
EXIT_ERROR = 1
@ -428,27 +427,6 @@ def delete_shard_ranges(broker, args):
return EXIT_SUCCESS
def combine_shard_ranges(new_shard_ranges, existing_shard_ranges):
"""
Combines new and existing shard ranges based on most recent state.
:param new_shard_ranges: a list of ShardRange instances.
:param existing_shard_ranges: a list of ShardRange instances.
:return: a list of ShardRange instances.
"""
new_shard_ranges = [dict(sr) for sr in new_shard_ranges]
existing_shard_ranges = [dict(sr) for sr in existing_shard_ranges]
to_add, to_delete = sift_shard_ranges(
new_shard_ranges,
dict((sr['name'], sr) for sr in existing_shard_ranges))
result = [ShardRange.from_dict(existing)
for existing in existing_shard_ranges
if existing['name'] not in to_delete]
result.extend([ShardRange.from_dict(sr) for sr in to_add])
return sorted([sr for sr in result if not sr.deleted],
key=ShardRange.sort_key)
def merge_shard_ranges(broker, args):
_check_own_shard_range(broker, args)
shard_data = _load_and_validate_shard_data(args, require_index=False)

72
swift/common/utils.py
View File

@ -5330,6 +5330,8 @@ class ShardRange(object):
SHRUNK: 'shrunk'}
STATES_BY_NAME = dict((v, k) for k, v in STATES.items())
SHRINKING_STATES = (SHRINKING, SHRUNK)
SHARDING_STATES = (SHARDING, SHARDED)
CLEAVING_STATES = SHRINKING_STATES + SHARDING_STATES
@functools.total_ordering
class MaxBound(ShardRangeOuterBound):
@ -5434,6 +5436,76 @@ class ShardRange(object):
== ShardName.hash_container_name(parent.container)
)
def _find_root(self, parsed_name, shard_ranges):
for sr in shard_ranges:
if parsed_name.root_container == sr.container:
return sr
return None
def find_root(self, shard_ranges):
"""
Find this shard range's root shard range in the given ``shard_ranges``.
:param shard_ranges: a list of instances of
:class:`~swift.common.utils.ShardRange`
:return: this shard range's root shard range if it is found in the
list, otherwise None.
"""
try:
self_parsed_name = ShardName.parse(self.name)
except ValueError:
# not a shard
return None
return self._find_root(self_parsed_name, shard_ranges)
def find_ancestors(self, shard_ranges):
"""
Find this shard range's ancestor ranges in the given ``shard_ranges``.
This method makes a best-effort attempt to identify this shard range's
parent shard range, the parent's parent, etc., up to and including the
root shard range. It is only possible to directly identify the parent
of a particular shard range, so the search is recursive; if any member
of the ancestry is not found then the search ends and older ancestors
that may be in the list are not identified. The root shard range,
however, will always be identified if it is present in the list.
For example, given a list that contains parent, grandparent,
great-great-grandparent and root shard ranges, but is missing the
great-grandparent shard range, only the parent, grand-parent and root
shard ranges will be identified.
:param shard_ranges: a list of instances of
:class:`~swift.common.utils.ShardRange`
:return: a list of instances of
:class:`~swift.common.utils.ShardRange` containing items in the
given ``shard_ranges`` that can be identified as ancestors of this
shard range. The list may not be complete if there are gaps in the
ancestry, but is guaranteed to contain at least the parent and
root shard ranges if they are present.
"""
if not shard_ranges:
return []
try:
self_parsed_name = ShardName.parse(self.name)
except ValueError:
# not a shard
return []
ancestors = []
for sr in shard_ranges:
if self.is_child_of(sr):
ancestors.append(sr)
break
if ancestors:
ancestors.extend(ancestors[0].find_ancestors(shard_ranges))
else:
root_sr = self._find_root(self_parsed_name, shard_ranges)
if root_sr:
ancestors.append(root_sr)
return ancestors
@classmethod
def make_path(cls, shards_account, root_container, parent_container,
timestamp, index):

5
swift/container/backend.py
View File

@ -454,10 +454,7 @@ class ContainerBroker(DatabaseBroker):
for sharding to have been initiated, False otherwise.
"""
own_shard_range = self.get_own_shard_range()
if own_shard_range.state in (ShardRange.SHARDING,
ShardRange.SHRINKING,
ShardRange.SHARDED,
ShardRange.SHRUNK):
if own_shard_range.state in ShardRange.CLEAVING_STATES:
return bool(self.get_shard_ranges())
return False

125
swift/container/sharder.py
View File

@ -40,7 +40,7 @@ from swift.common.utils import get_logger, config_true_value, \
Everything, config_auto_int_value, ShardRangeList, config_percent_value
from swift.container.backend import ContainerBroker, \
RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, COLLAPSED, \
SHARD_UPDATE_STATES
SHARD_UPDATE_STATES, sift_shard_ranges
from swift.container.replicator import ContainerReplicator
@ -101,7 +101,7 @@ def _find_discontinuity(paths, start):
return longest_start_path, longest_end_path
def find_paths_with_gaps(shard_ranges):
def find_paths_with_gaps(shard_ranges, within_range=None):
"""
Find gaps in the shard ranges and pairs of shard range paths that lead to
and from those gaps. For each gap a single pair of adjacent paths is
@ -109,6 +109,9 @@ def find_paths_with_gaps(shard_ranges):
entire namespace with no overlaps.
:param shard_ranges: a list of instances of ShardRange.
:param within_range: an optional ShardRange that constrains the search
space; the method will only return gaps within this range. The default
is the entire namespace.
:return: A list of tuples of ``(start_path, gap_range, end_path)`` where
``start_path`` is a list of ShardRanges leading to the gap,
``gap_range`` is a ShardRange synthesized to describe the namespace
@ -119,6 +122,7 @@ def find_paths_with_gaps(shard_ranges):
namespace.
"""
timestamp = Timestamp.now()
within_range = within_range or ShardRange('entire/namespace', timestamp)
shard_ranges = ShardRangeList(shard_ranges)
# note: find_paths results do not include shrinking ranges
paths = find_paths(shard_ranges)
@ -149,7 +153,8 @@ def find_paths_with_gaps(shard_ranges):
timestamp,
lower=start_path.upper,
upper=end_path.lower)
paths_with_gaps.append((start_path, gap_range, end_path))
if gap_range.overlaps(within_range):
paths_with_gaps.append((start_path, gap_range, end_path))
return paths_with_gaps
@ -497,6 +502,27 @@ def rank_paths(paths, shard_range_to_span):
return paths
def combine_shard_ranges(new_shard_ranges, existing_shard_ranges):
"""
Combines new and existing shard ranges based on most recent state.
:param new_shard_ranges: a list of ShardRange instances.
:param existing_shard_ranges: a list of ShardRange instances.
:return: a list of ShardRange instances.
"""
new_shard_ranges = [dict(sr) for sr in new_shard_ranges]
existing_shard_ranges = [dict(sr) for sr in existing_shard_ranges]
to_add, to_delete = sift_shard_ranges(
new_shard_ranges,
dict((sr['name'], sr) for sr in existing_shard_ranges))
result = [ShardRange.from_dict(existing)
for existing in existing_shard_ranges
if existing['name'] not in to_delete]
result.extend([ShardRange.from_dict(sr) for sr in to_add])
return sorted([sr for sr in result if not sr.deleted],
key=ShardRange.sort_key)
class CleavingContext(object):
"""
Encapsulates metadata associated with the process of cleaving a retiring
@ -916,9 +942,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
if db_state not in (UNSHARDED, SHARDING, SHARDED):
return
own_shard_range = broker.get_own_shard_range()
if own_shard_range.state not in (
ShardRange.SHARDING, ShardRange.SHARDED,
ShardRange.SHRINKING, ShardRange.SHRUNK):
if own_shard_range.state not in ShardRange.CLEAVING_STATES:
return
if db_state == SHARDED:
@ -1159,7 +1183,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
warnings = []
own_shard_range = broker.get_own_shard_range()
if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED):
if own_shard_range.state in ShardRange.SHARDING_STATES:
shard_ranges = [sr for sr in broker.get_shard_ranges()
if sr.state != ShardRange.SHRINKING]
paths_with_gaps = find_paths_with_gaps(shard_ranges)
@ -1245,9 +1269,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
own_shard_range = broker.get_own_shard_range()
if (orig_own_shard_range != own_shard_range or
orig_own_shard_range.state != own_shard_range.state):
self.logger.info(
'Updated own shard range from %s to %s',
orig_own_shard_range, own_shard_range)
self.logger.info('Updated own shard range from %s to %s',
orig_own_shard_range, own_shard_range)
elif shard_range.is_child_of(own_shard_range):
children_shard_ranges.append(shard_range)
else:
@ -1262,19 +1285,70 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
len(children_shard_ranges))
broker.merge_shard_ranges(children_shard_ranges)
if (other_shard_ranges and
own_shard_range.state in ShardRange.SHRINKING_STATES):
# If own_shard_range state is shrinking, save off *all* shards
# returned because these may contain shards into which this
# shard is to shrink itself; shrinking is the only case when we
# want to learn about *other* shard ranges from the root.
# We need to include shrunk state too, because one replica of a
# shard may already have moved the own_shard_range state to
# shrunk while another replica may still be in the process of
# shrinking.
self.logger.debug('Updating %s other shard range(s) from root',
len(other_shard_ranges))
broker.merge_shard_ranges(other_shard_ranges)
if (other_shard_ranges
and own_shard_range.state in ShardRange.CLEAVING_STATES
and not broker.is_sharded()):
# Other shard ranges returned from the root may need to be merged
# for the purposes of sharding or shrinking this shard:
#
# Shrinking states: If the up-to-date state is shrinking, the
# shards fetched from root may contain shards into which this shard
# is to shrink itself. Shrinking is initiated by modifying multiple
# neighboring shard range states *in the root*, rather than
# modifying a shard directly. We therefore need to learn about
# *other* neighboring shard ranges from the root, possibly
# including the root itself. We need to include shrunk state too,
# because one replica of a shard may already have moved the
# own_shard_range state to shrunk while another replica may still
# be in the process of shrinking.
#
# Sharding states: Normally a shard will shard to its own children.
# However, in some circumstances a shard may need to shard to other
# non-children sub-shards. For example, a shard range repair may
# cause a child sub-shard to be deleted and its namespace covered
# by another 'acceptor' shard.
#
# Therefore, if the up-to-date own_shard_range state indicates that
# sharding or shrinking is in progress, then other shard ranges
# will be merged, with the following caveats: we never expect a
# shard to shard to any ancestor shard range including the root,
# but containers might ultimately *shrink* to root; we never want
# to cleave to a container that is itself sharding or shrinking;
# the merged shard ranges should not result in gaps or overlaps in
# the namespace of this shard.
#
# Note: the search for ancestors is guaranteed to find the parent
# and root *if they are present*, but if any ancestor is missing
# then there is a chance that older generations in the
# other_shard_ranges will not be filtered and could be merged. That
# is only a problem if they are somehow still in ACTIVE state, and
# no overlap is detected, so the ancestor is merged.
ancestor_names = [
sr.name for sr in own_shard_range.find_ancestors(shard_ranges)]
filtered_other_shard_ranges = [
sr for sr in other_shard_ranges
if (sr.name not in ancestor_names
and (sr.state not in ShardRange.CLEAVING_STATES
or sr.deleted))
]
if own_shard_range.state in ShardRange.SHRINKING_STATES:
root_shard_range = own_shard_range.find_root(
other_shard_ranges)
if (root_shard_range and
root_shard_range.state == ShardRange.ACTIVE):
filtered_other_shard_ranges.append(root_shard_range)
existing_shard_ranges = broker.get_shard_ranges()
combined_shard_ranges = combine_shard_ranges(
filtered_other_shard_ranges, existing_shard_ranges)
overlaps = find_overlapping_ranges(combined_shard_ranges)
paths_with_gaps = find_paths_with_gaps(
combined_shard_ranges, own_shard_range)
if not (overlaps or paths_with_gaps):
# only merge if shard ranges appear to be *good*
self.logger.debug(
'Updating %s other shard range(s) from root',
len(filtered_other_shard_ranges))
broker.merge_shard_ranges(filtered_other_shard_ranges)
return own_shard_range, own_shard_range_from_root
@ -2168,10 +2242,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
broker, shard_ranges=[broker.get_own_shard_range()])
own_shard_range = broker.get_own_shard_range()
if own_shard_range.state in (ShardRange.SHARDING,
ShardRange.SHRINKING,
ShardRange.SHARDED,
ShardRange.SHRUNK):
if own_shard_range.state in ShardRange.CLEAVING_STATES:
if broker.get_shard_ranges():
# container has been given shard ranges rather than
# found them e.g. via replication or a shrink event,

9
test/probe/test_sharder.py
View File

@ -1801,13 +1801,12 @@ class TestContainerSharding(BaseAutoContainerSharding):
donor = orig_shard_ranges[0]
shard_nodes_data = self.direct_get_container_shard_ranges(
donor.account, donor.container)
# the donor's shard range will have the acceptor's projected stats;
# donor also has copy of root shard range that will be ignored;
# note: expected_shards does not include the sharded root range
# donor has the acceptor shard range but not the root shard range
# because the root is still in ACTIVE state;
# the donor's shard range will have the acceptor's projected stats
obj_count, bytes_used = check_shard_nodes_data(
shard_nodes_data, expected_state='sharded', expected_shards=1,
exp_obj_count=len(second_shard_objects) + 1,
exp_sharded_root_range=True)
exp_obj_count=len(second_shard_objects) + 1)
# but the donor is empty and so reports zero stats
self.assertEqual(0, obj_count)
self.assertEqual(0, bytes_used)

45
test/unit/cli/test_manage_shard_ranges.py
View File

@ -24,7 +24,7 @@ from tempfile import mkdtemp
import six
from six.moves import cStringIO as StringIO
from swift.cli.manage_shard_ranges import main, combine_shard_ranges
from swift.cli.manage_shard_ranges import main
from swift.common import utils
from swift.common.utils import Timestamp, ShardRange
from swift.container.backend import ContainerBroker
@ -2858,46 +2858,3 @@ class TestManageShardRanges(unittest.TestCase):
self.assertIn(
"argument --yes/-y: not allowed with argument --dry-run/-n",
err_lines[-2], err_lines)
def test_combine_shard_ranges(self):
ts_iter = make_timestamp_iter()
this = ShardRange('a/o', next(ts_iter).internal)
that = ShardRange('a/o', next(ts_iter).internal)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
ts = next(ts_iter).internal
this = ShardRange('a/o', ts, state=ShardRange.ACTIVE,
state_timestamp=next(ts_iter))
that = ShardRange('a/o', ts, state=ShardRange.CREATED,
state_timestamp=next(ts_iter))
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
that.update_meta(1, 2, meta_timestamp=next(ts_iter))
this.update_meta(3, 4, meta_timestamp=next(ts_iter))
expected = that.copy(object_count=this.object_count,
bytes_used=this.bytes_used,
meta_timestamp=this.meta_timestamp)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(expected)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(expected)], [dict(sr) for sr in actual])
this = ShardRange('a/o', next(ts_iter).internal)
that = ShardRange('a/o', next(ts_iter).internal, deleted=True)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertFalse(actual, [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertFalse(actual, [dict(sr) for sr in actual])
this = ShardRange('a/o', next(ts_iter).internal, deleted=True)
that = ShardRange('a/o', next(ts_iter).internal)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])

127
test/unit/common/test_utils.py
View File

@ -8061,6 +8061,19 @@ class TestShardRange(unittest.TestCase):
def setUp(self):
self.ts_iter = make_timestamp_iter()
def test_constants(self):
self.assertEqual({utils.ShardRange.SHARDING,
utils.ShardRange.SHARDED,
utils.ShardRange.SHRINKING,
utils.ShardRange.SHRUNK},
set(utils.ShardRange.CLEAVING_STATES))
self.assertEqual({utils.ShardRange.SHARDING,
utils.ShardRange.SHARDED},
set(utils.ShardRange.SHARDING_STATES))
self.assertEqual({utils.ShardRange.SHRINKING,
utils.ShardRange.SHRUNK},
set(utils.ShardRange.SHRINKING_STATES))
def test_min_max_bounds(self):
with self.assertRaises(TypeError):
utils.ShardRangeOuterBound()
@ -9099,6 +9112,120 @@ class TestShardRange(unittest.TestCase):
self.assertTrue(a1_r1_gp1.is_child_of(a2_r1))
self.assertTrue(a2_r1_gp1.is_child_of(a1_r1))
def test_find_root(self):
# account 1
ts = next(self.ts_iter)
a1_r1 = utils.ShardRange('a1/r1', ts)
ts = next(self.ts_iter)
a1_r1_gp1 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', 'r1', ts, 1), ts, '', 'l')
ts = next(self.ts_iter)
a1_r1_gp1_p1 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', a1_r1_gp1.container, ts, 1), ts, 'a', 'k')
ts = next(self.ts_iter)
a1_r1_gp1_p1_c1 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', a1_r1_gp1_p1.container, ts, 1), ts, 'a', 'j')
ts = next(self.ts_iter)
a1_r1_gp1_p2 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', a1_r1_gp1.container, ts, 2), ts, 'k', 'l')
ts = next(self.ts_iter)
a1_r1_gp2 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', 'r1', ts, 2), ts, 'l', '') # different index
# full ancestry plus some others
all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1,
a1_r1_gp1_p2, a1_r1_gp2]
random.shuffle(all_shard_ranges)
self.assertIsNone(a1_r1.find_root(all_shard_ranges))
self.assertEqual(a1_r1, a1_r1_gp1.find_root(all_shard_ranges))
self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges))
self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges))
# missing a1_r1_gp1_p1
all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1_c1,
a1_r1_gp1_p2, a1_r1_gp2]
random.shuffle(all_shard_ranges)
self.assertIsNone(a1_r1.find_root(all_shard_ranges))
self.assertEqual(a1_r1, a1_r1_gp1.find_root(all_shard_ranges))
self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges))
self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges))
# empty list
self.assertIsNone(a1_r1_gp1_p1_c1.find_root([]))
# double entry
all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1, a1_r1_gp1]
random.shuffle(all_shard_ranges)
self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges))
self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges))
def test_find_ancestors(self):
# account 1
ts = next(self.ts_iter)
a1_r1 = utils.ShardRange('a1/r1', ts)
ts = next(self.ts_iter)
a1_r1_gp1 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', 'r1', ts, 1), ts, '', 'l')
ts = next(self.ts_iter)
a1_r1_gp1_p1 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', a1_r1_gp1.container, ts, 1), ts, 'a', 'k')
ts = next(self.ts_iter)
a1_r1_gp1_p1_c1 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', a1_r1_gp1_p1.container, ts, 1), ts, 'a', 'j')
ts = next(self.ts_iter)
a1_r1_gp1_p2 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', a1_r1_gp1.container, ts, 2), ts, 'k', 'l')
ts = next(self.ts_iter)
a1_r1_gp2 = utils.ShardRange(utils.ShardRange.make_path(
'.shards_a1', 'r1', 'r1', ts, 2), ts, 'l', '') # different index
# full ancestry plus some others
all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1,
a1_r1_gp1_p2, a1_r1_gp2]
random.shuffle(all_shard_ranges)
self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1_gp1, a1_r1],
a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1_gp1_p1, a1_r1_gp1, a1_r1],
a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
# missing a1_r1_gp1_p1
all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1_c1,
a1_r1_gp1_p2, a1_r1_gp2]
random.shuffle(all_shard_ranges)
self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1_gp1, a1_r1],
a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1],
a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
# missing a1_r1_gp1
all_shard_ranges = [a1_r1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1,
a1_r1_gp1_p2, a1_r1_gp2]
random.shuffle(all_shard_ranges)
self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1],
a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1_gp1_p1, a1_r1],
a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
# empty list
self.assertEqual([], a1_r1_gp1_p1_c1.find_ancestors([]))
# double entry
all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1, a1_r1_gp1]
random.shuffle(all_shard_ranges)
self.assertEqual([a1_r1_gp1, a1_r1],
a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
self.assertEqual([a1_r1],
a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1_p1, a1_r1_gp1_p1]
random.shuffle(all_shard_ranges)
self.assertEqual([a1_r1_gp1_p1, a1_r1],
a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
def test_expand(self):
bounds = (('', 'd'), ('d', 'k'), ('k', 't'), ('t', ''))
donors = [

636
test/unit/container/test_sharder.py
View File

@ -43,7 +43,7 @@ from swift.container.sharder import ContainerSharder, sharding_enabled, \
find_shrinking_candidates, process_compactible_shard_sequences, \
find_compactible_shard_sequences, is_shrinking_candidate, \
is_sharding_candidate, find_paths, rank_paths, ContainerSharderConf, \
find_paths_with_gaps
find_paths_with_gaps, combine_shard_ranges
from swift.common.utils import ShardRange, Timestamp, hash_path, \
encode_timestamps, parse_db_filename, quorum_size, Everything, md5, \
ShardName
@ -5840,7 +5840,7 @@ class TestSharder(BaseTestSharder):
def _do_test_audit_shard_container_merge_other_ranges(self, *args):
# verify that shard only merges other ranges from root when it is
# shrinking or shrunk
# cleaving
shard_bounds = (
('a', 'p'), ('k', 't'), ('p', 'u'))
shard_states = (
@ -5855,7 +5855,7 @@ class TestSharder(BaseTestSharder):
broker.set_sharding_sysmeta(*args)
shard_ranges[1].name = broker.path
# make own shard range match shard_ranges[1]
# make shard's own shard range match shard_ranges[1]
own_sr = shard_ranges[1]
expected_stats = {'attempted': 1, 'success': 1, 'failure': 0}
self.assertTrue(own_sr.update_state(own_state,
@ -5903,7 +5903,7 @@ class TestSharder(BaseTestSharder):
self.assertEqual(root_state, own_shard_range.state)
self.assertEqual(root_ts, own_shard_range.state_timestamp)
updated_ranges = broker.get_shard_ranges(include_own=True)
if root_state in (ShardRange.SHRINKING, ShardRange.SHRUNK):
if root_state in ShardRange.CLEAVING_STATES:
# check other shard ranges from root are merged
self.assertEqual(shard_ranges, updated_ranges)
else:
@ -5925,7 +5925,7 @@ class TestSharder(BaseTestSharder):
self.assertEqual(own_state, own_shard_range.state)
self.assertEqual(own_ts, own_shard_range.state_timestamp)
updated_ranges = broker.get_shard_ranges(include_own=True)
if own_state in (ShardRange.SHRINKING, ShardRange.SHRUNK):
if own_state in ShardRange.CLEAVING_STATES:
# check other shard ranges from root are merged
self.assertEqual(shard_ranges, updated_ranges)
else:
@ -5940,7 +5940,7 @@ class TestSharder(BaseTestSharder):
'a/c')
def _assert_merge_into_shard(self, own_shard_range, shard_ranges,
root_shard_ranges, expected, *args):
root_shard_ranges, expected, *args, **kwargs):
# create a shard broker, initialise with shard_ranges, run audit on it
# supplying given root_shard_ranges and verify that the broker ends up
# with expected shard ranges.
@ -5948,6 +5948,13 @@ class TestSharder(BaseTestSharder):
container=own_shard_range.container)
broker.set_sharding_sysmeta(*args)
broker.merge_shard_ranges([own_shard_range] + shard_ranges)
db_state = kwargs.get('db_state', UNSHARDED)
if db_state == SHARDING:
broker.set_sharding_state()
if db_state == SHARDED:
broker.set_sharding_state()
broker.set_sharded_state()
self.assertEqual(db_state, broker.get_db_state())
self.assertFalse(broker.is_root_container())
sharder, mock_swift = self.call_audit_container(
@ -5998,26 +6005,26 @@ class TestSharder(BaseTestSharder):
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.STATES:
if own_state in ShardRange.SHRINKING_STATES:
# shrinking states are covered by other tests
if own_state in ShardRange.CLEAVING_STATES:
# cleaving states are covered by other tests
continue
for acceptor_state in ShardRange.STATES:
for root_state in ShardRange.STATES:
do_test(own_state, acceptor_state, root_state)
def test_audit_old_style_shard_root_ranges_not_merged_not_shrinking(self):
def test_audit_old_style_shard_root_ranges_not_merged_not_cleaving(self):
# verify that other shard ranges from root are NOT merged into shard
# when it is NOT in a shrinking state
# when it is NOT in a cleaving state
self._do_test_audit_shard_root_ranges_not_merged('Root', 'a/c')
def test_audit_shard_root_ranges_not_merged_not_shrinking(self):
def test_audit_shard_root_ranges_not_merged_not_cleaving(self):
# verify that other shard ranges from root are NOT merged into shard
# when it is NOT in a shrinking state
# when it is NOT in a cleaving state
self._do_test_audit_shard_root_ranges_not_merged('Quoted-Root', 'a/c')
def test_audit_shard_root_ranges_with_own_merged_while_shrinking(self):
# Verify that shrinking shard will merge root and other ranges,
# including root range.
# Verify that shrinking shard will merge other ranges, but not
# in-ACTIVE root range.
# Make root and other ranges that fully contain the shard namespace...
root_own_sr = ShardRange('a/c', next(self.ts_iter))
acceptor = ShardRange(
@ -6034,7 +6041,7 @@ class TestSharder(BaseTestSharder):
own_sr = ShardRange(
str(ShardName.create('.shards_a', 'c', 'c', ts, 0)),
ts, lower='a', upper='b', state=own_state, state_timestamp=ts)
expected = [acceptor_from_root, root_from_root]
expected = [acceptor_from_root]
with annotate_failure('with states %s %s %s'
% (own_state, acceptor_state, root_state)):
sharder = self._assert_merge_into_shard(
@ -6047,12 +6054,21 @@ class TestSharder(BaseTestSharder):
for own_state in ShardRange.SHRINKING_STATES:
for acceptor_state in ShardRange.STATES:
if acceptor_state in ShardRange.CLEAVING_STATES:
# special case covered in other tests
continue
for root_state in ShardRange.STATES:
do_test(own_state, acceptor_state, root_state)
if root_state == ShardRange.ACTIVE:
# special case: ACTIVE root *is* merged
continue
with annotate_failure(
'with states %s %s %s'
% (own_state, acceptor_state, root_state)):
do_test(own_state, acceptor_state, root_state)
def test_audit_shard_root_ranges_missing_own_merged_while_shrinking(self):
# Verify that shrinking shard will merge root and other ranges,
# including root range.
# Verify that shrinking shard will merge other ranges, but not
# in-ACTIVE root range, even when root does not have shard's own range.
# Make root and other ranges that fully contain the shard namespace...
root_own_sr = ShardRange('a/c', next(self.ts_iter))
acceptor = ShardRange(
@ -6069,7 +6085,7 @@ class TestSharder(BaseTestSharder):
own_sr = ShardRange(
str(ShardName.create('.shards_a', 'c', 'c', ts, 0)),
ts, lower='a', upper='b', state=own_state, state_timestamp=ts)
expected = [acceptor_from_root, root_from_root]
expected = [acceptor_from_root]
with annotate_failure('with states %s %s %s'
% (own_state, acceptor_state, root_state)):
sharder = self._assert_merge_into_shard(
@ -6085,8 +6101,86 @@ class TestSharder(BaseTestSharder):
for own_state in ShardRange.SHRINKING_STATES:
for acceptor_state in ShardRange.STATES:
if acceptor_state in ShardRange.CLEAVING_STATES:
# special case covered in other tests
continue
for root_state in ShardRange.STATES:
do_test(own_state, acceptor_state, root_state)
if root_state == ShardRange.ACTIVE:
# special case: ACTIVE root *is* merged
continue
with annotate_failure(
'with states %s %s %s'
% (own_state, acceptor_state, root_state)):
do_test(own_state, acceptor_state, root_state)
def test_audit_shard_root_range_not_merged_while_shrinking(self):
# Verify that shrinking shard will not merge an in-active root range
def do_test(own_state, root_state):
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
expected = []
sharder = self._assert_merge_into_shard(
own_sr, [], [own_sr, root_own_sr],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.SHRINKING_STATES:
for root_state in ShardRange.STATES:
if root_state == ShardRange.ACTIVE:
continue # special case tested below
with annotate_failure((own_state, root_state)):
do_test(own_state, root_state)
def test_audit_shard_root_range_overlap_not_merged_while_shrinking(self):
# Verify that shrinking shard will not merge an active root range that
# overlaps with an exosting sub-shard
def do_test(own_state):
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.ACTIVE)
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
ts = next(self.ts_iter)
sub_shard = ShardRange(
str(ShardName.create(
'.shards_a', 'c', own_sr.container, ts, 0)),
ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
expected = [sub_shard]
sharder = self._assert_merge_into_shard(
own_sr, [sub_shard], [own_sr, root_own_sr],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.SHRINKING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_active_root_range_merged_while_shrinking(self):
# Verify that shrinking shard will merge an active root range
def do_test(own_state):
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.ACTIVE)
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
expected = [root_own_sr]
sharder = self._assert_merge_into_shard(
own_sr, [], [own_sr, root_own_sr],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.SHRINKING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_root_ranges_fetch_fails_while_shrinking(self):
# check audit copes with failed response while shard is shrinking
@ -6106,6 +6200,425 @@ class TestSharder(BaseTestSharder):
warning_lines[1])
self.assertFalse(sharder.logger.get_lines_for_level('error'))
def test_audit_shard_root_ranges_merge_while_unsharded(self):
# Verify that unsharded shard with no existing shard ranges will merge
# other ranges, but not root range.
root_own_sr = ShardRange('a/c', next(self.ts_iter))
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
def do_test(own_state, acceptor_state, root_state):
acceptor_from_root = acceptor.copy(
timestamp=next(self.ts_iter), state=acceptor_state)
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
root_from_root = root_own_sr.copy(
timestamp=next(self.ts_iter), state=root_state)
expected = [acceptor_from_root]
sharder = self._assert_merge_into_shard(
own_sr, [],
[own_sr, acceptor_from_root, root_from_root],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.SHARDING_STATES:
for acceptor_state in ShardRange.STATES:
if acceptor_state in ShardRange.CLEAVING_STATES:
# special case covered in other tests
continue
for root_state in ShardRange.STATES:
with annotate_failure(
'with states %s %s %s'
% (own_state, acceptor_state, root_state)):
do_test(own_state, acceptor_state, root_state)
def test_audit_shard_root_ranges_merge_while_sharding(self):
# Verify that sharding shard with no existing shard ranges will merge
# other ranges, but not root range.
root_own_sr = ShardRange('a/c', next(self.ts_iter))
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
def do_test(own_state, acceptor_state, root_state):
acceptor_from_root = acceptor.copy(
timestamp=next(self.ts_iter), state=acceptor_state)
ts = next(self.ts_iter)
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', ts, 0)),
ts, 'a', 'b', epoch=ts, state=own_state)
root_from_root = root_own_sr.copy(
timestamp=next(self.ts_iter), state=root_state)
expected = [acceptor_from_root]
sharder = self._assert_merge_into_shard(
own_sr, [],
[own_sr, acceptor_from_root, root_from_root],
expected, 'Quoted-Root', 'a/c', db_state=SHARDING)
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.SHARDING_STATES:
for acceptor_state in ShardRange.STATES:
if acceptor_state in ShardRange.CLEAVING_STATES:
# special case covered in other tests
continue
for root_state in ShardRange.STATES:
with annotate_failure(
'with states %s %s %s'
% (own_state, acceptor_state, root_state)):
do_test(own_state, acceptor_state, root_state)
def test_audit_shard_root_ranges_not_merged_once_sharded(self):
# Verify that sharded shard will not merge other ranges from root
root_own_sr = ShardRange('a/c', next(self.ts_iter))
# the acceptor complements the single existing sub-shard...
other_sub_shard = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'ab', 'c', state=ShardRange.ACTIVE)
def do_test(own_state, other_sub_shard_state, root_state):
sub_shard_from_root = other_sub_shard.copy(
timestamp=next(self.ts_iter), state=other_sub_shard_state)
ts = next(self.ts_iter)
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', ts, 0)),
ts, 'a', 'b', epoch=ts, state=own_state)
ts = next(self.ts_iter)
sub_shard = ShardRange(
str(ShardName.create(
'.shards_a', 'c', own_sr.container, ts, 0)),
ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
root_from_root = root_own_sr.copy(
timestamp=next(self.ts_iter), state=root_state)
expected = [sub_shard]
sharder = self._assert_merge_into_shard(
own_sr, [sub_shard],
[own_sr, sub_shard_from_root, root_from_root],
expected, 'Quoted-Root', 'a/c', db_state=SHARDED)
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in (ShardRange.SHARDED, ShardRange.SHRUNK):
for other_sub_shard_state in ShardRange.STATES:
for root_state in ShardRange.STATES:
with annotate_failure(
'with states %s %s %s'
% (own_state, other_sub_shard_state, root_state)):
do_test(own_state, other_sub_shard_state, root_state)
def test_audit_shard_root_ranges_replace_existing_while_cleaving(self):
# Verify that sharding shard with stale existing sub-shard ranges will
# merge other ranges, but not root range.
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
ts = next(self.ts_iter)
acceptor_sub_shards = [ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', acceptor.container, ts, i)),
ts, lower, upper, state=ShardRange.ACTIVE)
for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
# shard has incomplete existing shard ranges, ranges from root delete
# existing sub-shard and replace with other acceptor sub-shards
def do_test(own_state):
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
ts = next(self.ts_iter)
sub_shard = ShardRange(
str(ShardName.create(
'.shards_a', 'c', own_sr.container, ts, 0)),
ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
deleted_sub_shard = sub_shard.copy(
timestamp=next(self.ts_iter), state=ShardRange.SHARDED,
deleted=1)
expected = acceptor_sub_shards
sharder = self._assert_merge_into_shard(
own_sr, [sub_shard],
[root_own_sr, own_sr, deleted_sub_shard] + acceptor_sub_shards,
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.CLEAVING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_root_ranges_supplement_deleted_while_cleaving(self):
# Verify that sharding shard with deleted existing sub-shard ranges
# will merge other ranges while sharding, but not root range.
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
ts = next(self.ts_iter)
acceptor_sub_shards = [ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', acceptor.container, ts, i)),
ts, lower, upper, state=ShardRange.ACTIVE)
for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
# shard already has deleted existing shard ranges
expected = acceptor_sub_shards
def do_test(own_state):
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
ts = next(self.ts_iter)
deleted_sub_shards = [ShardRange(
str(ShardName.create(
'.shards_a', 'c', own_sr.container, ts, i)),
ts, lower, upper, state=ShardRange.SHARDED, deleted=1)
for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'b'))]
sharder = self._assert_merge_into_shard(
own_sr, deleted_sub_shards,
[own_sr, root_own_sr] + acceptor_sub_shards,
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.CLEAVING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_root_ranges_supplement_existing_while_cleaving(self):
# Verify that sharding shard with incomplete existing sub-shard ranges
# will merge other ranges that fill the gap, but not root range.
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
ts = next(self.ts_iter)
acceptor_sub_shards = [ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', acceptor.container, ts, i)),
ts, lower, upper, state=ShardRange.ACTIVE)
for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
# shard has incomplete existing shard ranges and range from root fills
# the gap
def do_test(own_state):
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
ts = next(self.ts_iter)
sub_shard = ShardRange(
str(ShardName.create(
'.shards_a', 'c', own_sr.container, ts, 0)),
ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
expected = [sub_shard] + acceptor_sub_shards[1:]
sharder = self._assert_merge_into_shard(
own_sr, [sub_shard],
[own_sr, root_own_sr] + acceptor_sub_shards[1:],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.CLEAVING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_root_ranges_cleaving_not_merged_while_cleaving(self):
# Verify that sharding shard will not merge other ranges that are in a
# cleaving state.
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
def do_test(own_state, acceptor_state, root_state):
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
root_from_root = root_own_sr.copy(
timestamp=next(self.ts_iter), state=root_state)
acceptor_from_root = acceptor.copy(
timestamp=next(self.ts_iter), state=acceptor_state)
if (own_state in ShardRange.SHRINKING_STATES and
root_state == ShardRange.ACTIVE):
# special case: when shrinking, ACTIVE root shard *is* merged
expected = [root_from_root]
else:
expected = []
sharder = self._assert_merge_into_shard(
own_sr, [],
[own_sr, acceptor_from_root, root_from_root],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
# ranges from root that are in a cleaving state are not merged...
for own_state in ShardRange.CLEAVING_STATES:
for acceptor_state in ShardRange.CLEAVING_STATES:
for root_state in ShardRange.STATES:
with annotate_failure(
'with states %s %s %s'
% (own_state, acceptor_state, root_state)):
do_test(own_state, acceptor_state, root_state)
def test_audit_shard_root_ranges_overlap_not_merged_while_cleaving_1(self):
# Verify that sharding/shrinking shard will not merge other ranges that
# would create an overlap; shard has complete existing shard ranges,
# newer range from root ignored
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
def do_test(own_state):
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
ts = next(self.ts_iter)
sub_shards = [ShardRange(
str(ShardName.create(
'.shards_a', 'c', own_sr.container, ts, i)),
ts, lower, upper, state=ShardRange.ACTIVE)
for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'b'))]
acceptor_from_root = acceptor.copy(timestamp=next(self.ts_iter))
expected = sub_shards
sharder = self._assert_merge_into_shard(
own_sr, sub_shards,
[own_sr, acceptor_from_root, root_own_sr],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.CLEAVING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_root_ranges_overlap_not_merged_while_cleaving_2(self):
# Verify that sharding/shrinking shard will not merge other ranges that
# would create an overlap; shard has incomplete existing shard ranges
# but ranges from root overlaps
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
ts = next(self.ts_iter)
acceptor_sub_shards = [ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', acceptor.container, ts, i)),
ts, lower, upper, state=ShardRange.ACTIVE)
for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
def do_test(own_state):
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
ts = next(self.ts_iter)
sub_shard = ShardRange(
str(ShardName.create(
'.shards_a', 'c', own_sr.container, ts, 0)),
ts, lower='a', upper='abc', state=ShardRange.ACTIVE)
expected = [sub_shard]
sharder = self._assert_merge_into_shard(
own_sr, [sub_shard],
acceptor_sub_shards[1:] + [own_sr, root_own_sr],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.CLEAVING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_root_ranges_with_gap_not_merged_while_cleaving(self):
# Verify that sharding/shrinking shard will not merge other ranges that
# would leave a gap.
root_own_sr = ShardRange('a/c', next(self.ts_iter),
state=ShardRange.SHARDED)
acceptor = ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', 'c', next(self.ts_iter), 1)),
next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
ts = next(self.ts_iter)
acceptor_sub_shards = [ShardRange(
str(ShardRange.make_path(
'.shards_a', 'c', acceptor.container, ts, i)),
ts, lower, upper, state=ShardRange.ACTIVE)
for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
def do_test(own_state):
own_sr = ShardRange(
str(ShardName.create(
'.shards_a', 'c', 'c', next(self.ts_iter), 0)),
next(self.ts_iter), 'a', 'b', state=own_state)
# root ranges have gaps w.r.t. the shard namespace
existing = expected = []
sharder = self._assert_merge_into_shard(
own_sr, existing,
acceptor_sub_shards[:1] + [own_sr, root_own_sr],
expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
for own_state in ShardRange.CLEAVING_STATES:
with annotate_failure(own_state):
do_test(own_state)
def test_audit_shard_container_ancestors_not_merged_while_sharding(self):
# Verify that sharding shard will not merge parent and root shard
# ranges even when the sharding shard has no other ranges
root_sr = ShardRange('a/root', next(self.ts_iter),
state=ShardRange.SHARDED)
grandparent_path = ShardRange.make_path(
'.shards_a', 'root', root_sr.container, next(self.ts_iter), 2)
grandparent_sr = ShardRange(grandparent_path, next(self.ts_iter),
'', 'd', state=ShardRange.ACTIVE)
self.assertTrue(grandparent_sr.is_child_of(root_sr))
parent_path = ShardRange.make_path(
'.shards_a', 'root', grandparent_sr.container, next(self.ts_iter),
2)
parent_sr = ShardRange(parent_path, next(self.ts_iter), '', 'd',
state=ShardRange.ACTIVE)
self.assertTrue(parent_sr.is_child_of(grandparent_sr))
child_path = ShardRange.make_path(
'.shards_a', 'root', parent_sr.container, next(self.ts_iter), 2)
child_own_sr = ShardRange(child_path, next(self.ts_iter), 'a', 'b',
state=ShardRange.SHARDING)
self.assertTrue(child_own_sr.is_child_of(parent_sr))
ranges_from_root = [grandparent_sr, parent_sr, root_sr, child_own_sr]
expected = []
sharder = self._assert_merge_into_shard(
child_own_sr, [], ranges_from_root, expected, 'Quoted-Root', 'a/c')
self.assertFalse(sharder.logger.get_lines_for_level('warning'))
self.assertFalse(sharder.logger.get_lines_for_level('error'))
def test_audit_shard_container_children_merged_while_sharding(self):
# Verify that sharding shard will always merge children shard ranges
def do_test(child_deleted, child_state):
@ -6157,9 +6670,11 @@ class TestSharder(BaseTestSharder):
'GET', '/v1/a/c', expected_headers,
acceptable_statuses=(2,), params=params)
expected = child_srs + [parent_sr]
if child_deleted:
expected.append(other_sr)
self._assert_shard_ranges_equal(
sorted(child_srs + [parent_sr],
key=ShardRange.sort_key),
sorted(expected, key=ShardRange.sort_key),
sorted(broker.get_shard_ranges(
include_own=True, include_deleted=True),
key=ShardRange.sort_key))
@ -8141,7 +8656,7 @@ class TestSharderFunctions(BaseTestSharder):
bounds, ShardRange.ACTIVE,
timestamp=next(self.ts_iter), object_count=1)
paths_with_gaps = find_paths_with_gaps(ranges)
self.assertEqual(3, len(paths_with_gaps))
self.assertEqual(3, len(paths_with_gaps), paths_with_gaps)
self.assertEqual(
[(ShardRange.MIN, ShardRange.MIN),
(ShardRange.MIN, 'a'),
@ -8161,6 +8676,38 @@ class TestSharderFunctions(BaseTestSharder):
[(r.lower, r.upper) for r in paths_with_gaps[2]]
)
range_of_interest = ShardRange('test/range', next(self.ts_iter))
range_of_interest.lower = 'a'
paths_with_gaps = find_paths_with_gaps(ranges, range_of_interest)
self.assertEqual(2, len(paths_with_gaps), paths_with_gaps)
self.assertEqual(
[('k', 'p'),
('p', 'q'),
('q', 'y')],
[(r.lower, r.upper) for r in paths_with_gaps[0]]
)
self.assertEqual(
[('q', 'y'),
('y', ShardRange.MAX),
(ShardRange.MAX, ShardRange.MAX)],
[(r.lower, r.upper) for r in paths_with_gaps[1]]
)
range_of_interest.lower = 'b'
range_of_interest.upper = 'x'
paths_with_gaps = find_paths_with_gaps(ranges, range_of_interest)
self.assertEqual(1, len(paths_with_gaps), paths_with_gaps)
self.assertEqual(
[('k', 'p'),
('p', 'q'),
('q', 'y')],
[(r.lower, r.upper) for r in paths_with_gaps[0]]
)
range_of_interest.upper = 'c'
paths_with_gaps = find_paths_with_gaps(ranges, range_of_interest)
self.assertFalse(paths_with_gaps)
class TestContainerSharderConf(unittest.TestCase):
def test_default(self):
@ -8337,3 +8884,46 @@ class TestContainerSharderConf(unittest.TestCase):
assert_bad({'shard_container_threshold': 100,
'expansion_limit': 100})
assert_ok({'expansion_limit': 100000001})
def test_combine_shard_ranges(self):
ts_iter = make_timestamp_iter()
this = ShardRange('a/o', next(ts_iter).internal)
that = ShardRange('a/o', next(ts_iter).internal)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
ts = next(ts_iter).internal
this = ShardRange('a/o', ts, state=ShardRange.ACTIVE,
state_timestamp=next(ts_iter))
that = ShardRange('a/o', ts, state=ShardRange.CREATED,
state_timestamp=next(ts_iter))
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
that.update_meta(1, 2, meta_timestamp=next(ts_iter))
this.update_meta(3, 4, meta_timestamp=next(ts_iter))
expected = that.copy(object_count=this.object_count,
bytes_used=this.bytes_used,
meta_timestamp=this.meta_timestamp)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(expected)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(expected)], [dict(sr) for sr in actual])
this = ShardRange('a/o', next(ts_iter).internal)
that = ShardRange('a/o', next(ts_iter).internal, deleted=True)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertFalse(actual, [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertFalse(actual, [dict(sr) for sr in actual])
this = ShardRange('a/o', next(ts_iter).internal, deleted=True)
that = ShardRange('a/o', next(ts_iter).internal)
actual = combine_shard_ranges([dict(this)], [dict(that)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])
actual = combine_shard_ranges([dict(that)], [dict(this)])
self.assertEqual([dict(that)], [dict(sr) for sr in actual])