diff --git a/swift/cli/manage_shard_ranges.py b/swift/cli/manage_shard_ranges.py index 31ff894f0c..8d7dfbf3f0 100644 --- a/swift/cli/manage_shard_ranges.py +++ b/swift/cli/manage_shard_ranges.py @@ -168,13 +168,12 @@ from six.moves import input from swift.common.utils import Timestamp, get_logger, ShardRange, readconf, \ ShardRangeList, non_negative_int, config_positive_int_value -from swift.container.backend import ContainerBroker, UNSHARDED, \ - sift_shard_ranges +from swift.container.backend import ContainerBroker, UNSHARDED from swift.container.sharder import make_shard_ranges, sharding_enabled, \ CleavingContext, process_compactible_shard_sequences, \ find_compactible_shard_sequences, find_overlapping_ranges, \ find_paths, rank_paths, finalize_shrinking, DEFAULT_SHARDER_CONF, \ - ContainerSharderConf, find_paths_with_gaps + ContainerSharderConf, find_paths_with_gaps, combine_shard_ranges EXIT_SUCCESS = 0 EXIT_ERROR = 1 @@ -428,27 +427,6 @@ def delete_shard_ranges(broker, args): return EXIT_SUCCESS -def combine_shard_ranges(new_shard_ranges, existing_shard_ranges): - """ - Combines new and existing shard ranges based on most recent state. - - :param new_shard_ranges: a list of ShardRange instances. - :param existing_shard_ranges: a list of ShardRange instances. - :return: a list of ShardRange instances. - """ - new_shard_ranges = [dict(sr) for sr in new_shard_ranges] - existing_shard_ranges = [dict(sr) for sr in existing_shard_ranges] - to_add, to_delete = sift_shard_ranges( - new_shard_ranges, - dict((sr['name'], sr) for sr in existing_shard_ranges)) - result = [ShardRange.from_dict(existing) - for existing in existing_shard_ranges - if existing['name'] not in to_delete] - result.extend([ShardRange.from_dict(sr) for sr in to_add]) - return sorted([sr for sr in result if not sr.deleted], - key=ShardRange.sort_key) - - def merge_shard_ranges(broker, args): _check_own_shard_range(broker, args) shard_data = _load_and_validate_shard_data(args, require_index=False) diff --git a/swift/common/utils.py b/swift/common/utils.py index 2663a50717..6fb492a773 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -5330,6 +5330,8 @@ class ShardRange(object): SHRUNK: 'shrunk'} STATES_BY_NAME = dict((v, k) for k, v in STATES.items()) SHRINKING_STATES = (SHRINKING, SHRUNK) + SHARDING_STATES = (SHARDING, SHARDED) + CLEAVING_STATES = SHRINKING_STATES + SHARDING_STATES @functools.total_ordering class MaxBound(ShardRangeOuterBound): @@ -5434,6 +5436,76 @@ class ShardRange(object): == ShardName.hash_container_name(parent.container) ) + def _find_root(self, parsed_name, shard_ranges): + for sr in shard_ranges: + if parsed_name.root_container == sr.container: + return sr + return None + + def find_root(self, shard_ranges): + """ + Find this shard range's root shard range in the given ``shard_ranges``. + + :param shard_ranges: a list of instances of + :class:`~swift.common.utils.ShardRange` + :return: this shard range's root shard range if it is found in the + list, otherwise None. + """ + try: + self_parsed_name = ShardName.parse(self.name) + except ValueError: + # not a shard + return None + return self._find_root(self_parsed_name, shard_ranges) + + def find_ancestors(self, shard_ranges): + """ + Find this shard range's ancestor ranges in the given ``shard_ranges``. + + This method makes a best-effort attempt to identify this shard range's + parent shard range, the parent's parent, etc., up to and including the + root shard range. It is only possible to directly identify the parent + of a particular shard range, so the search is recursive; if any member + of the ancestry is not found then the search ends and older ancestors + that may be in the list are not identified. The root shard range, + however, will always be identified if it is present in the list. + + For example, given a list that contains parent, grandparent, + great-great-grandparent and root shard ranges, but is missing the + great-grandparent shard range, only the parent, grand-parent and root + shard ranges will be identified. + + :param shard_ranges: a list of instances of + :class:`~swift.common.utils.ShardRange` + :return: a list of instances of + :class:`~swift.common.utils.ShardRange` containing items in the + given ``shard_ranges`` that can be identified as ancestors of this + shard range. The list may not be complete if there are gaps in the + ancestry, but is guaranteed to contain at least the parent and + root shard ranges if they are present. + """ + if not shard_ranges: + return [] + + try: + self_parsed_name = ShardName.parse(self.name) + except ValueError: + # not a shard + return [] + + ancestors = [] + for sr in shard_ranges: + if self.is_child_of(sr): + ancestors.append(sr) + break + if ancestors: + ancestors.extend(ancestors[0].find_ancestors(shard_ranges)) + else: + root_sr = self._find_root(self_parsed_name, shard_ranges) + if root_sr: + ancestors.append(root_sr) + return ancestors + @classmethod def make_path(cls, shards_account, root_container, parent_container, timestamp, index): diff --git a/swift/container/backend.py b/swift/container/backend.py index 9806161930..760a726977 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -454,10 +454,7 @@ class ContainerBroker(DatabaseBroker): for sharding to have been initiated, False otherwise. """ own_shard_range = self.get_own_shard_range() - if own_shard_range.state in (ShardRange.SHARDING, - ShardRange.SHRINKING, - ShardRange.SHARDED, - ShardRange.SHRUNK): + if own_shard_range.state in ShardRange.CLEAVING_STATES: return bool(self.get_shard_ranges()) return False diff --git a/swift/container/sharder.py b/swift/container/sharder.py index 71f2140c63..ab25a19ca5 100644 --- a/swift/container/sharder.py +++ b/swift/container/sharder.py @@ -40,7 +40,7 @@ from swift.common.utils import get_logger, config_true_value, \ Everything, config_auto_int_value, ShardRangeList, config_percent_value from swift.container.backend import ContainerBroker, \ RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, COLLAPSED, \ - SHARD_UPDATE_STATES + SHARD_UPDATE_STATES, sift_shard_ranges from swift.container.replicator import ContainerReplicator @@ -101,7 +101,7 @@ def _find_discontinuity(paths, start): return longest_start_path, longest_end_path -def find_paths_with_gaps(shard_ranges): +def find_paths_with_gaps(shard_ranges, within_range=None): """ Find gaps in the shard ranges and pairs of shard range paths that lead to and from those gaps. For each gap a single pair of adjacent paths is @@ -109,6 +109,9 @@ def find_paths_with_gaps(shard_ranges): entire namespace with no overlaps. :param shard_ranges: a list of instances of ShardRange. + :param within_range: an optional ShardRange that constrains the search + space; the method will only return gaps within this range. The default + is the entire namespace. :return: A list of tuples of ``(start_path, gap_range, end_path)`` where ``start_path`` is a list of ShardRanges leading to the gap, ``gap_range`` is a ShardRange synthesized to describe the namespace @@ -119,6 +122,7 @@ def find_paths_with_gaps(shard_ranges): namespace. """ timestamp = Timestamp.now() + within_range = within_range or ShardRange('entire/namespace', timestamp) shard_ranges = ShardRangeList(shard_ranges) # note: find_paths results do not include shrinking ranges paths = find_paths(shard_ranges) @@ -149,7 +153,8 @@ def find_paths_with_gaps(shard_ranges): timestamp, lower=start_path.upper, upper=end_path.lower) - paths_with_gaps.append((start_path, gap_range, end_path)) + if gap_range.overlaps(within_range): + paths_with_gaps.append((start_path, gap_range, end_path)) return paths_with_gaps @@ -497,6 +502,27 @@ def rank_paths(paths, shard_range_to_span): return paths +def combine_shard_ranges(new_shard_ranges, existing_shard_ranges): + """ + Combines new and existing shard ranges based on most recent state. + + :param new_shard_ranges: a list of ShardRange instances. + :param existing_shard_ranges: a list of ShardRange instances. + :return: a list of ShardRange instances. + """ + new_shard_ranges = [dict(sr) for sr in new_shard_ranges] + existing_shard_ranges = [dict(sr) for sr in existing_shard_ranges] + to_add, to_delete = sift_shard_ranges( + new_shard_ranges, + dict((sr['name'], sr) for sr in existing_shard_ranges)) + result = [ShardRange.from_dict(existing) + for existing in existing_shard_ranges + if existing['name'] not in to_delete] + result.extend([ShardRange.from_dict(sr) for sr in to_add]) + return sorted([sr for sr in result if not sr.deleted], + key=ShardRange.sort_key) + + class CleavingContext(object): """ Encapsulates metadata associated with the process of cleaving a retiring @@ -916,9 +942,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): if db_state not in (UNSHARDED, SHARDING, SHARDED): return own_shard_range = broker.get_own_shard_range() - if own_shard_range.state not in ( - ShardRange.SHARDING, ShardRange.SHARDED, - ShardRange.SHRINKING, ShardRange.SHRUNK): + if own_shard_range.state not in ShardRange.CLEAVING_STATES: return if db_state == SHARDED: @@ -1159,7 +1183,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): warnings = [] own_shard_range = broker.get_own_shard_range() - if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED): + if own_shard_range.state in ShardRange.SHARDING_STATES: shard_ranges = [sr for sr in broker.get_shard_ranges() if sr.state != ShardRange.SHRINKING] paths_with_gaps = find_paths_with_gaps(shard_ranges) @@ -1245,9 +1269,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): own_shard_range = broker.get_own_shard_range() if (orig_own_shard_range != own_shard_range or orig_own_shard_range.state != own_shard_range.state): - self.logger.info( - 'Updated own shard range from %s to %s', - orig_own_shard_range, own_shard_range) + self.logger.info('Updated own shard range from %s to %s', + orig_own_shard_range, own_shard_range) elif shard_range.is_child_of(own_shard_range): children_shard_ranges.append(shard_range) else: @@ -1262,19 +1285,70 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): len(children_shard_ranges)) broker.merge_shard_ranges(children_shard_ranges) - if (other_shard_ranges and - own_shard_range.state in ShardRange.SHRINKING_STATES): - # If own_shard_range state is shrinking, save off *all* shards - # returned because these may contain shards into which this - # shard is to shrink itself; shrinking is the only case when we - # want to learn about *other* shard ranges from the root. - # We need to include shrunk state too, because one replica of a - # shard may already have moved the own_shard_range state to - # shrunk while another replica may still be in the process of - # shrinking. - self.logger.debug('Updating %s other shard range(s) from root', - len(other_shard_ranges)) - broker.merge_shard_ranges(other_shard_ranges) + if (other_shard_ranges + and own_shard_range.state in ShardRange.CLEAVING_STATES + and not broker.is_sharded()): + # Other shard ranges returned from the root may need to be merged + # for the purposes of sharding or shrinking this shard: + # + # Shrinking states: If the up-to-date state is shrinking, the + # shards fetched from root may contain shards into which this shard + # is to shrink itself. Shrinking is initiated by modifying multiple + # neighboring shard range states *in the root*, rather than + # modifying a shard directly. We therefore need to learn about + # *other* neighboring shard ranges from the root, possibly + # including the root itself. We need to include shrunk state too, + # because one replica of a shard may already have moved the + # own_shard_range state to shrunk while another replica may still + # be in the process of shrinking. + # + # Sharding states: Normally a shard will shard to its own children. + # However, in some circumstances a shard may need to shard to other + # non-children sub-shards. For example, a shard range repair may + # cause a child sub-shard to be deleted and its namespace covered + # by another 'acceptor' shard. + # + # Therefore, if the up-to-date own_shard_range state indicates that + # sharding or shrinking is in progress, then other shard ranges + # will be merged, with the following caveats: we never expect a + # shard to shard to any ancestor shard range including the root, + # but containers might ultimately *shrink* to root; we never want + # to cleave to a container that is itself sharding or shrinking; + # the merged shard ranges should not result in gaps or overlaps in + # the namespace of this shard. + # + # Note: the search for ancestors is guaranteed to find the parent + # and root *if they are present*, but if any ancestor is missing + # then there is a chance that older generations in the + # other_shard_ranges will not be filtered and could be merged. That + # is only a problem if they are somehow still in ACTIVE state, and + # no overlap is detected, so the ancestor is merged. + ancestor_names = [ + sr.name for sr in own_shard_range.find_ancestors(shard_ranges)] + filtered_other_shard_ranges = [ + sr for sr in other_shard_ranges + if (sr.name not in ancestor_names + and (sr.state not in ShardRange.CLEAVING_STATES + or sr.deleted)) + ] + if own_shard_range.state in ShardRange.SHRINKING_STATES: + root_shard_range = own_shard_range.find_root( + other_shard_ranges) + if (root_shard_range and + root_shard_range.state == ShardRange.ACTIVE): + filtered_other_shard_ranges.append(root_shard_range) + existing_shard_ranges = broker.get_shard_ranges() + combined_shard_ranges = combine_shard_ranges( + filtered_other_shard_ranges, existing_shard_ranges) + overlaps = find_overlapping_ranges(combined_shard_ranges) + paths_with_gaps = find_paths_with_gaps( + combined_shard_ranges, own_shard_range) + if not (overlaps or paths_with_gaps): + # only merge if shard ranges appear to be *good* + self.logger.debug( + 'Updating %s other shard range(s) from root', + len(filtered_other_shard_ranges)) + broker.merge_shard_ranges(filtered_other_shard_ranges) return own_shard_range, own_shard_range_from_root @@ -2168,10 +2242,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator): broker, shard_ranges=[broker.get_own_shard_range()]) own_shard_range = broker.get_own_shard_range() - if own_shard_range.state in (ShardRange.SHARDING, - ShardRange.SHRINKING, - ShardRange.SHARDED, - ShardRange.SHRUNK): + if own_shard_range.state in ShardRange.CLEAVING_STATES: if broker.get_shard_ranges(): # container has been given shard ranges rather than # found them e.g. via replication or a shrink event, diff --git a/test/probe/test_sharder.py b/test/probe/test_sharder.py index 02f66778b0..4382d83145 100644 --- a/test/probe/test_sharder.py +++ b/test/probe/test_sharder.py @@ -1801,13 +1801,12 @@ class TestContainerSharding(BaseAutoContainerSharding): donor = orig_shard_ranges[0] shard_nodes_data = self.direct_get_container_shard_ranges( donor.account, donor.container) - # the donor's shard range will have the acceptor's projected stats; - # donor also has copy of root shard range that will be ignored; - # note: expected_shards does not include the sharded root range + # donor has the acceptor shard range but not the root shard range + # because the root is still in ACTIVE state; + # the donor's shard range will have the acceptor's projected stats obj_count, bytes_used = check_shard_nodes_data( shard_nodes_data, expected_state='sharded', expected_shards=1, - exp_obj_count=len(second_shard_objects) + 1, - exp_sharded_root_range=True) + exp_obj_count=len(second_shard_objects) + 1) # but the donor is empty and so reports zero stats self.assertEqual(0, obj_count) self.assertEqual(0, bytes_used) diff --git a/test/unit/cli/test_manage_shard_ranges.py b/test/unit/cli/test_manage_shard_ranges.py index a71187d4fb..4173c04bcd 100644 --- a/test/unit/cli/test_manage_shard_ranges.py +++ b/test/unit/cli/test_manage_shard_ranges.py @@ -24,7 +24,7 @@ from tempfile import mkdtemp import six from six.moves import cStringIO as StringIO -from swift.cli.manage_shard_ranges import main, combine_shard_ranges +from swift.cli.manage_shard_ranges import main from swift.common import utils from swift.common.utils import Timestamp, ShardRange from swift.container.backend import ContainerBroker @@ -2858,46 +2858,3 @@ class TestManageShardRanges(unittest.TestCase): self.assertIn( "argument --yes/-y: not allowed with argument --dry-run/-n", err_lines[-2], err_lines) - - def test_combine_shard_ranges(self): - ts_iter = make_timestamp_iter() - this = ShardRange('a/o', next(ts_iter).internal) - that = ShardRange('a/o', next(ts_iter).internal) - actual = combine_shard_ranges([dict(this)], [dict(that)]) - self.assertEqual([dict(that)], [dict(sr) for sr in actual]) - actual = combine_shard_ranges([dict(that)], [dict(this)]) - self.assertEqual([dict(that)], [dict(sr) for sr in actual]) - - ts = next(ts_iter).internal - this = ShardRange('a/o', ts, state=ShardRange.ACTIVE, - state_timestamp=next(ts_iter)) - that = ShardRange('a/o', ts, state=ShardRange.CREATED, - state_timestamp=next(ts_iter)) - actual = combine_shard_ranges([dict(this)], [dict(that)]) - self.assertEqual([dict(that)], [dict(sr) for sr in actual]) - actual = combine_shard_ranges([dict(that)], [dict(this)]) - self.assertEqual([dict(that)], [dict(sr) for sr in actual]) - - that.update_meta(1, 2, meta_timestamp=next(ts_iter)) - this.update_meta(3, 4, meta_timestamp=next(ts_iter)) - expected = that.copy(object_count=this.object_count, - bytes_used=this.bytes_used, - meta_timestamp=this.meta_timestamp) - actual = combine_shard_ranges([dict(this)], [dict(that)]) - self.assertEqual([dict(expected)], [dict(sr) for sr in actual]) - actual = combine_shard_ranges([dict(that)], [dict(this)]) - self.assertEqual([dict(expected)], [dict(sr) for sr in actual]) - - this = ShardRange('a/o', next(ts_iter).internal) - that = ShardRange('a/o', next(ts_iter).internal, deleted=True) - actual = combine_shard_ranges([dict(this)], [dict(that)]) - self.assertFalse(actual, [dict(sr) for sr in actual]) - actual = combine_shard_ranges([dict(that)], [dict(this)]) - self.assertFalse(actual, [dict(sr) for sr in actual]) - - this = ShardRange('a/o', next(ts_iter).internal, deleted=True) - that = ShardRange('a/o', next(ts_iter).internal) - actual = combine_shard_ranges([dict(this)], [dict(that)]) - self.assertEqual([dict(that)], [dict(sr) for sr in actual]) - actual = combine_shard_ranges([dict(that)], [dict(this)]) - self.assertEqual([dict(that)], [dict(sr) for sr in actual]) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 46a431ec6d..02b32f8330 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -8061,6 +8061,19 @@ class TestShardRange(unittest.TestCase): def setUp(self): self.ts_iter = make_timestamp_iter() + def test_constants(self): + self.assertEqual({utils.ShardRange.SHARDING, + utils.ShardRange.SHARDED, + utils.ShardRange.SHRINKING, + utils.ShardRange.SHRUNK}, + set(utils.ShardRange.CLEAVING_STATES)) + self.assertEqual({utils.ShardRange.SHARDING, + utils.ShardRange.SHARDED}, + set(utils.ShardRange.SHARDING_STATES)) + self.assertEqual({utils.ShardRange.SHRINKING, + utils.ShardRange.SHRUNK}, + set(utils.ShardRange.SHRINKING_STATES)) + def test_min_max_bounds(self): with self.assertRaises(TypeError): utils.ShardRangeOuterBound() @@ -9099,6 +9112,120 @@ class TestShardRange(unittest.TestCase): self.assertTrue(a1_r1_gp1.is_child_of(a2_r1)) self.assertTrue(a2_r1_gp1.is_child_of(a1_r1)) + def test_find_root(self): + # account 1 + ts = next(self.ts_iter) + a1_r1 = utils.ShardRange('a1/r1', ts) + ts = next(self.ts_iter) + a1_r1_gp1 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', 'r1', ts, 1), ts, '', 'l') + ts = next(self.ts_iter) + a1_r1_gp1_p1 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', a1_r1_gp1.container, ts, 1), ts, 'a', 'k') + ts = next(self.ts_iter) + a1_r1_gp1_p1_c1 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', a1_r1_gp1_p1.container, ts, 1), ts, 'a', 'j') + ts = next(self.ts_iter) + a1_r1_gp1_p2 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', a1_r1_gp1.container, ts, 2), ts, 'k', 'l') + ts = next(self.ts_iter) + a1_r1_gp2 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', 'r1', ts, 2), ts, 'l', '') # different index + + # full ancestry plus some others + all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1, + a1_r1_gp1_p2, a1_r1_gp2] + random.shuffle(all_shard_ranges) + self.assertIsNone(a1_r1.find_root(all_shard_ranges)) + self.assertEqual(a1_r1, a1_r1_gp1.find_root(all_shard_ranges)) + self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges)) + self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges)) + + # missing a1_r1_gp1_p1 + all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1_c1, + a1_r1_gp1_p2, a1_r1_gp2] + random.shuffle(all_shard_ranges) + self.assertIsNone(a1_r1.find_root(all_shard_ranges)) + self.assertEqual(a1_r1, a1_r1_gp1.find_root(all_shard_ranges)) + self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges)) + self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges)) + + # empty list + self.assertIsNone(a1_r1_gp1_p1_c1.find_root([])) + + # double entry + all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1, a1_r1_gp1] + random.shuffle(all_shard_ranges) + self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges)) + self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges)) + + def test_find_ancestors(self): + # account 1 + ts = next(self.ts_iter) + a1_r1 = utils.ShardRange('a1/r1', ts) + ts = next(self.ts_iter) + a1_r1_gp1 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', 'r1', ts, 1), ts, '', 'l') + ts = next(self.ts_iter) + a1_r1_gp1_p1 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', a1_r1_gp1.container, ts, 1), ts, 'a', 'k') + ts = next(self.ts_iter) + a1_r1_gp1_p1_c1 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', a1_r1_gp1_p1.container, ts, 1), ts, 'a', 'j') + ts = next(self.ts_iter) + a1_r1_gp1_p2 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', a1_r1_gp1.container, ts, 2), ts, 'k', 'l') + ts = next(self.ts_iter) + a1_r1_gp2 = utils.ShardRange(utils.ShardRange.make_path( + '.shards_a1', 'r1', 'r1', ts, 2), ts, 'l', '') # different index + + # full ancestry plus some others + all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1, + a1_r1_gp1_p2, a1_r1_gp2] + random.shuffle(all_shard_ranges) + self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1_gp1, a1_r1], + a1_r1_gp1_p1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1_gp1_p1, a1_r1_gp1, a1_r1], + a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges)) + + # missing a1_r1_gp1_p1 + all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1_c1, + a1_r1_gp1_p2, a1_r1_gp2] + random.shuffle(all_shard_ranges) + self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1_gp1, a1_r1], + a1_r1_gp1_p1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1], + a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges)) + + # missing a1_r1_gp1 + all_shard_ranges = [a1_r1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1, + a1_r1_gp1_p2, a1_r1_gp2] + random.shuffle(all_shard_ranges) + self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1], + a1_r1_gp1_p1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1_gp1_p1, a1_r1], + a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges)) + + # empty list + self.assertEqual([], a1_r1_gp1_p1_c1.find_ancestors([])) + # double entry + all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1, a1_r1_gp1] + random.shuffle(all_shard_ranges) + self.assertEqual([a1_r1_gp1, a1_r1], + a1_r1_gp1_p1.find_ancestors(all_shard_ranges)) + self.assertEqual([a1_r1], + a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges)) + all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1_p1, a1_r1_gp1_p1] + random.shuffle(all_shard_ranges) + self.assertEqual([a1_r1_gp1_p1, a1_r1], + a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges)) + def test_expand(self): bounds = (('', 'd'), ('d', 'k'), ('k', 't'), ('t', '')) donors = [ diff --git a/test/unit/container/test_sharder.py b/test/unit/container/test_sharder.py index 42de0a46df..cd97f00cd8 100644 --- a/test/unit/container/test_sharder.py +++ b/test/unit/container/test_sharder.py @@ -43,7 +43,7 @@ from swift.container.sharder import ContainerSharder, sharding_enabled, \ find_shrinking_candidates, process_compactible_shard_sequences, \ find_compactible_shard_sequences, is_shrinking_candidate, \ is_sharding_candidate, find_paths, rank_paths, ContainerSharderConf, \ - find_paths_with_gaps + find_paths_with_gaps, combine_shard_ranges from swift.common.utils import ShardRange, Timestamp, hash_path, \ encode_timestamps, parse_db_filename, quorum_size, Everything, md5, \ ShardName @@ -5840,7 +5840,7 @@ class TestSharder(BaseTestSharder): def _do_test_audit_shard_container_merge_other_ranges(self, *args): # verify that shard only merges other ranges from root when it is - # shrinking or shrunk + # cleaving shard_bounds = ( ('a', 'p'), ('k', 't'), ('p', 'u')) shard_states = ( @@ -5855,7 +5855,7 @@ class TestSharder(BaseTestSharder): broker.set_sharding_sysmeta(*args) shard_ranges[1].name = broker.path - # make own shard range match shard_ranges[1] + # make shard's own shard range match shard_ranges[1] own_sr = shard_ranges[1] expected_stats = {'attempted': 1, 'success': 1, 'failure': 0} self.assertTrue(own_sr.update_state(own_state, @@ -5903,7 +5903,7 @@ class TestSharder(BaseTestSharder): self.assertEqual(root_state, own_shard_range.state) self.assertEqual(root_ts, own_shard_range.state_timestamp) updated_ranges = broker.get_shard_ranges(include_own=True) - if root_state in (ShardRange.SHRINKING, ShardRange.SHRUNK): + if root_state in ShardRange.CLEAVING_STATES: # check other shard ranges from root are merged self.assertEqual(shard_ranges, updated_ranges) else: @@ -5925,7 +5925,7 @@ class TestSharder(BaseTestSharder): self.assertEqual(own_state, own_shard_range.state) self.assertEqual(own_ts, own_shard_range.state_timestamp) updated_ranges = broker.get_shard_ranges(include_own=True) - if own_state in (ShardRange.SHRINKING, ShardRange.SHRUNK): + if own_state in ShardRange.CLEAVING_STATES: # check other shard ranges from root are merged self.assertEqual(shard_ranges, updated_ranges) else: @@ -5940,7 +5940,7 @@ class TestSharder(BaseTestSharder): 'a/c') def _assert_merge_into_shard(self, own_shard_range, shard_ranges, - root_shard_ranges, expected, *args): + root_shard_ranges, expected, *args, **kwargs): # create a shard broker, initialise with shard_ranges, run audit on it # supplying given root_shard_ranges and verify that the broker ends up # with expected shard ranges. @@ -5948,6 +5948,13 @@ class TestSharder(BaseTestSharder): container=own_shard_range.container) broker.set_sharding_sysmeta(*args) broker.merge_shard_ranges([own_shard_range] + shard_ranges) + db_state = kwargs.get('db_state', UNSHARDED) + if db_state == SHARDING: + broker.set_sharding_state() + if db_state == SHARDED: + broker.set_sharding_state() + broker.set_sharded_state() + self.assertEqual(db_state, broker.get_db_state()) self.assertFalse(broker.is_root_container()) sharder, mock_swift = self.call_audit_container( @@ -5998,26 +6005,26 @@ class TestSharder(BaseTestSharder): self.assertFalse(sharder.logger.get_lines_for_level('error')) for own_state in ShardRange.STATES: - if own_state in ShardRange.SHRINKING_STATES: - # shrinking states are covered by other tests + if own_state in ShardRange.CLEAVING_STATES: + # cleaving states are covered by other tests continue for acceptor_state in ShardRange.STATES: for root_state in ShardRange.STATES: do_test(own_state, acceptor_state, root_state) - def test_audit_old_style_shard_root_ranges_not_merged_not_shrinking(self): + def test_audit_old_style_shard_root_ranges_not_merged_not_cleaving(self): # verify that other shard ranges from root are NOT merged into shard - # when it is NOT in a shrinking state + # when it is NOT in a cleaving state self._do_test_audit_shard_root_ranges_not_merged('Root', 'a/c') - def test_audit_shard_root_ranges_not_merged_not_shrinking(self): + def test_audit_shard_root_ranges_not_merged_not_cleaving(self): # verify that other shard ranges from root are NOT merged into shard - # when it is NOT in a shrinking state + # when it is NOT in a cleaving state self._do_test_audit_shard_root_ranges_not_merged('Quoted-Root', 'a/c') def test_audit_shard_root_ranges_with_own_merged_while_shrinking(self): - # Verify that shrinking shard will merge root and other ranges, - # including root range. + # Verify that shrinking shard will merge other ranges, but not + # in-ACTIVE root range. # Make root and other ranges that fully contain the shard namespace... root_own_sr = ShardRange('a/c', next(self.ts_iter)) acceptor = ShardRange( @@ -6034,7 +6041,7 @@ class TestSharder(BaseTestSharder): own_sr = ShardRange( str(ShardName.create('.shards_a', 'c', 'c', ts, 0)), ts, lower='a', upper='b', state=own_state, state_timestamp=ts) - expected = [acceptor_from_root, root_from_root] + expected = [acceptor_from_root] with annotate_failure('with states %s %s %s' % (own_state, acceptor_state, root_state)): sharder = self._assert_merge_into_shard( @@ -6047,12 +6054,21 @@ class TestSharder(BaseTestSharder): for own_state in ShardRange.SHRINKING_STATES: for acceptor_state in ShardRange.STATES: + if acceptor_state in ShardRange.CLEAVING_STATES: + # special case covered in other tests + continue for root_state in ShardRange.STATES: - do_test(own_state, acceptor_state, root_state) + if root_state == ShardRange.ACTIVE: + # special case: ACTIVE root *is* merged + continue + with annotate_failure( + 'with states %s %s %s' + % (own_state, acceptor_state, root_state)): + do_test(own_state, acceptor_state, root_state) def test_audit_shard_root_ranges_missing_own_merged_while_shrinking(self): - # Verify that shrinking shard will merge root and other ranges, - # including root range. + # Verify that shrinking shard will merge other ranges, but not + # in-ACTIVE root range, even when root does not have shard's own range. # Make root and other ranges that fully contain the shard namespace... root_own_sr = ShardRange('a/c', next(self.ts_iter)) acceptor = ShardRange( @@ -6069,7 +6085,7 @@ class TestSharder(BaseTestSharder): own_sr = ShardRange( str(ShardName.create('.shards_a', 'c', 'c', ts, 0)), ts, lower='a', upper='b', state=own_state, state_timestamp=ts) - expected = [acceptor_from_root, root_from_root] + expected = [acceptor_from_root] with annotate_failure('with states %s %s %s' % (own_state, acceptor_state, root_state)): sharder = self._assert_merge_into_shard( @@ -6085,8 +6101,86 @@ class TestSharder(BaseTestSharder): for own_state in ShardRange.SHRINKING_STATES: for acceptor_state in ShardRange.STATES: + if acceptor_state in ShardRange.CLEAVING_STATES: + # special case covered in other tests + continue for root_state in ShardRange.STATES: - do_test(own_state, acceptor_state, root_state) + if root_state == ShardRange.ACTIVE: + # special case: ACTIVE root *is* merged + continue + with annotate_failure( + 'with states %s %s %s' + % (own_state, acceptor_state, root_state)): + do_test(own_state, acceptor_state, root_state) + + def test_audit_shard_root_range_not_merged_while_shrinking(self): + # Verify that shrinking shard will not merge an in-active root range + def do_test(own_state, root_state): + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + expected = [] + sharder = self._assert_merge_into_shard( + own_sr, [], [own_sr, root_own_sr], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.SHRINKING_STATES: + for root_state in ShardRange.STATES: + if root_state == ShardRange.ACTIVE: + continue # special case tested below + with annotate_failure((own_state, root_state)): + do_test(own_state, root_state) + + def test_audit_shard_root_range_overlap_not_merged_while_shrinking(self): + # Verify that shrinking shard will not merge an active root range that + # overlaps with an exosting sub-shard + def do_test(own_state): + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.ACTIVE) + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + ts = next(self.ts_iter) + sub_shard = ShardRange( + str(ShardName.create( + '.shards_a', 'c', own_sr.container, ts, 0)), + ts, lower='a', upper='ab', state=ShardRange.ACTIVE) + expected = [sub_shard] + sharder = self._assert_merge_into_shard( + own_sr, [sub_shard], [own_sr, root_own_sr], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.SHRINKING_STATES: + with annotate_failure(own_state): + do_test(own_state) + + def test_audit_shard_active_root_range_merged_while_shrinking(self): + # Verify that shrinking shard will merge an active root range + def do_test(own_state): + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.ACTIVE) + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + expected = [root_own_sr] + sharder = self._assert_merge_into_shard( + own_sr, [], [own_sr, root_own_sr], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.SHRINKING_STATES: + with annotate_failure(own_state): + do_test(own_state) def test_audit_shard_root_ranges_fetch_fails_while_shrinking(self): # check audit copes with failed response while shard is shrinking @@ -6106,6 +6200,425 @@ class TestSharder(BaseTestSharder): warning_lines[1]) self.assertFalse(sharder.logger.get_lines_for_level('error')) + def test_audit_shard_root_ranges_merge_while_unsharded(self): + # Verify that unsharded shard with no existing shard ranges will merge + # other ranges, but not root range. + root_own_sr = ShardRange('a/c', next(self.ts_iter)) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + + def do_test(own_state, acceptor_state, root_state): + acceptor_from_root = acceptor.copy( + timestamp=next(self.ts_iter), state=acceptor_state) + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + root_from_root = root_own_sr.copy( + timestamp=next(self.ts_iter), state=root_state) + expected = [acceptor_from_root] + sharder = self._assert_merge_into_shard( + own_sr, [], + [own_sr, acceptor_from_root, root_from_root], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.SHARDING_STATES: + for acceptor_state in ShardRange.STATES: + if acceptor_state in ShardRange.CLEAVING_STATES: + # special case covered in other tests + continue + for root_state in ShardRange.STATES: + with annotate_failure( + 'with states %s %s %s' + % (own_state, acceptor_state, root_state)): + do_test(own_state, acceptor_state, root_state) + + def test_audit_shard_root_ranges_merge_while_sharding(self): + # Verify that sharding shard with no existing shard ranges will merge + # other ranges, but not root range. + root_own_sr = ShardRange('a/c', next(self.ts_iter)) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + + def do_test(own_state, acceptor_state, root_state): + acceptor_from_root = acceptor.copy( + timestamp=next(self.ts_iter), state=acceptor_state) + ts = next(self.ts_iter) + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', ts, 0)), + ts, 'a', 'b', epoch=ts, state=own_state) + root_from_root = root_own_sr.copy( + timestamp=next(self.ts_iter), state=root_state) + expected = [acceptor_from_root] + sharder = self._assert_merge_into_shard( + own_sr, [], + [own_sr, acceptor_from_root, root_from_root], + expected, 'Quoted-Root', 'a/c', db_state=SHARDING) + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.SHARDING_STATES: + for acceptor_state in ShardRange.STATES: + if acceptor_state in ShardRange.CLEAVING_STATES: + # special case covered in other tests + continue + for root_state in ShardRange.STATES: + with annotate_failure( + 'with states %s %s %s' + % (own_state, acceptor_state, root_state)): + do_test(own_state, acceptor_state, root_state) + + def test_audit_shard_root_ranges_not_merged_once_sharded(self): + # Verify that sharded shard will not merge other ranges from root + root_own_sr = ShardRange('a/c', next(self.ts_iter)) + # the acceptor complements the single existing sub-shard... + other_sub_shard = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'ab', 'c', state=ShardRange.ACTIVE) + + def do_test(own_state, other_sub_shard_state, root_state): + sub_shard_from_root = other_sub_shard.copy( + timestamp=next(self.ts_iter), state=other_sub_shard_state) + ts = next(self.ts_iter) + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', ts, 0)), + ts, 'a', 'b', epoch=ts, state=own_state) + ts = next(self.ts_iter) + sub_shard = ShardRange( + str(ShardName.create( + '.shards_a', 'c', own_sr.container, ts, 0)), + ts, lower='a', upper='ab', state=ShardRange.ACTIVE) + root_from_root = root_own_sr.copy( + timestamp=next(self.ts_iter), state=root_state) + expected = [sub_shard] + sharder = self._assert_merge_into_shard( + own_sr, [sub_shard], + [own_sr, sub_shard_from_root, root_from_root], + expected, 'Quoted-Root', 'a/c', db_state=SHARDED) + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in (ShardRange.SHARDED, ShardRange.SHRUNK): + for other_sub_shard_state in ShardRange.STATES: + for root_state in ShardRange.STATES: + with annotate_failure( + 'with states %s %s %s' + % (own_state, other_sub_shard_state, root_state)): + do_test(own_state, other_sub_shard_state, root_state) + + def test_audit_shard_root_ranges_replace_existing_while_cleaving(self): + # Verify that sharding shard with stale existing sub-shard ranges will + # merge other ranges, but not root range. + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + ts = next(self.ts_iter) + acceptor_sub_shards = [ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', acceptor.container, ts, i)), + ts, lower, upper, state=ShardRange.ACTIVE) + for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))] + + # shard has incomplete existing shard ranges, ranges from root delete + # existing sub-shard and replace with other acceptor sub-shards + def do_test(own_state): + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + ts = next(self.ts_iter) + sub_shard = ShardRange( + str(ShardName.create( + '.shards_a', 'c', own_sr.container, ts, 0)), + ts, lower='a', upper='ab', state=ShardRange.ACTIVE) + deleted_sub_shard = sub_shard.copy( + timestamp=next(self.ts_iter), state=ShardRange.SHARDED, + deleted=1) + expected = acceptor_sub_shards + sharder = self._assert_merge_into_shard( + own_sr, [sub_shard], + [root_own_sr, own_sr, deleted_sub_shard] + acceptor_sub_shards, + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.CLEAVING_STATES: + with annotate_failure(own_state): + do_test(own_state) + + def test_audit_shard_root_ranges_supplement_deleted_while_cleaving(self): + # Verify that sharding shard with deleted existing sub-shard ranges + # will merge other ranges while sharding, but not root range. + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + ts = next(self.ts_iter) + acceptor_sub_shards = [ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', acceptor.container, ts, i)), + ts, lower, upper, state=ShardRange.ACTIVE) + for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))] + + # shard already has deleted existing shard ranges + expected = acceptor_sub_shards + + def do_test(own_state): + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + ts = next(self.ts_iter) + deleted_sub_shards = [ShardRange( + str(ShardName.create( + '.shards_a', 'c', own_sr.container, ts, i)), + ts, lower, upper, state=ShardRange.SHARDED, deleted=1) + for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'b'))] + sharder = self._assert_merge_into_shard( + own_sr, deleted_sub_shards, + [own_sr, root_own_sr] + acceptor_sub_shards, + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.CLEAVING_STATES: + with annotate_failure(own_state): + do_test(own_state) + + def test_audit_shard_root_ranges_supplement_existing_while_cleaving(self): + # Verify that sharding shard with incomplete existing sub-shard ranges + # will merge other ranges that fill the gap, but not root range. + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + ts = next(self.ts_iter) + acceptor_sub_shards = [ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', acceptor.container, ts, i)), + ts, lower, upper, state=ShardRange.ACTIVE) + for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))] + + # shard has incomplete existing shard ranges and range from root fills + # the gap + def do_test(own_state): + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + ts = next(self.ts_iter) + sub_shard = ShardRange( + str(ShardName.create( + '.shards_a', 'c', own_sr.container, ts, 0)), + ts, lower='a', upper='ab', state=ShardRange.ACTIVE) + expected = [sub_shard] + acceptor_sub_shards[1:] + sharder = self._assert_merge_into_shard( + own_sr, [sub_shard], + [own_sr, root_own_sr] + acceptor_sub_shards[1:], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.CLEAVING_STATES: + with annotate_failure(own_state): + do_test(own_state) + + def test_audit_shard_root_ranges_cleaving_not_merged_while_cleaving(self): + # Verify that sharding shard will not merge other ranges that are in a + # cleaving state. + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + + def do_test(own_state, acceptor_state, root_state): + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + root_from_root = root_own_sr.copy( + timestamp=next(self.ts_iter), state=root_state) + acceptor_from_root = acceptor.copy( + timestamp=next(self.ts_iter), state=acceptor_state) + + if (own_state in ShardRange.SHRINKING_STATES and + root_state == ShardRange.ACTIVE): + # special case: when shrinking, ACTIVE root shard *is* merged + expected = [root_from_root] + else: + expected = [] + + sharder = self._assert_merge_into_shard( + own_sr, [], + [own_sr, acceptor_from_root, root_from_root], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + # ranges from root that are in a cleaving state are not merged... + for own_state in ShardRange.CLEAVING_STATES: + for acceptor_state in ShardRange.CLEAVING_STATES: + for root_state in ShardRange.STATES: + with annotate_failure( + 'with states %s %s %s' + % (own_state, acceptor_state, root_state)): + do_test(own_state, acceptor_state, root_state) + + def test_audit_shard_root_ranges_overlap_not_merged_while_cleaving_1(self): + # Verify that sharding/shrinking shard will not merge other ranges that + # would create an overlap; shard has complete existing shard ranges, + # newer range from root ignored + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + + def do_test(own_state): + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + ts = next(self.ts_iter) + sub_shards = [ShardRange( + str(ShardName.create( + '.shards_a', 'c', own_sr.container, ts, i)), + ts, lower, upper, state=ShardRange.ACTIVE) + for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'b'))] + acceptor_from_root = acceptor.copy(timestamp=next(self.ts_iter)) + expected = sub_shards + sharder = self._assert_merge_into_shard( + own_sr, sub_shards, + [own_sr, acceptor_from_root, root_own_sr], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.CLEAVING_STATES: + with annotate_failure(own_state): + do_test(own_state) + + def test_audit_shard_root_ranges_overlap_not_merged_while_cleaving_2(self): + # Verify that sharding/shrinking shard will not merge other ranges that + # would create an overlap; shard has incomplete existing shard ranges + # but ranges from root overlaps + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + ts = next(self.ts_iter) + acceptor_sub_shards = [ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', acceptor.container, ts, i)), + ts, lower, upper, state=ShardRange.ACTIVE) + for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))] + + def do_test(own_state): + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + ts = next(self.ts_iter) + sub_shard = ShardRange( + str(ShardName.create( + '.shards_a', 'c', own_sr.container, ts, 0)), + ts, lower='a', upper='abc', state=ShardRange.ACTIVE) + expected = [sub_shard] + sharder = self._assert_merge_into_shard( + own_sr, [sub_shard], + acceptor_sub_shards[1:] + [own_sr, root_own_sr], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.CLEAVING_STATES: + with annotate_failure(own_state): + do_test(own_state) + + def test_audit_shard_root_ranges_with_gap_not_merged_while_cleaving(self): + # Verify that sharding/shrinking shard will not merge other ranges that + # would leave a gap. + root_own_sr = ShardRange('a/c', next(self.ts_iter), + state=ShardRange.SHARDED) + acceptor = ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', 'c', next(self.ts_iter), 1)), + next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE) + ts = next(self.ts_iter) + acceptor_sub_shards = [ShardRange( + str(ShardRange.make_path( + '.shards_a', 'c', acceptor.container, ts, i)), + ts, lower, upper, state=ShardRange.ACTIVE) + for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))] + + def do_test(own_state): + own_sr = ShardRange( + str(ShardName.create( + '.shards_a', 'c', 'c', next(self.ts_iter), 0)), + next(self.ts_iter), 'a', 'b', state=own_state) + # root ranges have gaps w.r.t. the shard namespace + existing = expected = [] + sharder = self._assert_merge_into_shard( + own_sr, existing, + acceptor_sub_shards[:1] + [own_sr, root_own_sr], + expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + + for own_state in ShardRange.CLEAVING_STATES: + with annotate_failure(own_state): + do_test(own_state) + + def test_audit_shard_container_ancestors_not_merged_while_sharding(self): + # Verify that sharding shard will not merge parent and root shard + # ranges even when the sharding shard has no other ranges + root_sr = ShardRange('a/root', next(self.ts_iter), + state=ShardRange.SHARDED) + grandparent_path = ShardRange.make_path( + '.shards_a', 'root', root_sr.container, next(self.ts_iter), 2) + grandparent_sr = ShardRange(grandparent_path, next(self.ts_iter), + '', 'd', state=ShardRange.ACTIVE) + self.assertTrue(grandparent_sr.is_child_of(root_sr)) + parent_path = ShardRange.make_path( + '.shards_a', 'root', grandparent_sr.container, next(self.ts_iter), + 2) + parent_sr = ShardRange(parent_path, next(self.ts_iter), '', 'd', + state=ShardRange.ACTIVE) + self.assertTrue(parent_sr.is_child_of(grandparent_sr)) + child_path = ShardRange.make_path( + '.shards_a', 'root', parent_sr.container, next(self.ts_iter), 2) + child_own_sr = ShardRange(child_path, next(self.ts_iter), 'a', 'b', + state=ShardRange.SHARDING) + self.assertTrue(child_own_sr.is_child_of(parent_sr)) + + ranges_from_root = [grandparent_sr, parent_sr, root_sr, child_own_sr] + expected = [] + sharder = self._assert_merge_into_shard( + child_own_sr, [], ranges_from_root, expected, 'Quoted-Root', 'a/c') + self.assertFalse(sharder.logger.get_lines_for_level('warning')) + self.assertFalse(sharder.logger.get_lines_for_level('error')) + def test_audit_shard_container_children_merged_while_sharding(self): # Verify that sharding shard will always merge children shard ranges def do_test(child_deleted, child_state): @@ -6157,9 +6670,11 @@ class TestSharder(BaseTestSharder): 'GET', '/v1/a/c', expected_headers, acceptable_statuses=(2,), params=params) + expected = child_srs + [parent_sr] + if child_deleted: + expected.append(other_sr) self._assert_shard_ranges_equal( - sorted(child_srs + [parent_sr], - key=ShardRange.sort_key), + sorted(expected, key=ShardRange.sort_key), sorted(broker.get_shard_ranges( include_own=True, include_deleted=True), key=ShardRange.sort_key)) @@ -8141,7 +8656,7 @@ class TestSharderFunctions(BaseTestSharder): bounds, ShardRange.ACTIVE, timestamp=next(self.ts_iter), object_count=1) paths_with_gaps = find_paths_with_gaps(ranges) - self.assertEqual(3, len(paths_with_gaps)) + self.assertEqual(3, len(paths_with_gaps), paths_with_gaps) self.assertEqual( [(ShardRange.MIN, ShardRange.MIN), (ShardRange.MIN, 'a'), @@ -8161,6 +8676,38 @@ class TestSharderFunctions(BaseTestSharder): [(r.lower, r.upper) for r in paths_with_gaps[2]] ) + range_of_interest = ShardRange('test/range', next(self.ts_iter)) + range_of_interest.lower = 'a' + paths_with_gaps = find_paths_with_gaps(ranges, range_of_interest) + self.assertEqual(2, len(paths_with_gaps), paths_with_gaps) + self.assertEqual( + [('k', 'p'), + ('p', 'q'), + ('q', 'y')], + [(r.lower, r.upper) for r in paths_with_gaps[0]] + ) + self.assertEqual( + [('q', 'y'), + ('y', ShardRange.MAX), + (ShardRange.MAX, ShardRange.MAX)], + [(r.lower, r.upper) for r in paths_with_gaps[1]] + ) + + range_of_interest.lower = 'b' + range_of_interest.upper = 'x' + paths_with_gaps = find_paths_with_gaps(ranges, range_of_interest) + self.assertEqual(1, len(paths_with_gaps), paths_with_gaps) + self.assertEqual( + [('k', 'p'), + ('p', 'q'), + ('q', 'y')], + [(r.lower, r.upper) for r in paths_with_gaps[0]] + ) + + range_of_interest.upper = 'c' + paths_with_gaps = find_paths_with_gaps(ranges, range_of_interest) + self.assertFalse(paths_with_gaps) + class TestContainerSharderConf(unittest.TestCase): def test_default(self): @@ -8337,3 +8884,46 @@ class TestContainerSharderConf(unittest.TestCase): assert_bad({'shard_container_threshold': 100, 'expansion_limit': 100}) assert_ok({'expansion_limit': 100000001}) + + def test_combine_shard_ranges(self): + ts_iter = make_timestamp_iter() + this = ShardRange('a/o', next(ts_iter).internal) + that = ShardRange('a/o', next(ts_iter).internal) + actual = combine_shard_ranges([dict(this)], [dict(that)]) + self.assertEqual([dict(that)], [dict(sr) for sr in actual]) + actual = combine_shard_ranges([dict(that)], [dict(this)]) + self.assertEqual([dict(that)], [dict(sr) for sr in actual]) + + ts = next(ts_iter).internal + this = ShardRange('a/o', ts, state=ShardRange.ACTIVE, + state_timestamp=next(ts_iter)) + that = ShardRange('a/o', ts, state=ShardRange.CREATED, + state_timestamp=next(ts_iter)) + actual = combine_shard_ranges([dict(this)], [dict(that)]) + self.assertEqual([dict(that)], [dict(sr) for sr in actual]) + actual = combine_shard_ranges([dict(that)], [dict(this)]) + self.assertEqual([dict(that)], [dict(sr) for sr in actual]) + + that.update_meta(1, 2, meta_timestamp=next(ts_iter)) + this.update_meta(3, 4, meta_timestamp=next(ts_iter)) + expected = that.copy(object_count=this.object_count, + bytes_used=this.bytes_used, + meta_timestamp=this.meta_timestamp) + actual = combine_shard_ranges([dict(this)], [dict(that)]) + self.assertEqual([dict(expected)], [dict(sr) for sr in actual]) + actual = combine_shard_ranges([dict(that)], [dict(this)]) + self.assertEqual([dict(expected)], [dict(sr) for sr in actual]) + + this = ShardRange('a/o', next(ts_iter).internal) + that = ShardRange('a/o', next(ts_iter).internal, deleted=True) + actual = combine_shard_ranges([dict(this)], [dict(that)]) + self.assertFalse(actual, [dict(sr) for sr in actual]) + actual = combine_shard_ranges([dict(that)], [dict(this)]) + self.assertFalse(actual, [dict(sr) for sr in actual]) + + this = ShardRange('a/o', next(ts_iter).internal, deleted=True) + that = ShardRange('a/o', next(ts_iter).internal) + actual = combine_shard_ranges([dict(this)], [dict(that)]) + self.assertEqual([dict(that)], [dict(sr) for sr in actual]) + actual = combine_shard_ranges([dict(that)], [dict(this)]) + self.assertEqual([dict(that)], [dict(sr) for sr in actual])