sharder: merge shard shard_ranges from root while sharding

We've seen shards become stuck while sharding because they had
incomplete or stale deleted shard ranges. The root container had more
complete and useful shard ranges into which objects could have been
cleaved, but the shard never merged the root's shard ranges.

While the sharder is auditing shard container DBs it would previously
only merge shard ranges fetched from root into the shard DB if the
shard was shrinking or the shard ranges were known to be children of
the shard. With this patch the sharder will now merge other shard
ranges from root during sharding as well as shrinking.

Shard ranges from root are only merged if they would not result in
overlaps or gaps in the set of shard ranges in the shard DB. Shard
ranges that are known to be ancestors of the shard are never merged,
except the root shard range which may be merged into a shrinking
shard. These checks were not previously applied when merging
shard ranges into a shrinking shard.

The two substantive changes with this patch are therefore:

  - shard ranges from root are now merged during sharding,
    subject to checks.
  - shard ranges from root are still merged during shrinking,
    but are now subjected to checks.

Change-Id: I066cfbd9062c43cd9638710882ae9bd85a5b4c37

swift/cli/manage_shard_ranges.py

@@ -168,13 +168,12 @@ from six.moves import input
 
 from swift.common.utils import Timestamp, get_logger, ShardRange, readconf, \
     ShardRangeList, non_negative_int, config_positive_int_value
-from swift.container.backend import ContainerBroker, UNSHARDED, \
-    sift_shard_ranges
+from swift.container.backend import ContainerBroker, UNSHARDED
 from swift.container.sharder import make_shard_ranges, sharding_enabled, \
     CleavingContext, process_compactible_shard_sequences, \
     find_compactible_shard_sequences, find_overlapping_ranges, \
     find_paths, rank_paths, finalize_shrinking, DEFAULT_SHARDER_CONF, \
-    ContainerSharderConf, find_paths_with_gaps
+    ContainerSharderConf, find_paths_with_gaps, combine_shard_ranges
 
 EXIT_SUCCESS = 0
 EXIT_ERROR = 1
@@ -428,27 +427,6 @@ def delete_shard_ranges(broker, args):
     return EXIT_SUCCESS
 
 
-def combine_shard_ranges(new_shard_ranges, existing_shard_ranges):
-    """
-    Combines new and existing shard ranges based on most recent state.
-
-    :param new_shard_ranges: a list of ShardRange instances.
-    :param existing_shard_ranges: a list of ShardRange instances.
-    :return: a list of ShardRange instances.
-    """
-    new_shard_ranges = [dict(sr) for sr in new_shard_ranges]
-    existing_shard_ranges = [dict(sr) for sr in existing_shard_ranges]
-    to_add, to_delete = sift_shard_ranges(
-        new_shard_ranges,
-        dict((sr['name'], sr) for sr in existing_shard_ranges))
-    result = [ShardRange.from_dict(existing)
-              for existing in existing_shard_ranges
-              if existing['name'] not in to_delete]
-    result.extend([ShardRange.from_dict(sr) for sr in to_add])
-    return sorted([sr for sr in result if not sr.deleted],
-                  key=ShardRange.sort_key)
-
-
 def merge_shard_ranges(broker, args):
     _check_own_shard_range(broker, args)
     shard_data = _load_and_validate_shard_data(args, require_index=False)

swift/common/utils.py

@@ -5330,6 +5330,8 @@ class ShardRange(object):
               SHRUNK: 'shrunk'}
     STATES_BY_NAME = dict((v, k) for k, v in STATES.items())
     SHRINKING_STATES = (SHRINKING, SHRUNK)
+    SHARDING_STATES = (SHARDING, SHARDED)
+    CLEAVING_STATES = SHRINKING_STATES + SHARDING_STATES
 
     @functools.total_ordering
     class MaxBound(ShardRangeOuterBound):
@@ -5434,6 +5436,76 @@ class ShardRange(object):
             == ShardName.hash_container_name(parent.container)
         )
 
+    def _find_root(self, parsed_name, shard_ranges):
+        for sr in shard_ranges:
+            if parsed_name.root_container == sr.container:
+                return sr
+        return None
+
+    def find_root(self, shard_ranges):
+        """
+        Find this shard range's root shard range in the given ``shard_ranges``.
+
+        :param shard_ranges: a list of instances of
+            :class:`~swift.common.utils.ShardRange`
+        :return: this shard range's root shard range if it is found in the
+            list, otherwise None.
+        """
+        try:
+            self_parsed_name = ShardName.parse(self.name)
+        except ValueError:
+            # not a shard
+            return None
+        return self._find_root(self_parsed_name, shard_ranges)
+
+    def find_ancestors(self, shard_ranges):
+        """
+        Find this shard range's ancestor ranges in the given ``shard_ranges``.
+
+        This method makes a best-effort attempt to identify this shard range's
+        parent shard range, the parent's parent, etc., up to and including the
+        root shard range. It is only possible to directly identify the parent
+        of a particular shard range, so the search is recursive; if any member
+        of the ancestry is not found then the search ends and older ancestors
+        that may be in the list are not identified. The root shard range,
+        however, will always be identified if it is present in the list.
+
+        For example, given a list that contains parent, grandparent,
+        great-great-grandparent and root shard ranges, but is missing the
+        great-grandparent shard range, only the parent, grand-parent and root
+        shard ranges will be identified.
+
+        :param shard_ranges: a list of instances of
+            :class:`~swift.common.utils.ShardRange`
+        :return: a list of instances of
+            :class:`~swift.common.utils.ShardRange` containing items in the
+            given ``shard_ranges`` that can be identified as ancestors of this
+            shard range. The list may not be complete if there are gaps in the
+            ancestry, but is guaranteed to contain at least the parent and
+            root shard ranges if they are present.
+        """
+        if not shard_ranges:
+            return []
+
+        try:
+            self_parsed_name = ShardName.parse(self.name)
+        except ValueError:
+            # not a shard
+            return []
+
+        ancestors = []
+        for sr in shard_ranges:
+            if self.is_child_of(sr):
+                ancestors.append(sr)
+                break
+        if ancestors:
+            ancestors.extend(ancestors[0].find_ancestors(shard_ranges))
+        else:
+            root_sr = self._find_root(self_parsed_name, shard_ranges)
+            if root_sr:
+                ancestors.append(root_sr)
+        return ancestors
+
     @classmethod
     def make_path(cls, shards_account, root_container, parent_container,
                   timestamp, index):

swift/container/backend.py

@@ -454,10 +454,7 @@ class ContainerBroker(DatabaseBroker):
         for sharding to have been initiated, False otherwise.
         """
         own_shard_range = self.get_own_shard_range()
-        if own_shard_range.state in (ShardRange.SHARDING,
-                                     ShardRange.SHRINKING,
-                                     ShardRange.SHARDED,
-                                     ShardRange.SHRUNK):
+        if own_shard_range.state in ShardRange.CLEAVING_STATES:
             return bool(self.get_shard_ranges())
         return False
 

swift/container/sharder.py

@@ -40,7 +40,7 @@ from swift.common.utils import get_logger, config_true_value, \
     Everything, config_auto_int_value, ShardRangeList, config_percent_value
 from swift.container.backend import ContainerBroker, \
     RECORD_TYPE_SHARD, UNSHARDED, SHARDING, SHARDED, COLLAPSED, \
-    SHARD_UPDATE_STATES
+    SHARD_UPDATE_STATES, sift_shard_ranges
 from swift.container.replicator import ContainerReplicator
 
 
@@ -101,7 +101,7 @@ def _find_discontinuity(paths, start):
     return longest_start_path, longest_end_path
 
 
-def find_paths_with_gaps(shard_ranges):
+def find_paths_with_gaps(shard_ranges, within_range=None):
     """
     Find gaps in the shard ranges and pairs of shard range paths that lead to
     and from those gaps. For each gap a single pair of adjacent paths is
@@ -109,6 +109,9 @@ def find_paths_with_gaps(shard_ranges):
     entire namespace with no overlaps.
 
     :param shard_ranges: a list of instances of ShardRange.
+    :param within_range: an optional ShardRange that constrains the search
+        space; the method will only return gaps within this range. The default
+        is the entire namespace.
     :return: A list of tuples of ``(start_path, gap_range, end_path)`` where
         ``start_path`` is a list of ShardRanges leading to the gap,
         ``gap_range`` is a ShardRange synthesized to describe the namespace
@@ -119,6 +122,7 @@ def find_paths_with_gaps(shard_ranges):
         namespace.
     """
     timestamp = Timestamp.now()
+    within_range = within_range or ShardRange('entire/namespace', timestamp)
     shard_ranges = ShardRangeList(shard_ranges)
     # note: find_paths results do not include shrinking ranges
     paths = find_paths(shard_ranges)
@@ -149,7 +153,8 @@ def find_paths_with_gaps(shard_ranges):
                                timestamp,
                                lower=start_path.upper,
                                upper=end_path.lower)
-        paths_with_gaps.append((start_path, gap_range, end_path))
+        if gap_range.overlaps(within_range):
+            paths_with_gaps.append((start_path, gap_range, end_path))
     return paths_with_gaps
 
 
@@ -497,6 +502,27 @@ def rank_paths(paths, shard_range_to_span):
     return paths
 
 
+def combine_shard_ranges(new_shard_ranges, existing_shard_ranges):
+    """
+    Combines new and existing shard ranges based on most recent state.
+
+    :param new_shard_ranges: a list of ShardRange instances.
+    :param existing_shard_ranges: a list of ShardRange instances.
+    :return: a list of ShardRange instances.
+    """
+    new_shard_ranges = [dict(sr) for sr in new_shard_ranges]
+    existing_shard_ranges = [dict(sr) for sr in existing_shard_ranges]
+    to_add, to_delete = sift_shard_ranges(
+        new_shard_ranges,
+        dict((sr['name'], sr) for sr in existing_shard_ranges))
+    result = [ShardRange.from_dict(existing)
+              for existing in existing_shard_ranges
+              if existing['name'] not in to_delete]
+    result.extend([ShardRange.from_dict(sr) for sr in to_add])
+    return sorted([sr for sr in result if not sr.deleted],
+                  key=ShardRange.sort_key)
+
+
 class CleavingContext(object):
     """
     Encapsulates metadata associated with the process of cleaving a retiring
@@ -916,9 +942,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
         if db_state not in (UNSHARDED, SHARDING, SHARDED):
             return
         own_shard_range = broker.get_own_shard_range()
-        if own_shard_range.state not in (
-                ShardRange.SHARDING, ShardRange.SHARDED,
-                ShardRange.SHRINKING, ShardRange.SHRUNK):
+        if own_shard_range.state not in ShardRange.CLEAVING_STATES:
             return
 
         if db_state == SHARDED:
@@ -1159,7 +1183,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
         warnings = []
         own_shard_range = broker.get_own_shard_range()
 
-        if own_shard_range.state in (ShardRange.SHARDING, ShardRange.SHARDED):
+        if own_shard_range.state in ShardRange.SHARDING_STATES:
             shard_ranges = [sr for sr in broker.get_shard_ranges()
                             if sr.state != ShardRange.SHRINKING]
             paths_with_gaps = find_paths_with_gaps(shard_ranges)
@@ -1245,9 +1269,8 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                 own_shard_range = broker.get_own_shard_range()
                 if (orig_own_shard_range != own_shard_range or
                         orig_own_shard_range.state != own_shard_range.state):
-                    self.logger.info(
-                        'Updated own shard range from %s to %s',
-                        orig_own_shard_range, own_shard_range)
+                    self.logger.info('Updated own shard range from %s to %s',
+                                     orig_own_shard_range, own_shard_range)
             elif shard_range.is_child_of(own_shard_range):
                 children_shard_ranges.append(shard_range)
             else:
@@ -1262,19 +1285,70 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                               len(children_shard_ranges))
             broker.merge_shard_ranges(children_shard_ranges)
 
-        if (other_shard_ranges and
-                own_shard_range.state in ShardRange.SHRINKING_STATES):
-            # If own_shard_range state is shrinking, save off *all* shards
-            # returned because these may contain shards into which this
-            # shard is to shrink itself; shrinking is the only case when we
-            # want to learn about *other* shard ranges from the root.
-            # We need to include shrunk state too, because one replica of a
-            # shard may already have moved the own_shard_range state to
-            # shrunk while another replica may still be in the process of
-            # shrinking.
-            self.logger.debug('Updating %s other shard range(s) from root',
-                              len(other_shard_ranges))
-            broker.merge_shard_ranges(other_shard_ranges)
+        if (other_shard_ranges
+                and own_shard_range.state in ShardRange.CLEAVING_STATES
+                and not broker.is_sharded()):
+            # Other shard ranges returned from the root may need to be merged
+            # for the purposes of sharding or shrinking this shard:
+            #
+            # Shrinking states: If the up-to-date state is shrinking, the
+            # shards fetched from root may contain shards into which this shard
+            # is to shrink itself. Shrinking is initiated by modifying multiple
+            # neighboring shard range states *in the root*, rather than
+            # modifying a shard directly. We therefore need to learn about
+            # *other* neighboring shard ranges from the root, possibly
+            # including the root itself. We need to include shrunk state too,
+            # because one replica of a shard may already have moved the
+            # own_shard_range state to shrunk while another replica may still
+            # be in the process of shrinking.
+            #
+            # Sharding states: Normally a shard will shard to its own children.
+            # However, in some circumstances a shard may need to shard to other
+            # non-children sub-shards. For example, a shard range repair may
+            # cause a child sub-shard to be deleted and its namespace covered
+            # by another 'acceptor' shard.
+            #
+            # Therefore, if the up-to-date own_shard_range state indicates that
+            # sharding or shrinking is in progress, then other shard ranges
+            # will be merged, with the following caveats: we never expect a
+            # shard to shard to any ancestor shard range including the root,
+            # but containers might ultimately *shrink* to root; we never want
+            # to cleave to a container that is itself sharding or shrinking;
+            # the merged shard ranges should not result in gaps or overlaps in
+            # the namespace of this shard.
+            #
+            # Note: the search for ancestors is guaranteed to find the parent
+            # and root *if they are present*, but if any ancestor is missing
+            # then there is a chance that older generations in the
+            # other_shard_ranges will not be filtered and could be merged. That
+            # is only a problem if they are somehow still in ACTIVE state, and
+            # no overlap is detected, so the ancestor is merged.
+            ancestor_names = [
+                sr.name for sr in own_shard_range.find_ancestors(shard_ranges)]
+            filtered_other_shard_ranges = [
+                sr for sr in other_shard_ranges
+                if (sr.name not in ancestor_names
+                    and (sr.state not in ShardRange.CLEAVING_STATES
+                         or sr.deleted))
+            ]
+            if own_shard_range.state in ShardRange.SHRINKING_STATES:
+                root_shard_range = own_shard_range.find_root(
+                    other_shard_ranges)
+                if (root_shard_range and
+                        root_shard_range.state == ShardRange.ACTIVE):
+                    filtered_other_shard_ranges.append(root_shard_range)
+            existing_shard_ranges = broker.get_shard_ranges()
+            combined_shard_ranges = combine_shard_ranges(
+                filtered_other_shard_ranges, existing_shard_ranges)
+            overlaps = find_overlapping_ranges(combined_shard_ranges)
+            paths_with_gaps = find_paths_with_gaps(
+                combined_shard_ranges, own_shard_range)
+            if not (overlaps or paths_with_gaps):
+                # only merge if shard ranges appear to be *good*
+                self.logger.debug(
+                    'Updating %s other shard range(s) from root',
+                    len(filtered_other_shard_ranges))
+                broker.merge_shard_ranges(filtered_other_shard_ranges)
 
         return own_shard_range, own_shard_range_from_root
 
@@ -2168,10 +2242,7 @@ class ContainerSharder(ContainerSharderConf, ContainerReplicator):
                     broker, shard_ranges=[broker.get_own_shard_range()])
 
             own_shard_range = broker.get_own_shard_range()
-            if own_shard_range.state in (ShardRange.SHARDING,
-                                         ShardRange.SHRINKING,
-                                         ShardRange.SHARDED,
-                                         ShardRange.SHRUNK):
+            if own_shard_range.state in ShardRange.CLEAVING_STATES:
                 if broker.get_shard_ranges():
                     # container has been given shard ranges rather than
                     # found them e.g. via replication or a shrink event,

test/probe/test_sharder.py

@@ -1801,13 +1801,12 @@ class TestContainerSharding(BaseAutoContainerSharding):
             donor = orig_shard_ranges[0]
             shard_nodes_data = self.direct_get_container_shard_ranges(
                 donor.account, donor.container)
-            # the donor's shard range will have the acceptor's projected stats;
-            # donor also has copy of root shard range that will be ignored;
-            # note: expected_shards does not include the sharded root range
+            # donor has the acceptor shard range but not the root shard range
+            # because the root is still in ACTIVE state;
+            # the donor's shard range will have the acceptor's projected stats
             obj_count, bytes_used = check_shard_nodes_data(
                 shard_nodes_data, expected_state='sharded', expected_shards=1,
-                exp_obj_count=len(second_shard_objects) + 1,
-                exp_sharded_root_range=True)
+                exp_obj_count=len(second_shard_objects) + 1)
             # but the donor is empty and so reports zero stats
             self.assertEqual(0, obj_count)
             self.assertEqual(0, bytes_used)

test/unit/cli/test_manage_shard_ranges.py

@@ -24,7 +24,7 @@ from tempfile import mkdtemp
 import six
 from six.moves import cStringIO as StringIO
 
-from swift.cli.manage_shard_ranges import main, combine_shard_ranges
+from swift.cli.manage_shard_ranges import main
 from swift.common import utils
 from swift.common.utils import Timestamp, ShardRange
 from swift.container.backend import ContainerBroker
@@ -2858,46 +2858,3 @@ class TestManageShardRanges(unittest.TestCase):
         self.assertIn(
             "argument --yes/-y: not allowed with argument --dry-run/-n",
             err_lines[-2], err_lines)
-
-    def test_combine_shard_ranges(self):
-        ts_iter = make_timestamp_iter()
-        this = ShardRange('a/o', next(ts_iter).internal)
-        that = ShardRange('a/o', next(ts_iter).internal)
-        actual = combine_shard_ranges([dict(this)], [dict(that)])
-        self.assertEqual([dict(that)], [dict(sr) for sr in actual])
-        actual = combine_shard_ranges([dict(that)], [dict(this)])
-        self.assertEqual([dict(that)], [dict(sr) for sr in actual])
-
-        ts = next(ts_iter).internal
-        this = ShardRange('a/o', ts, state=ShardRange.ACTIVE,
-                          state_timestamp=next(ts_iter))
-        that = ShardRange('a/o', ts, state=ShardRange.CREATED,
-                          state_timestamp=next(ts_iter))
-        actual = combine_shard_ranges([dict(this)], [dict(that)])
-        self.assertEqual([dict(that)], [dict(sr) for sr in actual])
-        actual = combine_shard_ranges([dict(that)], [dict(this)])
-        self.assertEqual([dict(that)], [dict(sr) for sr in actual])
-
-        that.update_meta(1, 2, meta_timestamp=next(ts_iter))
-        this.update_meta(3, 4, meta_timestamp=next(ts_iter))
-        expected = that.copy(object_count=this.object_count,
-                             bytes_used=this.bytes_used,
-                             meta_timestamp=this.meta_timestamp)
-        actual = combine_shard_ranges([dict(this)], [dict(that)])
-        self.assertEqual([dict(expected)], [dict(sr) for sr in actual])
-        actual = combine_shard_ranges([dict(that)], [dict(this)])
-        self.assertEqual([dict(expected)], [dict(sr) for sr in actual])
-
-        this = ShardRange('a/o', next(ts_iter).internal)
-        that = ShardRange('a/o', next(ts_iter).internal, deleted=True)
-        actual = combine_shard_ranges([dict(this)], [dict(that)])
-        self.assertFalse(actual, [dict(sr) for sr in actual])
-        actual = combine_shard_ranges([dict(that)], [dict(this)])
-        self.assertFalse(actual, [dict(sr) for sr in actual])
-
-        this = ShardRange('a/o', next(ts_iter).internal, deleted=True)
-        that = ShardRange('a/o', next(ts_iter).internal)
-        actual = combine_shard_ranges([dict(this)], [dict(that)])
-        self.assertEqual([dict(that)], [dict(sr) for sr in actual])
-        actual = combine_shard_ranges([dict(that)], [dict(this)])
-        self.assertEqual([dict(that)], [dict(sr) for sr in actual])

test/unit/common/test_utils.py

@@ -8061,6 +8061,19 @@ class TestShardRange(unittest.TestCase):
     def setUp(self):
         self.ts_iter = make_timestamp_iter()
 
+    def test_constants(self):
+        self.assertEqual({utils.ShardRange.SHARDING,
+                          utils.ShardRange.SHARDED,
+                          utils.ShardRange.SHRINKING,
+                          utils.ShardRange.SHRUNK},
+                         set(utils.ShardRange.CLEAVING_STATES))
+        self.assertEqual({utils.ShardRange.SHARDING,
+                          utils.ShardRange.SHARDED},
+                         set(utils.ShardRange.SHARDING_STATES))
+        self.assertEqual({utils.ShardRange.SHRINKING,
+                          utils.ShardRange.SHRUNK},
+                         set(utils.ShardRange.SHRINKING_STATES))
+
     def test_min_max_bounds(self):
         with self.assertRaises(TypeError):
             utils.ShardRangeOuterBound()
@@ -9099,6 +9112,120 @@ class TestShardRange(unittest.TestCase):
         self.assertTrue(a1_r1_gp1.is_child_of(a2_r1))
         self.assertTrue(a2_r1_gp1.is_child_of(a1_r1))
 
+    def test_find_root(self):
+        # account 1
+        ts = next(self.ts_iter)
+        a1_r1 = utils.ShardRange('a1/r1', ts)
+        ts = next(self.ts_iter)
+        a1_r1_gp1 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', 'r1', ts, 1), ts, '', 'l')
+        ts = next(self.ts_iter)
+        a1_r1_gp1_p1 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', a1_r1_gp1.container, ts, 1), ts, 'a', 'k')
+        ts = next(self.ts_iter)
+        a1_r1_gp1_p1_c1 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', a1_r1_gp1_p1.container, ts, 1), ts, 'a', 'j')
+        ts = next(self.ts_iter)
+        a1_r1_gp1_p2 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', a1_r1_gp1.container, ts, 2), ts, 'k', 'l')
+        ts = next(self.ts_iter)
+        a1_r1_gp2 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', 'r1', ts, 2), ts, 'l', '')  # different index
+
+        # full ancestry plus some others
+        all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1,
+                            a1_r1_gp1_p2, a1_r1_gp2]
+        random.shuffle(all_shard_ranges)
+        self.assertIsNone(a1_r1.find_root(all_shard_ranges))
+        self.assertEqual(a1_r1, a1_r1_gp1.find_root(all_shard_ranges))
+        self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges))
+        self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges))
+
+        # missing a1_r1_gp1_p1
+        all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1_c1,
+                            a1_r1_gp1_p2, a1_r1_gp2]
+        random.shuffle(all_shard_ranges)
+        self.assertIsNone(a1_r1.find_root(all_shard_ranges))
+        self.assertEqual(a1_r1, a1_r1_gp1.find_root(all_shard_ranges))
+        self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges))
+        self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges))
+
+        # empty list
+        self.assertIsNone(a1_r1_gp1_p1_c1.find_root([]))
+
+        # double entry
+        all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1, a1_r1_gp1]
+        random.shuffle(all_shard_ranges)
+        self.assertEqual(a1_r1, a1_r1_gp1_p1.find_root(all_shard_ranges))
+        self.assertEqual(a1_r1, a1_r1_gp1_p1_c1.find_root(all_shard_ranges))
+
+    def test_find_ancestors(self):
+        # account 1
+        ts = next(self.ts_iter)
+        a1_r1 = utils.ShardRange('a1/r1', ts)
+        ts = next(self.ts_iter)
+        a1_r1_gp1 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', 'r1', ts, 1), ts, '', 'l')
+        ts = next(self.ts_iter)
+        a1_r1_gp1_p1 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', a1_r1_gp1.container, ts, 1), ts, 'a', 'k')
+        ts = next(self.ts_iter)
+        a1_r1_gp1_p1_c1 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', a1_r1_gp1_p1.container, ts, 1), ts, 'a', 'j')
+        ts = next(self.ts_iter)
+        a1_r1_gp1_p2 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', a1_r1_gp1.container, ts, 2), ts, 'k', 'l')
+        ts = next(self.ts_iter)
+        a1_r1_gp2 = utils.ShardRange(utils.ShardRange.make_path(
+            '.shards_a1', 'r1', 'r1', ts, 2), ts, 'l', '')  # different index
+
+        # full ancestry plus some others
+        all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1,
+                            a1_r1_gp1_p2, a1_r1_gp2]
+        random.shuffle(all_shard_ranges)
+        self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1_gp1, a1_r1],
+                         a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1_gp1_p1, a1_r1_gp1, a1_r1],
+                         a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
+
+        # missing a1_r1_gp1_p1
+        all_shard_ranges = [a1_r1, a1_r1_gp1, a1_r1_gp1_p1_c1,
+                            a1_r1_gp1_p2, a1_r1_gp2]
+        random.shuffle(all_shard_ranges)
+        self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1_gp1, a1_r1],
+                         a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1],
+                         a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
+
+        # missing a1_r1_gp1
+        all_shard_ranges = [a1_r1, a1_r1_gp1_p1, a1_r1_gp1_p1_c1,
+                            a1_r1_gp1_p2, a1_r1_gp2]
+        random.shuffle(all_shard_ranges)
+        self.assertEqual([], a1_r1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1], a1_r1_gp1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1],
+                         a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1_gp1_p1, a1_r1],
+                         a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
+
+        # empty list
+        self.assertEqual([], a1_r1_gp1_p1_c1.find_ancestors([]))
+        # double entry
+        all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1, a1_r1_gp1]
+        random.shuffle(all_shard_ranges)
+        self.assertEqual([a1_r1_gp1, a1_r1],
+                         a1_r1_gp1_p1.find_ancestors(all_shard_ranges))
+        self.assertEqual([a1_r1],
+                         a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
+        all_shard_ranges = [a1_r1, a1_r1, a1_r1_gp1_p1, a1_r1_gp1_p1]
+        random.shuffle(all_shard_ranges)
+        self.assertEqual([a1_r1_gp1_p1, a1_r1],
+                         a1_r1_gp1_p1_c1.find_ancestors(all_shard_ranges))
+
     def test_expand(self):
         bounds = (('', 'd'), ('d', 'k'), ('k', 't'), ('t', ''))
         donors = [

test/unit/container/test_sharder.py

@@ -43,7 +43,7 @@ from swift.container.sharder import ContainerSharder, sharding_enabled, \
     find_shrinking_candidates, process_compactible_shard_sequences, \
     find_compactible_shard_sequences, is_shrinking_candidate, \
     is_sharding_candidate, find_paths, rank_paths, ContainerSharderConf, \
-    find_paths_with_gaps
+    find_paths_with_gaps, combine_shard_ranges
 from swift.common.utils import ShardRange, Timestamp, hash_path, \
     encode_timestamps, parse_db_filename, quorum_size, Everything, md5, \
     ShardName
@@ -5840,7 +5840,7 @@ class TestSharder(BaseTestSharder):
 
     def _do_test_audit_shard_container_merge_other_ranges(self, *args):
         # verify that shard only merges other ranges from root when it is
-        # shrinking or shrunk
+        # cleaving
         shard_bounds = (
             ('a', 'p'), ('k', 't'), ('p', 'u'))
         shard_states = (
@@ -5855,7 +5855,7 @@ class TestSharder(BaseTestSharder):
             broker.set_sharding_sysmeta(*args)
             shard_ranges[1].name = broker.path
 
-            # make own shard range match shard_ranges[1]
+            # make shard's own shard range match shard_ranges[1]
             own_sr = shard_ranges[1]
             expected_stats = {'attempted': 1, 'success': 1, 'failure': 0}
             self.assertTrue(own_sr.update_state(own_state,
@@ -5903,7 +5903,7 @@ class TestSharder(BaseTestSharder):
                     self.assertEqual(root_state, own_shard_range.state)
                     self.assertEqual(root_ts, own_shard_range.state_timestamp)
                     updated_ranges = broker.get_shard_ranges(include_own=True)
-                    if root_state in (ShardRange.SHRINKING, ShardRange.SHRUNK):
+                    if root_state in ShardRange.CLEAVING_STATES:
                         # check other shard ranges from root are merged
                         self.assertEqual(shard_ranges, updated_ranges)
                     else:
@@ -5925,7 +5925,7 @@ class TestSharder(BaseTestSharder):
                     self.assertEqual(own_state, own_shard_range.state)
                     self.assertEqual(own_ts, own_shard_range.state_timestamp)
                     updated_ranges = broker.get_shard_ranges(include_own=True)
-                    if own_state in (ShardRange.SHRINKING, ShardRange.SHRUNK):
+                    if own_state in ShardRange.CLEAVING_STATES:
                         # check other shard ranges from root are merged
                         self.assertEqual(shard_ranges, updated_ranges)
                     else:
@@ -5940,7 +5940,7 @@ class TestSharder(BaseTestSharder):
                                                                'a/c')
 
     def _assert_merge_into_shard(self, own_shard_range, shard_ranges,
-                                 root_shard_ranges, expected, *args):
+                                 root_shard_ranges, expected, *args, **kwargs):
         # create a shard broker, initialise with shard_ranges, run audit on it
         # supplying given root_shard_ranges and verify that the broker ends up
         # with expected shard ranges.
@@ -5948,6 +5948,13 @@ class TestSharder(BaseTestSharder):
                                    container=own_shard_range.container)
         broker.set_sharding_sysmeta(*args)
         broker.merge_shard_ranges([own_shard_range] + shard_ranges)
+        db_state = kwargs.get('db_state', UNSHARDED)
+        if db_state == SHARDING:
+            broker.set_sharding_state()
+        if db_state == SHARDED:
+            broker.set_sharding_state()
+            broker.set_sharded_state()
+        self.assertEqual(db_state, broker.get_db_state())
         self.assertFalse(broker.is_root_container())
 
         sharder, mock_swift = self.call_audit_container(
@@ -5998,26 +6005,26 @@ class TestSharder(BaseTestSharder):
                 self.assertFalse(sharder.logger.get_lines_for_level('error'))
 
         for own_state in ShardRange.STATES:
-            if own_state in ShardRange.SHRINKING_STATES:
-                # shrinking states are covered by other tests
+            if own_state in ShardRange.CLEAVING_STATES:
+                # cleaving states are covered by other tests
                 continue
             for acceptor_state in ShardRange.STATES:
                 for root_state in ShardRange.STATES:
                     do_test(own_state, acceptor_state, root_state)
 
-    def test_audit_old_style_shard_root_ranges_not_merged_not_shrinking(self):
+    def test_audit_old_style_shard_root_ranges_not_merged_not_cleaving(self):
         # verify that other shard ranges from root are NOT merged into shard
-        # when it is NOT in a shrinking state
+        # when it is NOT in a cleaving state
         self._do_test_audit_shard_root_ranges_not_merged('Root', 'a/c')
 
-    def test_audit_shard_root_ranges_not_merged_not_shrinking(self):
+    def test_audit_shard_root_ranges_not_merged_not_cleaving(self):
         # verify that other shard ranges from root are NOT merged into shard
-        # when it is NOT in a shrinking state
+        # when it is NOT in a cleaving state
         self._do_test_audit_shard_root_ranges_not_merged('Quoted-Root', 'a/c')
 
     def test_audit_shard_root_ranges_with_own_merged_while_shrinking(self):
-        # Verify that shrinking shard will merge root and other ranges,
-        # including root range.
+        # Verify that shrinking shard will merge other ranges, but not
+        # in-ACTIVE root range.
         # Make root and other ranges that fully contain the shard namespace...
         root_own_sr = ShardRange('a/c', next(self.ts_iter))
         acceptor = ShardRange(
@@ -6034,7 +6041,7 @@ class TestSharder(BaseTestSharder):
             own_sr = ShardRange(
                 str(ShardName.create('.shards_a', 'c', 'c', ts, 0)),
                 ts, lower='a', upper='b', state=own_state, state_timestamp=ts)
-            expected = [acceptor_from_root, root_from_root]
+            expected = [acceptor_from_root]
             with annotate_failure('with states %s %s %s'
                                   % (own_state, acceptor_state, root_state)):
                 sharder = self._assert_merge_into_shard(
@@ -6047,12 +6054,21 @@ class TestSharder(BaseTestSharder):
 
         for own_state in ShardRange.SHRINKING_STATES:
             for acceptor_state in ShardRange.STATES:
+                if acceptor_state in ShardRange.CLEAVING_STATES:
+                    # special case covered in other tests
+                    continue
                 for root_state in ShardRange.STATES:
-                    do_test(own_state, acceptor_state, root_state)
+                    if root_state == ShardRange.ACTIVE:
+                        # special case: ACTIVE root *is* merged
+                        continue
+                    with annotate_failure(
+                            'with states %s %s %s'
+                            % (own_state, acceptor_state, root_state)):
+                        do_test(own_state, acceptor_state, root_state)
 
     def test_audit_shard_root_ranges_missing_own_merged_while_shrinking(self):
-        # Verify that shrinking shard will merge root and other ranges,
-        # including root range.
+        # Verify that shrinking shard will merge other ranges, but not
+        # in-ACTIVE root range, even when root does not have shard's own range.
         # Make root and other ranges that fully contain the shard namespace...
         root_own_sr = ShardRange('a/c', next(self.ts_iter))
         acceptor = ShardRange(
@@ -6069,7 +6085,7 @@ class TestSharder(BaseTestSharder):
             own_sr = ShardRange(
                 str(ShardName.create('.shards_a', 'c', 'c', ts, 0)),
                 ts, lower='a', upper='b', state=own_state, state_timestamp=ts)
-            expected = [acceptor_from_root, root_from_root]
+            expected = [acceptor_from_root]
             with annotate_failure('with states %s %s %s'
                                   % (own_state, acceptor_state, root_state)):
                 sharder = self._assert_merge_into_shard(
@@ -6085,8 +6101,86 @@ class TestSharder(BaseTestSharder):
 
         for own_state in ShardRange.SHRINKING_STATES:
             for acceptor_state in ShardRange.STATES:
+                if acceptor_state in ShardRange.CLEAVING_STATES:
+                    # special case covered in other tests
+                    continue
                 for root_state in ShardRange.STATES:
-                    do_test(own_state, acceptor_state, root_state)
+                    if root_state == ShardRange.ACTIVE:
+                        # special case: ACTIVE root *is* merged
+                        continue
+                    with annotate_failure(
+                            'with states %s %s %s'
+                            % (own_state, acceptor_state, root_state)):
+                        do_test(own_state, acceptor_state, root_state)
+
+    def test_audit_shard_root_range_not_merged_while_shrinking(self):
+        # Verify that shrinking shard will not merge an in-active root range
+        def do_test(own_state, root_state):
+            root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                     state=ShardRange.SHARDED)
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            expected = []
+            sharder = self._assert_merge_into_shard(
+                own_sr, [], [own_sr, root_own_sr],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.SHRINKING_STATES:
+            for root_state in ShardRange.STATES:
+                if root_state == ShardRange.ACTIVE:
+                    continue  # special case tested below
+                with annotate_failure((own_state, root_state)):
+                    do_test(own_state, root_state)
+
+    def test_audit_shard_root_range_overlap_not_merged_while_shrinking(self):
+        # Verify that shrinking shard will not merge an active root range that
+        # overlaps with an exosting sub-shard
+        def do_test(own_state):
+            root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                     state=ShardRange.ACTIVE)
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            ts = next(self.ts_iter)
+            sub_shard = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', own_sr.container, ts, 0)),
+                ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
+            expected = [sub_shard]
+            sharder = self._assert_merge_into_shard(
+                own_sr, [sub_shard], [own_sr, root_own_sr],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.SHRINKING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
+
+    def test_audit_shard_active_root_range_merged_while_shrinking(self):
+        # Verify that shrinking shard will merge an active root range
+        def do_test(own_state):
+            root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                     state=ShardRange.ACTIVE)
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            expected = [root_own_sr]
+            sharder = self._assert_merge_into_shard(
+                own_sr, [], [own_sr, root_own_sr],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.SHRINKING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
 
     def test_audit_shard_root_ranges_fetch_fails_while_shrinking(self):
         # check audit copes with failed response while shard is shrinking
@@ -6106,6 +6200,425 @@ class TestSharder(BaseTestSharder):
                       warning_lines[1])
         self.assertFalse(sharder.logger.get_lines_for_level('error'))
 
+    def test_audit_shard_root_ranges_merge_while_unsharded(self):
+        # Verify that unsharded shard with no existing shard ranges will merge
+        # other ranges, but not root range.
+        root_own_sr = ShardRange('a/c', next(self.ts_iter))
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+
+        def do_test(own_state, acceptor_state, root_state):
+            acceptor_from_root = acceptor.copy(
+                timestamp=next(self.ts_iter), state=acceptor_state)
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            root_from_root = root_own_sr.copy(
+                timestamp=next(self.ts_iter), state=root_state)
+            expected = [acceptor_from_root]
+            sharder = self._assert_merge_into_shard(
+                own_sr, [],
+                [own_sr, acceptor_from_root, root_from_root],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.SHARDING_STATES:
+            for acceptor_state in ShardRange.STATES:
+                if acceptor_state in ShardRange.CLEAVING_STATES:
+                    # special case covered in other tests
+                    continue
+                for root_state in ShardRange.STATES:
+                    with annotate_failure(
+                            'with states %s %s %s'
+                            % (own_state, acceptor_state, root_state)):
+                        do_test(own_state, acceptor_state, root_state)
+
+    def test_audit_shard_root_ranges_merge_while_sharding(self):
+        # Verify that sharding shard with no existing shard ranges will merge
+        # other ranges, but not root range.
+        root_own_sr = ShardRange('a/c', next(self.ts_iter))
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+
+        def do_test(own_state, acceptor_state, root_state):
+            acceptor_from_root = acceptor.copy(
+                timestamp=next(self.ts_iter), state=acceptor_state)
+            ts = next(self.ts_iter)
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', ts, 0)),
+                ts, 'a', 'b', epoch=ts, state=own_state)
+            root_from_root = root_own_sr.copy(
+                timestamp=next(self.ts_iter), state=root_state)
+            expected = [acceptor_from_root]
+            sharder = self._assert_merge_into_shard(
+                own_sr, [],
+                [own_sr, acceptor_from_root, root_from_root],
+                expected, 'Quoted-Root', 'a/c', db_state=SHARDING)
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.SHARDING_STATES:
+            for acceptor_state in ShardRange.STATES:
+                if acceptor_state in ShardRange.CLEAVING_STATES:
+                    # special case covered in other tests
+                    continue
+                for root_state in ShardRange.STATES:
+                    with annotate_failure(
+                            'with states %s %s %s'
+                            % (own_state, acceptor_state, root_state)):
+                        do_test(own_state, acceptor_state, root_state)
+
+    def test_audit_shard_root_ranges_not_merged_once_sharded(self):
+        # Verify that sharded shard will not merge other ranges from root
+        root_own_sr = ShardRange('a/c', next(self.ts_iter))
+        # the acceptor complements the single existing sub-shard...
+        other_sub_shard = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'ab', 'c', state=ShardRange.ACTIVE)
+
+        def do_test(own_state, other_sub_shard_state, root_state):
+            sub_shard_from_root = other_sub_shard.copy(
+                timestamp=next(self.ts_iter), state=other_sub_shard_state)
+            ts = next(self.ts_iter)
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', ts, 0)),
+                ts, 'a', 'b', epoch=ts, state=own_state)
+            ts = next(self.ts_iter)
+            sub_shard = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', own_sr.container, ts, 0)),
+                ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
+            root_from_root = root_own_sr.copy(
+                timestamp=next(self.ts_iter), state=root_state)
+            expected = [sub_shard]
+            sharder = self._assert_merge_into_shard(
+                own_sr, [sub_shard],
+                [own_sr, sub_shard_from_root, root_from_root],
+                expected, 'Quoted-Root', 'a/c', db_state=SHARDED)
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in (ShardRange.SHARDED, ShardRange.SHRUNK):
+            for other_sub_shard_state in ShardRange.STATES:
+                for root_state in ShardRange.STATES:
+                    with annotate_failure(
+                            'with states %s %s %s'
+                            % (own_state, other_sub_shard_state, root_state)):
+                        do_test(own_state, other_sub_shard_state, root_state)
+
+    def test_audit_shard_root_ranges_replace_existing_while_cleaving(self):
+        # Verify that sharding shard with stale existing sub-shard ranges will
+        # merge other ranges, but not root range.
+        root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                 state=ShardRange.SHARDED)
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+        ts = next(self.ts_iter)
+        acceptor_sub_shards = [ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', acceptor.container, ts, i)),
+            ts, lower, upper, state=ShardRange.ACTIVE)
+            for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
+
+        # shard has incomplete existing shard ranges, ranges from root delete
+        # existing sub-shard and replace with other acceptor sub-shards
+        def do_test(own_state):
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            ts = next(self.ts_iter)
+            sub_shard = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', own_sr.container, ts, 0)),
+                ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
+            deleted_sub_shard = sub_shard.copy(
+                timestamp=next(self.ts_iter), state=ShardRange.SHARDED,
+                deleted=1)
+            expected = acceptor_sub_shards
+            sharder = self._assert_merge_into_shard(
+                own_sr, [sub_shard],
+                [root_own_sr, own_sr, deleted_sub_shard] + acceptor_sub_shards,
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.CLEAVING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
+
+    def test_audit_shard_root_ranges_supplement_deleted_while_cleaving(self):
+        # Verify that sharding shard with deleted existing sub-shard ranges
+        # will merge other ranges while sharding, but not root range.
+        root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                 state=ShardRange.SHARDED)
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+        ts = next(self.ts_iter)
+        acceptor_sub_shards = [ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', acceptor.container, ts, i)),
+            ts, lower, upper, state=ShardRange.ACTIVE)
+            for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
+
+        # shard already has deleted existing shard ranges
+        expected = acceptor_sub_shards
+
+        def do_test(own_state):
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            ts = next(self.ts_iter)
+            deleted_sub_shards = [ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', own_sr.container, ts, i)),
+                ts, lower, upper, state=ShardRange.SHARDED, deleted=1)
+                for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'b'))]
+            sharder = self._assert_merge_into_shard(
+                own_sr, deleted_sub_shards,
+                [own_sr, root_own_sr] + acceptor_sub_shards,
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.CLEAVING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
+
+    def test_audit_shard_root_ranges_supplement_existing_while_cleaving(self):
+        # Verify that sharding shard with incomplete existing sub-shard ranges
+        # will merge other ranges that fill the gap, but not root range.
+        root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                 state=ShardRange.SHARDED)
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+        ts = next(self.ts_iter)
+        acceptor_sub_shards = [ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', acceptor.container, ts, i)),
+            ts, lower, upper, state=ShardRange.ACTIVE)
+            for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
+
+        # shard has incomplete existing shard ranges and range from root fills
+        # the gap
+        def do_test(own_state):
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            ts = next(self.ts_iter)
+            sub_shard = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', own_sr.container, ts, 0)),
+                ts, lower='a', upper='ab', state=ShardRange.ACTIVE)
+            expected = [sub_shard] + acceptor_sub_shards[1:]
+            sharder = self._assert_merge_into_shard(
+                own_sr, [sub_shard],
+                [own_sr, root_own_sr] + acceptor_sub_shards[1:],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.CLEAVING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
+
+    def test_audit_shard_root_ranges_cleaving_not_merged_while_cleaving(self):
+        # Verify that sharding shard will not merge other ranges that are in a
+        # cleaving state.
+        root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                 state=ShardRange.SHARDED)
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+
+        def do_test(own_state, acceptor_state, root_state):
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            root_from_root = root_own_sr.copy(
+                timestamp=next(self.ts_iter), state=root_state)
+            acceptor_from_root = acceptor.copy(
+                timestamp=next(self.ts_iter), state=acceptor_state)
+
+            if (own_state in ShardRange.SHRINKING_STATES and
+                    root_state == ShardRange.ACTIVE):
+                # special case: when shrinking, ACTIVE root shard *is* merged
+                expected = [root_from_root]
+            else:
+                expected = []
+
+            sharder = self._assert_merge_into_shard(
+                own_sr, [],
+                [own_sr, acceptor_from_root, root_from_root],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        # ranges from root that are in a cleaving state are not merged...
+        for own_state in ShardRange.CLEAVING_STATES:
+            for acceptor_state in ShardRange.CLEAVING_STATES:
+                for root_state in ShardRange.STATES:
+                    with annotate_failure(
+                            'with states %s %s %s'
+                            % (own_state, acceptor_state, root_state)):
+                        do_test(own_state, acceptor_state, root_state)
+
+    def test_audit_shard_root_ranges_overlap_not_merged_while_cleaving_1(self):
+        # Verify that sharding/shrinking shard will not merge other ranges that
+        # would create an overlap; shard has complete existing shard ranges,
+        # newer range from root ignored
+        root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                 state=ShardRange.SHARDED)
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+
+        def do_test(own_state):
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            ts = next(self.ts_iter)
+            sub_shards = [ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', own_sr.container, ts, i)),
+                ts, lower, upper, state=ShardRange.ACTIVE)
+                for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'b'))]
+            acceptor_from_root = acceptor.copy(timestamp=next(self.ts_iter))
+            expected = sub_shards
+            sharder = self._assert_merge_into_shard(
+                own_sr, sub_shards,
+                [own_sr, acceptor_from_root, root_own_sr],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.CLEAVING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
+
+    def test_audit_shard_root_ranges_overlap_not_merged_while_cleaving_2(self):
+        # Verify that sharding/shrinking shard will not merge other ranges that
+        # would create an overlap; shard has incomplete existing shard ranges
+        # but ranges from root overlaps
+        root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                 state=ShardRange.SHARDED)
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+        ts = next(self.ts_iter)
+        acceptor_sub_shards = [ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', acceptor.container, ts, i)),
+            ts, lower, upper, state=ShardRange.ACTIVE)
+            for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
+
+        def do_test(own_state):
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            ts = next(self.ts_iter)
+            sub_shard = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', own_sr.container, ts, 0)),
+                ts, lower='a', upper='abc', state=ShardRange.ACTIVE)
+            expected = [sub_shard]
+            sharder = self._assert_merge_into_shard(
+                own_sr, [sub_shard],
+                acceptor_sub_shards[1:] + [own_sr, root_own_sr],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.CLEAVING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
+
+    def test_audit_shard_root_ranges_with_gap_not_merged_while_cleaving(self):
+        # Verify that sharding/shrinking shard will not merge other ranges that
+        # would leave a gap.
+        root_own_sr = ShardRange('a/c', next(self.ts_iter),
+                                 state=ShardRange.SHARDED)
+        acceptor = ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', 'c', next(self.ts_iter), 1)),
+            next(self.ts_iter), 'a', 'c', state=ShardRange.ACTIVE)
+        ts = next(self.ts_iter)
+        acceptor_sub_shards = [ShardRange(
+            str(ShardRange.make_path(
+                '.shards_a', 'c', acceptor.container, ts, i)),
+            ts, lower, upper, state=ShardRange.ACTIVE)
+            for i, lower, upper in ((0, 'a', 'ab'), (1, 'ab', 'c'))]
+
+        def do_test(own_state):
+            own_sr = ShardRange(
+                str(ShardName.create(
+                    '.shards_a', 'c', 'c', next(self.ts_iter), 0)),
+                next(self.ts_iter), 'a', 'b', state=own_state)
+            # root ranges have gaps w.r.t. the shard namespace
+            existing = expected = []
+            sharder = self._assert_merge_into_shard(
+                own_sr, existing,
+                acceptor_sub_shards[:1] + [own_sr, root_own_sr],
+                expected, 'Quoted-Root', 'a/c')
+            self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+            self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
+        for own_state in ShardRange.CLEAVING_STATES:
+            with annotate_failure(own_state):
+                do_test(own_state)
+
+    def test_audit_shard_container_ancestors_not_merged_while_sharding(self):
+        # Verify that sharding shard will not merge parent and root shard
+        # ranges even when the sharding shard has no other ranges
+        root_sr = ShardRange('a/root', next(self.ts_iter),
+                             state=ShardRange.SHARDED)
+        grandparent_path = ShardRange.make_path(
+            '.shards_a', 'root', root_sr.container, next(self.ts_iter), 2)
+        grandparent_sr = ShardRange(grandparent_path, next(self.ts_iter),
+                                    '', 'd', state=ShardRange.ACTIVE)
+        self.assertTrue(grandparent_sr.is_child_of(root_sr))
+        parent_path = ShardRange.make_path(
+            '.shards_a', 'root', grandparent_sr.container, next(self.ts_iter),
+            2)
+        parent_sr = ShardRange(parent_path, next(self.ts_iter), '', 'd',
+                               state=ShardRange.ACTIVE)
+        self.assertTrue(parent_sr.is_child_of(grandparent_sr))
+        child_path = ShardRange.make_path(
+            '.shards_a', 'root', parent_sr.container, next(self.ts_iter), 2)
+        child_own_sr = ShardRange(child_path, next(self.ts_iter), 'a', 'b',
+                                  state=ShardRange.SHARDING)
+        self.assertTrue(child_own_sr.is_child_of(parent_sr))
+
+        ranges_from_root = [grandparent_sr, parent_sr, root_sr, child_own_sr]
+        expected = []
+        sharder = self._assert_merge_into_shard(
+            child_own_sr, [], ranges_from_root, expected, 'Quoted-Root', 'a/c')
+        self.assertFalse(sharder.logger.get_lines_for_level('warning'))
+        self.assertFalse(sharder.logger.get_lines_for_level('error'))
+
     def test_audit_shard_container_children_merged_while_sharding(self):
         # Verify that sharding shard will always merge children shard ranges
         def do_test(child_deleted, child_state):
@@ -6157,9 +6670,11 @@ class TestSharder(BaseTestSharder):
                 'GET', '/v1/a/c', expected_headers,
                 acceptable_statuses=(2,), params=params)
 
+            expected = child_srs + [parent_sr]
+            if child_deleted:
+                expected.append(other_sr)
             self._assert_shard_ranges_equal(
-                sorted(child_srs + [parent_sr],
-                       key=ShardRange.sort_key),
+                sorted(expected, key=ShardRange.sort_key),
                 sorted(broker.get_shard_ranges(
                     include_own=True, include_deleted=True),
                     key=ShardRange.sort_key))
@@ -8141,7 +8656,7 @@ class TestSharderFunctions(BaseTestSharder):
             bounds, ShardRange.ACTIVE,
             timestamp=next(self.ts_iter), object_count=1)
         paths_with_gaps = find_paths_with_gaps(ranges)
-        self.assertEqual(3, len(paths_with_gaps))
+        self.assertEqual(3, len(paths_with_gaps), paths_with_gaps)
         self.assertEqual(
             [(ShardRange.MIN, ShardRange.MIN),
              (ShardRange.MIN, 'a'),
@@ -8161,6 +8676,38 @@ class TestSharderFunctions(BaseTestSharder):
             [(r.lower, r.upper) for r in paths_with_gaps[2]]
         )
 
+        range_of_interest = ShardRange('test/range', next(self.ts_iter))