From 6d1a0b334729f65f16e9875630996023f22511c4 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Tue, 7 Feb 2023 17:00:59 +1100 Subject: [PATCH] Don't clear x-container-sysmeta-sharding on delete_db In the container backend we have a delete_meta_whitelist which whitelists specific metadata from being cleared when we delete a broker. Currently the sharding root and quoted root are in there. They're there because we now process deleted shards and we need to know what a shard's root is. But since that time we've found and edge case bug where we've found old sharding brokers that are stuck because they're deleted (a deleted shard will clear the sysmeta-sharding truth value), but also has lost their shard-ranges. This makes them fail the sharding_enabled check and therefore never get a chance to pull new shard ranges from their root and are therefore stuck. A deleted shard wont be recreated, so it doesn't hurt to keep this sharding sysmeta value, and will be an extra line of defence stopping similar stuck container issues in the future. Change-Id: I0bef534eca71b9ce2b29927021b1977463ffbe74 --- swift/container/backend.py | 3 ++- test/unit/container/test_backend.py | 5 ++++- test/unit/container/test_sharder.py | 25 ++++++++++++++++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/swift/container/backend.py b/swift/container/backend.py index 132f9ed551..1130d828c2 100644 --- a/swift/container/backend.py +++ b/swift/container/backend.py @@ -379,7 +379,8 @@ class ContainerBroker(DatabaseBroker): db_contains_type = 'object' db_reclaim_timestamp = 'created_at' delete_meta_whitelist = ['x-container-sysmeta-shard-quoted-root', - 'x-container-sysmeta-shard-root'] + 'x-container-sysmeta-shard-root', + 'x-container-sysmeta-sharding'] def __init__(self, db_file, timeout=BROKER_TIMEOUT, logger=None, account=None, container=None, pending_timeout=None, diff --git a/test/unit/container/test_backend.py b/test/unit/container/test_backend.py index 0c614f4843..7e0a778723 100644 --- a/test/unit/container/test_backend.py +++ b/test/unit/container/test_backend.py @@ -1686,7 +1686,7 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(fresh_broker.get_db_state(), 'unsharded') @with_tempdir - def test_delete_db_does_not_clear_root_path(self, tempdir): + def test_delete_db_does_not_clear_particular_sharding_meta(self, tempdir): acct = '.sharded_a' cont = 'c' hsh = hash_path(acct, cont) @@ -1702,6 +1702,7 @@ class TestContainerBroker(unittest.TestCase): 'foo': ('bar', ts), 'icecream': ('sandwich', ts), 'X-Container-Sysmeta-Some': ('meta', ts), + 'X-Container-Sysmeta-Sharding': ('yes', ts), 'X-Container-Sysmeta-Shard-Quoted-Root': ('a/c', ts), 'X-Container-Sysmeta-Shard-Root': ('a/c', ts)}) @@ -1722,6 +1723,8 @@ class TestContainerBroker(unittest.TestCase): self.assertEqual(meta['X-Container-Sysmeta-Shard-Root'], ['a/c', ts]) self.assertEqual('a/c', broker.root_path) + self.assertEqual(meta['X-Container-Sysmeta-Sharding'], + ['yes', ts]) self.assertFalse(broker.is_root_container()) check_metadata(broker) diff --git a/test/unit/container/test_sharder.py b/test/unit/container/test_sharder.py index e76565b125..5e77c90714 100644 --- a/test/unit/container/test_sharder.py +++ b/test/unit/container/test_sharder.py @@ -4808,13 +4808,36 @@ class TestSharder(BaseTestSharder): def test_sharding_enabled(self): broker = self._make_broker() self.assertFalse(sharding_enabled(broker)) + # Setting sharding to a true value and sharding will be enabled broker.update_metadata( {'X-Container-Sysmeta-Sharding': ('yes', Timestamp.now().internal)}) self.assertTrue(sharding_enabled(broker)) - # deleting broker clears sharding sysmeta + + # deleting broker doesn't clear the Sysmeta-Sharding sysmeta broker.delete_db(Timestamp.now().internal) + self.assertTrue(sharding_enabled(broker)) + + # re-init the DB for the deleted tests + broker.set_storage_policy_index(0, Timestamp.now().internal) + broker.update_metadata( + {'X-Container-Sysmeta-Sharding': + ('yes', Timestamp.now().internal)}) + self.assertTrue(sharding_enabled(broker)) + + # if the Sysmeta-Sharding is falsy value then sharding isn't enabled + for value in ('', 'no', 'false', 'some_fish'): + broker.update_metadata( + {'X-Container-Sysmeta-Sharding': + (value, Timestamp.now().internal)}) + self.assertFalse(sharding_enabled(broker)) + # deleting broker doesn't clear the Sysmeta-Sharding sysmeta + broker.delete_db(Timestamp.now().internal) + self.assertEqual(broker.metadata['X-Container-Sysmeta-Sharding'][0], + 'some_fish') + # so it still isn't enabled (some_fish isn't a true value). self.assertFalse(sharding_enabled(broker)) + # but if broker has a shard range then sharding is enabled broker.merge_shard_ranges( ShardRange('acc/a_shard', Timestamp.now(), 'l', 'u'))