Merge "Added a test for delayed reap."

test/probe/test_account_reaper.py

@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from time import sleep
 import uuid
 import unittest
 
 from swiftclient import client
 
-from swift.common.storage_policy import POLICIES
+from swift.account import reaper
+from swift.common import utils
 from swift.common.manager import Manager
 from swift.common.direct_client import direct_delete_account, \
     direct_get_object, direct_head_container, ClientException
@@ -25,18 +27,19 @@ from test.probe.common import ReplProbeTest, ENABLED_POLICIES
 
 
 class TestAccountReaper(ReplProbeTest):
-
+    def setUp(self):
-    def test_sync(self):
+        super(TestAccountReaper, self).setUp()
-        all_objects = []
+        self.all_objects = []
         # upload some containers
+        body = 'test-body'
         for policy in ENABLED_POLICIES:
             container = 'container-%s-%s' % (policy.name, uuid.uuid4())
             client.put_container(self.url, self.token, container,
                                  headers={'X-Storage-Policy': policy.name})
             obj = 'object-%s' % uuid.uuid4()
-            body = 'test-body'
             client.put_object(self.url, self.token, container, obj, body)
-            all_objects.append((policy, container, obj))
+            self.all_objects.append((policy, container, obj))
+            policy.load_ring('/etc/swift')
 
         Manager(['container-updater']).once()
 
@@ -50,13 +53,18 @@ class TestAccountReaper(ReplProbeTest):
                          len(ENABLED_POLICIES) * len(body))
 
         part, nodes = self.account_ring.get_nodes(self.account)
+
         for node in nodes:
             direct_delete_account(node, part, self.account)
 
+    def test_sync(self):
         # run the reaper
         Manager(['account-reaper']).once()
 
-        for policy, container, obj in all_objects:
+        self._verify_account_reaped()
+
+    def _verify_account_reaped(self):
+        for policy, container, obj in self.all_objects:
             # verify that any container deletes were at same timestamp
             cpart, cnodes = self.container_ring.get_nodes(
                 self.account, container)
@@ -72,7 +80,6 @@ class TestAccountReaper(ReplProbeTest):
                     # 'X-Backend-DELETE-Timestamp' confirms it was deleted
                     self.assertTrue(delete_time)
                     delete_times.add(delete_time)
-
                 else:
                     # Container replicas may not yet be deleted if we have a
                     # policy with object replicas < container replicas, so
@@ -82,8 +89,8 @@ class TestAccountReaper(ReplProbeTest):
             self.assertEqual(1, len(delete_times), delete_times)
 
             # verify that all object deletes were at same timestamp
-            object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
+            part, nodes = policy.object_ring.get_nodes(self.account,
-            part, nodes = object_ring.get_nodes(self.account, container, obj)
+                                                       container, obj)
             headers = {'X-Backend-Storage-Policy-Index': int(policy)}
             delete_times = set()
             for node in nodes:
@@ -104,7 +111,7 @@ class TestAccountReaper(ReplProbeTest):
         # run replicators and updaters
         self.get_to_final_state()
 
-        for policy, container, obj in all_objects:
+        for policy, container, obj in self.all_objects:
             # verify that ALL container replicas are now deleted
             cpart, cnodes = self.container_ring.get_nodes(
                 self.account, container)
@@ -123,10 +130,11 @@ class TestAccountReaper(ReplProbeTest):
                 else:
                     self.fail('Found un-reaped /%s/%s on %r' %
                               (self.account, container, cnode))
+            self.assertEqual(1, len(delete_times))
 
             # sanity check that object state is still consistent...
-            object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
+            part, nodes = policy.object_ring.get_nodes(self.account,
-            part, nodes = object_ring.get_nodes(self.account, container, obj)
+                                                       container, obj)
             headers = {'X-Backend-Storage-Policy-Index': int(policy)}
             delete_times = set()
             for node in nodes:
@@ -144,6 +152,47 @@ class TestAccountReaper(ReplProbeTest):
                               (self.account, container, obj, node, policy))
             self.assertEqual(1, len(delete_times))
 
+    def test_delayed_reap(self):
+        # define reapers which are supposed to operate 3 seconds later
+        account_reapers = []
+        for conf_file in self.configs['account-server'].values():
+                conf = utils.readconf(conf_file, 'account-reaper')
+                conf['delay_reaping'] = '3'
+                account_reapers.append(reaper.AccountReaper(conf))
+
+        self.assertTrue(account_reapers)
+
+        # run reaper, and make sure that nothing is reaped
+        for account_reaper in account_reapers:
+            account_reaper.run_once()
+
+        for policy, container, obj in self.all_objects:
+            cpart, cnodes = self.container_ring.get_nodes(
+                self.account, container)
+            for cnode in cnodes:
+                try:
+                    direct_head_container(cnode, cpart, self.account,
+                                          container)
+                except ClientException:
+                    self.fail("Nothing should be reaped. Object should exist")
+
+            part, nodes = policy.object_ring.get_nodes(self.account,
+                                                       container, obj)
+            headers = {'X-Backend-Storage-Policy-Index': int(policy)}
+            for node in nodes:
+                try:
+                    direct_get_object(node, part, self.account,
+                                      container, obj, headers=headers)
+                except ClientException:
+                    self.fail("Nothing should be reaped. Object should exist")
+
+        # wait 3 seconds, run reaper, and make sure that all is reaped
+        sleep(3)
+        for account_reaper in account_reapers:
+            account_reaper.run_once()
+
+        self._verify_account_reaped()
+
 
 if __name__ == "__main__":
     unittest.main()