Merge "Test current reclaim_age handling"

test/unit/obj/test_diskfile.py

@@ -658,6 +658,42 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
                                    policy=policy, frag_index=frag_index,
                                    **kwargs)
 
+    def test_cleanup_uses_configured_reclaim_age(self):
+        # verify that the reclaim_age used when cleaning up tombstones is
+        # either the default or the configured value
+        def do_test(ts, expect_reclaim):
+            for policy in POLICIES:
+                self.df_router = diskfile.DiskFileRouter(
+                    self.conf, self.logger)
+                df = self._get_diskfile(policy)
+                df.delete(ts.internal)
+                suffix_dir = os.path.dirname(df._datadir)
+                part_dir = os.path.dirname(suffix_dir)
+                tombstone_file = os.path.join(df._datadir, ts.internal + '.ts')
+                # direct callers of this this method are expect to plumb
+                # reclaim_age as a kwarg
+                df._manager.cleanup_ondisk_files(
+                    os.path.dirname(tombstone_file))
+                # the default is ONE_WEEK
+                if time() - float(ts) > diskfile.ONE_WEEK:
+                    self.assertFalse(os.path.exists(tombstone_file))
+                else:
+                    self.assertTrue(os.path.exists(tombstone_file))
+                # the configured value is plumbed through on REPLICATE calls
+                df._manager._get_hashes(part_dir)
+                self.assertNotEqual(
+                    expect_reclaim, os.path.exists(tombstone_file))
+
+        # reclaim_age not configured so default should be used
+        do_test(Timestamp(time() - diskfile.ONE_WEEK - 1), True)
+        do_test(Timestamp(time() - diskfile.ONE_WEEK + 100), False)
+
+        # reclaim_age configured value should be used
+        self.conf['reclaim_age'] = 1000
+        do_test(Timestamp(time() - diskfile.ONE_WEEK + 100), True)
+        do_test(Timestamp(time() - 1001), True)
+        do_test(Timestamp(time() + 100), False)
+
     def _test_get_ondisk_files(self, scenarios, policy,
                                frag_index=None, **kwargs):
         class_under_test = self._get_diskfile(

test/unit/obj/test_server.py

@@ -35,6 +35,7 @@ from hashlib import md5
 import tempfile
 from collections import defaultdict
 from contextlib import contextmanager
+from textwrap import dedent
 
 from eventlet import sleep, spawn, wsgi, listen, Timeout, tpool, greenthread
 from eventlet.green import httplib
@@ -62,6 +63,7 @@ from swift.common.storage_policy import (StoragePolicy, ECStoragePolicy,
                                          POLICIES, EC_POLICY)
 from swift.common.exceptions import DiskFileDeviceUnavailable, \
     DiskFileNoSpace, DiskFileQuarantined
+from swift.common.wsgi import init_request_processor
 
 
 def mock_time(*args, **kwargs):
@@ -134,6 +136,11 @@ class TestObjectController(unittest.TestCase):
         mkdirs(os.path.join(self.testdir, 'sda1',
                             diskfile.get_tmp_dir(policy)))
 
+    def iter_policies(self):
+        for policy in POLICIES:
+            self.policy = policy
+            yield policy
+
     def check_all_api_methods(self, obj_name='o', alt_res=None):
         path = '/sda1/p/a/c/%s' % obj_name
         body = 'SPECIAL_STRING'
@@ -6164,6 +6171,70 @@ class TestObjectController(unittest.TestCase):
             resp = req.get_response(self.object_controller)
         self.assertEqual(resp.status_int, 507)
 
+    def test_REPLICATE_reclaims_tombstones(self):
+        conf = {'devices': self.testdir, 'mount_check': False,
+                'reclaim_age': 100}
+        self.object_controller = object_server.ObjectController(
+            conf, logger=self.logger)
+        for policy in self.iter_policies():
+            # create a tombstone
+            ts = next(self.ts)
+            delete_request = Request.blank(
+                '/sda1/0/a/c/o', method='DELETE',
+                headers={
+                    'x-backend-storage-policy-index': int(policy),
+                    'x-timestamp': ts.internal,
+                })
+            resp = delete_request.get_response(self.object_controller)
+            self.assertEqual(resp.status_int, 404)
+            objfile = self.df_mgr.get_diskfile('sda1', '0', 'a', 'c', 'o',
+                                               policy=policy)
+            tombstone_file = os.path.join(objfile._datadir,
+                                          '%s.ts' % ts.internal)
+            self.assertTrue(os.path.exists(tombstone_file))
+
+            # REPLICATE will hash it
+            req = Request.blank(
+                '/sda1/0', method='REPLICATE',
+                headers={
+                    'x-backend-storage-policy-index': int(policy),
+                })
+            resp = req.get_response(self.object_controller)
+            self.assertEqual(resp.status_int, 200)
+            suffix = pickle.loads(resp.body).keys()[0]
+            self.assertEqual(suffix, os.path.basename(
+                os.path.dirname(objfile._datadir)))
+            # tombsone still exists
+            self.assertTrue(os.path.exists(tombstone_file))
+
+            # after reclaim REPLICATE will rehash
+            replicate_request = Request.blank(
+                '/sda1/0/%s' % suffix, method='REPLICATE',
+                headers={
+                    'x-backend-storage-policy-index': int(policy),
+                })
+            the_future = time() + 200
+            with mock.patch('swift.obj.diskfile.time.time') as mock_time:
+                mock_time.return_value = the_future
+                resp = replicate_request.get_response(self.object_controller)
+            self.assertEqual(resp.status_int, 200)
+            self.assertEqual({}, pickle.loads(resp.body))
+            # and tombsone is reaped!
+            self.assertFalse(os.path.exists(tombstone_file))
+
+            # N.B. with a small reclaim age like this - if proxy clocks get far
+            # enough out of whack ...
+            with mock.patch('swift.obj.diskfile.time.time') as mock_time:
+                mock_time.return_value = the_future
+                resp = delete_request.get_response(self.object_controller)
+                # we'll still create the tombstone
+                self.assertTrue(os.path.exists(tombstone_file))
+                # but it will get reaped by REPLICATE
+                resp = replicate_request.get_response(self.object_controller)
+                self.assertEqual(resp.status_int, 200)
+                self.assertEqual({}, pickle.loads(resp.body))
+                self.assertFalse(os.path.exists(tombstone_file))
+
     def test_SSYNC_can_be_called(self):
         req = Request.blank('/sda1/0',
                             environ={'REQUEST_METHOD': 'SSYNC'},
@@ -7539,5 +7610,104 @@ class TestZeroCopy(unittest.TestCase):
         self.assertEqual(contents, '')
 
 
+class TestConfigOptionHandling(unittest.TestCase):
+
+    def setUp(self):
+        self.tmpdir = mkdtemp()
+
+    def tearDown(self):
+        rmtree(self.tmpdir)
+
+    def _app_config(self, config):
+        contents = dedent(config)
+        conf_file = os.path.join(self.tmpdir, 'object-server.conf')
+        with open(conf_file, 'w') as f:
+            f.write(contents)
+        with mock.patch('swift.common.wsgi.monkey_patch_mimetools'):
+            app = init_request_processor(conf_file, 'object-server')[:2]
+        return app
+
+    def test_default(self):
+        config = """
+        [DEFAULT]
+
+        [pipeline:main]
+        pipeline = object-server
+
+        [app:object-server]
+        use = egg:swift#object
+        """
+        app, config = self._app_config(config)
+        self.assertNotIn('reclaim_age', config)
+        self.assertEqual(app._diskfile_router[POLICIES.legacy].reclaim_age,
+                         604800)
+
+    def test_option_in_app(self):
+        config = """
+        [DEFAULT]
+
+        [pipeline:main]
+        pipeline = object-server
+
+        [app:object-server]
+        use = egg:swift#object
+        reclaim_age = 100
+        """
+        app, config = self._app_config(config)
+        self.assertEqual(config['reclaim_age'], '100')
+        self.assertEqual(app._diskfile_router[POLICIES.legacy].reclaim_age,
+                         100)
+
+    def test_option_in_default(self):
+        config = """
+        [DEFAULT]
+        reclaim_age = 200
+
+        [pipeline:main]
+        pipeline = object-server
+
+        [app:object-server]
+        use = egg:swift#object
+        """
+        app, config = self._app_config(config)
+        self.assertEqual(config['reclaim_age'], '200')
+        self.assertEqual(app._diskfile_router[POLICIES.legacy].reclaim_age,
+                         200)
+
+    def test_option_in_both(self):
+        config = """
+        [DEFAULT]
+        reclaim_age = 300
+
+        [pipeline:main]
+        pipeline = object-server
+
+        [app:object-server]
+        use = egg:swift#object
+        reclaim_age = 400
+        """
+        app, config = self._app_config(config)
+        self.assertEqual(config['reclaim_age'], '300')
+        self.assertEqual(app._diskfile_router[POLICIES.legacy].reclaim_age,
+                         300)
+
+        # use paste "set" syntax to override global config value
+        config = """
+        [DEFAULT]
+        reclaim_age = 500
+
+        [pipeline:main]
+        pipeline = object-server
+
+        [app:object-server]
+        use = egg:swift#object
+        set reclaim_age = 600
+        """
+        app, config = self._app_config(config)
+        self.assertEqual(config['reclaim_age'], '600')
+        self.assertEqual(app._diskfile_router[POLICIES.legacy].reclaim_age,
+                         600)
+
+
 if __name__ == '__main__':
     unittest.main()