Merge "expirer: account and container level delay_reaping"

doc/source/config/object_server_config.rst

@@ -746,4 +746,19 @@ ionice_priority               None                            I/O scheduling pri
                                                               priority of the process. Work only with
                                                               ionice_class.
                                                               Ignored if IOPRIO_CLASS_IDLE is set.
+delay_reaping_<ACCT>           0.0                            A dynamic configuration option for
+                                                              setting account level delay_reaping values.
+                                                              The delay_reaping value is configured for
+                                                              the account with the name placed in
+                                                              <ACCT>. The object expirer will reap objects in
+                                                              this account from disk only after this delay
+                                                              following their x-delete-at time.
+delay_reaping_<ACCT>/<CNTR>    0.0                            A dynamic configuration option for
+                                                              setting container level delay_reaping values.
+                                                              The delay_reaping value is configured for
+                                                              the container with the account name placed
+                                                              in <ACCT> and the container name in <CNTR>.
+                                                              The object expirer will reap objects in this
+                                                              container from disk only after this delay
+                                                              following their x-delete-at time.
 ============================= =============================== ==========================================

doc/source/overview_expiring_objects.rst

@@ -55,6 +55,49 @@ it will then look for and use the ``/etc/swift/object-expirer.conf`` config.
 The latter config file is considered deprecated and is searched for to aid
 in cluster upgrades.
 
+Delay Reaping of Objects from Disk
+----------------------------------
+
+Swift's expiring object ``x-delete-at`` feature can be used to have the cluster
+reap user's objects automatically from disk on their behalf when they no longer
+want them stored in their account. In some cases it may be necessary to
+"intervene" in the expected expiration process to prevent accidental or
+premature data loss if an object marked for expiration should NOT be deleted
+immediately when it expires for whatever reason. In these cases
+``swift-object-expirer`` offers configuration of a ``delay_reaping`` value
+on accounts and containers, which provides a delay between when an object
+is marked for deletion, or expired, and when it is actually reaped from disk.
+When this is set in the object expirer config the object expirer leaves expired
+objects on disk (and in container listings) for the ``delay_reaping`` time.
+After this delay has passed objects will be reaped as normal.
+
+The ``delay_reaping`` value can be set either at an account level or a
+container level. When set at an account level, the object expirer will
+only reap objects within the account after the delay. A container level
+``delay_reaping`` works similarly for containers and overrides an account
+level ``delay_reaping`` value.
+
+The ``delay_reaping`` values are set in the ``[object-expirer]`` section in
+either the object-server or object-expirer config files. They are configured
+with dynamic config option names prefixed with ``delay_reaping_<ACCT>``
+at the account level and ``delay_reaping_<ACCT>/<CNTR>`` at the container
+level, with the ``delay_reaping`` value in seconds.
+
+Here is an example of ``delay_reaping`` configs in the``object-expirer``
+section in the ``object-server.conf``::
+
+    [object-expirer]
+    delay_reaping_AUTH_test = 300.0
+    delay_reaping_AUTH_test2 = 86400.0
+    delay_reaping_AUTH_test/test = 0.0
+    delay_reaping_AUTH_test/test2 = 600.0
+
+.. note::
+    A container level ``delay_reaping`` value does not require an account level
+    ``delay_reaping`` value but overrides the account level value for the same
+    account if it exists. By default, no ``delay_reaping`` value is configured
+    for any accounts or containers.
+
 Upgrading impact: General Task Queue vs Legacy Queue
 ----------------------------------------------------
 

etc/object-expirer.conf-sample

@@ -72,6 +72,27 @@
 # queue.
 # reclaim_age = 604800
 #
+# The expirer can delay the reaping of expired objects on disk (and in
+# container listings) with an account level or container level delay_reaping
+# time.
+# After the delay_reaping time has passed objects will be reaped as normal.
+# You may configure this delay_reaping value in seconds with dynamic config
+# option names prefixed with delay_reaping_<ACCT> for account level delays
+# and delay_reaping_<ACCT>/<CNTR> for container level delays.
+# Special characters in <ACCT> or <CNTR> should be quoted.
+# The delay_reaping value should be a float value greater than or equal to
+# zero.
+# A container level delay_reaping does not require an account level
+# delay_reaping but overrides the account level delay_reaping for the same
+# account if it exists.
+# For example:
+# delay_reaping_AUTH_test = 300.0
+# delay_reaping_AUTH_test2 = 86400.0
+# delay_reaping_AUTH_test/test = 400.0
+# delay_reaping_AUTH_test/test2 = 600.0
+# N.B. By default no delay_reaping value is configured for any accounts or
+# containers.
+#
 # recon_cache_path = /var/cache/swift
 #
 # You can set scheduling priority of processes. Niceness values range from -20

etc/object-server.conf-sample

@@ -692,10 +692,31 @@ use = egg:swift#backend_ratelimit
 # ionice_class =
 # ionice_priority =
 #
-# Note: Put it at the beginning of the pipleline to profile all middleware. But
-# it is safer to put this after healthcheck.
+# The expirer can delay the reaping of expired objects on disk (and in
+# container listings) with an account level or container level delay_reaping
+# time.
+# After the delay_reaping time has passed objects will be reaped as normal.
+# You may configure this delay_reaping value in seconds with dynamic config
+# option names prefixed with delay_reaping_<ACCT> for account level delays
+# and delay_reaping_<ACCT>/<CNTR> for container level delays.
+# Special characters in <ACCT> or <CNTR> should be quoted.
+# The delay_reaping value should be a float value greater than or equal to
+# zero.
+# A container level delay_reaping does not require an account level
+# delay_reaping but overrides the account level delay_reaping for the same
+# account if it exists.
+# For example:
+# delay_reaping_AUTH_test = 300.0
+# delay_reaping_AUTH_test2 = 86400.0
+# delay_reaping_AUTH_test/test = 400.0
+# delay_reaping_AUTH_test/test2 = 600.0
+# N.B. By default no delay_reaping value is configured for any accounts or
+# containers.
+
 [filter:xprofile]
 use = egg:swift#xprofile
+# Note: Put it at the beginning of the pipleline to profile all middleware. But
+# it is safer to put this after healthcheck.
 # This option enable you to switch profilers which should inherit from python
 # standard profiler. Currently the supported value can be 'cProfile',
 # 'eventlet.green.profile' etc.

swift/obj/expirer.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 import six
+from six.moves import urllib
 
 from random import random
 from time import time
@@ -28,7 +29,7 @@ from swift.common.daemon import Daemon
 from swift.common.internal_client import InternalClient, UnexpectedResponse
 from swift.common.utils import get_logger, dump_recon_cache, split_path, \
     Timestamp, config_true_value, normalize_delete_at_timestamp, \
-    RateLimitedIterator, md5
+    RateLimitedIterator, md5, non_negative_float
 from swift.common.http import HTTP_NOT_FOUND, HTTP_CONFLICT, \
     HTTP_PRECONDITION_FAILED
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
@@ -66,6 +67,49 @@ def parse_task_obj(task_obj):
     return timestamp, target_account, target_container, target_obj
 
 
+def read_conf_for_delay_reaping_times(conf):
+    delay_reaping_times = {}
+    for conf_key in conf:
+        delay_reaping_prefix = "delay_reaping_"
+        if not conf_key.startswith(delay_reaping_prefix):
+            continue
+        delay_reaping_key = urllib.parse.unquote(
+            conf_key[len(delay_reaping_prefix):])
+        if delay_reaping_key.strip('/') != delay_reaping_key:
+            raise ValueError(
+                '%s '
+                'should be in the form delay_reaping_<account> '
+                'or delay_reaping_<account>/<container> '
+                '(leading or trailing "/" is not allowed)' % conf_key)
+        try:
+            # If split_path fails, have multiple '/' or
+            # account name is invalid
+            account, container = split_path(
+                '/' + delay_reaping_key, 1, 2
+            )
+        except ValueError:
+            raise ValueError(
+                '%s '
+                'should be in the form delay_reaping_<account> '
+                'or delay_reaping_<account>/<container> '
+                '(at most one "/" is allowed)' % conf_key)
+        try:
+            delay_reaping_times[(account, container)] = non_negative_float(
+                conf.get(conf_key)
+            )
+        except ValueError:
+            raise ValueError(
+                '%s must be a float '
+                'greater than or equal to 0' % conf_key)
+    return delay_reaping_times
+
+
+def get_delay_reaping(delay_reaping_times, target_account, target_container):
+    return delay_reaping_times.get(
+        (target_account, target_container),
+        delay_reaping_times.get((target_account, None), 0.0))
+
+
 class ObjectExpirer(Daemon):
     """
     Daemon that queries the internal hidden task accounts to discover objects
@@ -113,6 +157,8 @@ class ObjectExpirer(Daemon):
         # with the tombstone reclaim age in the consistency engine.
         self.reclaim_age = int(conf.get('reclaim_age', 604800))
 
+        self.delay_reaping_times = read_conf_for_delay_reaping_times(conf)
+
     def read_conf_for_queue_access(self, swift):
         self.expiring_objects_account = AUTO_CREATE_ACCOUNT_PREFIX + \
             (self.conf.get('expiring_objects_account_name') or
@@ -246,6 +292,10 @@ class ObjectExpirer(Daemon):
                 break
             yield task_container
 
+    def get_delay_reaping(self, target_account, target_container):
+        return get_delay_reaping(self.delay_reaping_times, target_account,
+                                 target_container)
+
     def iter_task_to_expire(self, task_account_container_list,
                             my_index, divisor):
         """
@@ -267,17 +317,24 @@ class ObjectExpirer(Daemon):
                     self.logger.exception('Unexcepted error handling task %r' %
                                           task_object)
                     continue
+                is_async = o.get('content_type') == ASYNC_DELETE_TYPE
+                delay_reaping = self.get_delay_reaping(target_account,
+                                                       target_container)
+
                 if delete_timestamp > Timestamp.now():
-                    # we shouldn't yield the object that doesn't reach
+                    # we shouldn't yield ANY more objects that can't reach
                     # the expiration date yet.
                     break
+                if delete_timestamp > Timestamp(time() - delay_reaping) \
+                        and not is_async:
+                    # we shouldn't yield the object during the delay
+                    continue
 
                 # Only one expirer daemon assigned for one task
                 if self.hash_mod('%s/%s' % (task_container, task_object),
                                  divisor) != my_index:
                     continue
 
-                is_async = o.get('content_type') == ASYNC_DELETE_TYPE
                 yield {'task_account': task_account,
                        'task_container': task_container,
                        'task_object': task_object,

test/unit/obj/test_expirer.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright (c) 2011 OpenStack Foundation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -284,6 +285,221 @@ class TestObjectExpirer(TestCase):
             x.get_process_values(vals)
         self.assertEqual(str(ctx.exception), expected_msg)
 
+    def test_valid_delay_reaping(self):
+        conf = {}
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(x.delay_reaping_times, {})
+
+        conf = {
+            'delay_reaping_a': 1.0,
+        }
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(x.delay_reaping_times, {('a', None): 1.0})
+
+        # allow delay_reaping to be 0
+        conf = {
+            'delay_reaping_a': 0.0,
+        }
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(x.delay_reaping_times, {('a', None): 0.0})
+
+        conf = {
+            'delay_reaping_a/b': 0.0,
+        }
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(x.delay_reaping_times, {('a', 'b'): 0.0})
+
+        # test configure multi-account delay_reaping
+        conf = {
+            'delay_reaping_a': 1.0,
+            'delay_reaping_b': '259200.0',
+            'delay_reaping_AUTH_aBC': 999,
+            u'delay_reaping_AUTH_aBáC': 555,
+        }
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(x.delay_reaping_times, {
+            ('a', None): 1.0,
+            ('b', None): 259200.0,
+            ('AUTH_aBC', None): 999,
+            (u'AUTH_aBáC', None): 555,
+        })
+
+        # test configure multi-account delay_reaping with containers
+        conf = {
+            'delay_reaping_a': 10.0,
+            'delay_reaping_a/test': 1.0,
+            'delay_reaping_b': '259200.0',
+            'delay_reaping_AUTH_aBC/test2': 999,
+            u'delay_reaping_AUTH_aBáC/tést': 555,
+            'delay_reaping_AUTH_test/special%0Achars%3Dare%20quoted': 777,
+            'delay_reaping_AUTH_test/plus+signs+are+preserved': 888,
+        }
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(x.delay_reaping_times, {
+            ('a', None): 10.0,
+            ('a', 'test'): 1.0,
+            ('b', None): 259200.0,
+            ('AUTH_aBC', 'test2'): 999,
+            (u'AUTH_aBáC', u'tést'): 555,
+            ('AUTH_test', 'special\nchars=are quoted'): 777,
+            ('AUTH_test', 'plus+signs+are+preserved'): 888,
+        })
+
+    def test_invalid_delay_reaping_keys(self):
+        # there is no global delay_reaping
+        conf = {
+            'delay_reaping': 0.0,
+        }
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(x.delay_reaping_times, {})
+
+        # Multiple "/" or invalid parsing
+        conf = {
+            'delay_reaping_A_U_TH_foo_bar/my-container_name/with/slash': 60400,
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_A_U_TH_foo_bar/my-container_name/with/slash '
+            'should be in the form delay_reaping_<account> '
+            'or delay_reaping_<account>/<container> '
+            '(at most one "/" is allowed)',
+            str(ctx.exception))
+
+        # Can't sneak around it by escaping
+        conf = {
+            'delay_reaping_AUTH_test/sneaky%2fsneaky': 60400,
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_AUTH_test/sneaky%2fsneaky '
+            'should be in the form delay_reaping_<account> '
+            'or delay_reaping_<account>/<container> '
+            '(at most one "/" is allowed)',
+            str(ctx.exception))
+
+        conf = {
+            'delay_reaping_': 60400
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_ '
+            'should be in the form delay_reaping_<account> '
+            'or delay_reaping_<account>/<container> '
+            '(at most one "/" is allowed)',
+            str(ctx.exception))
+
+        # Leading and trailing "/"
+        conf = {
+            'delay_reaping_/a': 60400,
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_/a '
+            'should be in the form delay_reaping_<account> '
+            'or delay_reaping_<account>/<container> '
+            '(leading or trailing "/" is not allowed)',
+            str(ctx.exception))
+
+        conf = {
+            'delay_reaping_a/': 60400,
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_a/ '
+            'should be in the form delay_reaping_<account> '
+            'or delay_reaping_<account>/<container> '
+            '(leading or trailing "/" is not allowed)',
+            str(ctx.exception))
+
+        conf = {
+            'delay_reaping_/a/c/': 60400,
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_/a/c/ '
+            'should be in the form delay_reaping_<account> '
+            'or delay_reaping_<account>/<container> '
+            '(leading or trailing "/" is not allowed)',
+            str(ctx.exception))
+
+    def test_invalid_delay_reaping_values(self):
+        # negative tests
+        conf = {
+            'delay_reaping_a': -1.0,
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_a must be a float greater than or equal to 0',
+            str(ctx.exception))
+        conf = {
+            'delay_reaping_a': '-259200.0'
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_a must be a float greater than or equal to 0',
+            str(ctx.exception))
+        conf = {
+            'delay_reaping_a': 'foo'
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_a must be a float greater than or equal to 0',
+            str(ctx.exception))
+
+        # negative tests with containers
+        conf = {
+            'delay_reaping_a/b': -100.0
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_a/b must be a float greater than or equal to 0',
+            str(ctx.exception))
+        conf = {
+            'delay_reaping_a/b': '-259200.0'
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_a/b must be a float greater than or equal to 0',
+            str(ctx.exception))
+        conf = {
+            'delay_reaping_a/b': 'foo'
+        }
+        with self.assertRaises(ValueError) as ctx:
+            expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(
+            'delay_reaping_a/b must be a float greater than or equal to 0',
+            str(ctx.exception))
+
+    def test_get_delay_reaping(self):
+        conf = {
+            'delay_reaping_a': 1.0,
+            'delay_reaping_a/test': 2.0,
+            'delay_reaping_b': '259200.0',
+            'delay_reaping_b/a': '0.0',
+            'delay_reaping_c/test': '3.0'
+        }
+        x = expirer.ObjectExpirer(conf, swift=self.fake_swift)
+        self.assertEqual(1.0, x.get_delay_reaping('a', None))
+        self.assertEqual(1.0, x.get_delay_reaping('a', 'not-test'))
+        self.assertEqual(2.0, x.get_delay_reaping('a', 'test'))
+        self.assertEqual(259200.0, x.get_delay_reaping('b', None))
+        self.assertEqual(0.0, x.get_delay_reaping('b', 'a'))
+        self.assertEqual(259200.0, x.get_delay_reaping('b', 'test'))
+        self.assertEqual(3.0, x.get_delay_reaping('c', 'test'))
+        self.assertEqual(0.0, x.get_delay_reaping('c', 'not-test'))
+        self.assertEqual(0.0, x.get_delay_reaping('no-conf', 'test'))
+
     def test_init_concurrency_too_small(self):
         conf = {
             'concurrency': 0,
@@ -746,6 +962,233 @@ class TestObjectExpirer(TestCase):
                 task_account_container_list, my_index, divisor)),
             expected)
 
+    def test_iter_task_to_expire_with_delay_reaping(self):
+        aco_dict = {
+            '.expiring_objects': {
+                self.past_time: [
+                    # tasks well past ready for execution
+                    {'name': self.past_time + '-a0/c0/o0'},
+                    {'name': self.past_time + '-a1/c1/o1'},
+                    {'name': self.past_time + '-a1/c2/o2'},
+                ],
+                self.just_past_time: [
+                    # tasks only just ready for execution
+                    {'name': self.just_past_time + '-a0/c0/o0'},
+                    {'name': self.just_past_time + '-a1/c1/o1'},
+                    {'name': self.just_past_time + '-a1/c2/o2'},
+                ],
+                self.future_time: [
+                    # tasks not yet ready for execution
+                    {'name': self.future_time + '-a0/c0/o0'},
+                    {'name': self.future_time + '-a1/c1/o1'},
+                    {'name': self.future_time + '-a1/c2/o2'},
+                ],
+            }
+        }
+        fake_swift = FakeInternalClient(aco_dict)
+        # sanity, no accounts configured with delay_reaping
+        x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+                                  swift=fake_swift)
+        # ... we expect tasks past time to yield
+        expected = [
+            self.make_task(self.past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                    'a1/c2/o2',
+                )
+            )
+        ] + [
+            self.make_task(self.just_past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                    'a1/c2/o2',
+                )
+            )
+        ]
+        task_account_container_list = [
+            ('.expiring_objects', self.past_time),
+            ('.expiring_objects', self.just_past_time),
+        ]
+        observed = list(x.iter_task_to_expire(
+            task_account_container_list, 0, 1))
+        self.assertEqual(expected, observed)
+
+        # configure delay for account a1
+        self.conf['delay_reaping_a1'] = 300.0
+        x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+                                  swift=fake_swift)
+        # ... and we don't expect *recent* a1 tasks or future tasks
+        expected = [
+            self.make_task(self.past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                    'a1/c2/o2',
+                )
+            )
+        ] + [
+            self.make_task(self.just_past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                )
+            )
+        ]
+        observed = list(x.iter_task_to_expire(
+            task_account_container_list, 0, 1))
+        self.assertEqual(expected, observed)
+
+        # configure delay for account a1 and for account a1 and container c2
+        # container a1/c2 expires expires almost immediately
+        # but other containers in account a1 remain (a1/c1 and a1/c3)
+        self.conf['delay_reaping_a1'] = 300.0
+        self.conf['delay_reaping_a1/c2'] = 0.1
+        x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+                                  swift=fake_swift)
+        # ... and we don't expect *recent* a1 tasks, excluding c2
+        # or future tasks
+        expected = [
+            self.make_task(self.past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                    'a1/c2/o2',
+                )
+            )
+        ] + [
+            self.make_task(self.just_past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c2/o2',
+                )
+            )
+        ]
+        observed = list(x.iter_task_to_expire(
+            task_account_container_list, 0, 1))
+        self.assertEqual(expected, observed)
+
+        # configure delay for account a1 and for account a1 and container c2
+        # container a1/c2 does not expire but others in account a1 do
+        self.conf['delay_reaping_a1'] = 0.1
+        self.conf['delay_reaping_a1/c2'] = 300.0
+        x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+                                  swift=fake_swift)
+        # ... and we don't expect *recent* a1 tasks, excluding c2
+        # or future tasks
+        expected = [
+            self.make_task(self.past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                    'a1/c2/o2',
+                )
+            )
+        ] + [
+            self.make_task(self.just_past_time, target_path)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                )
+            )
+        ]
+        observed = list(x.iter_task_to_expire(
+            task_account_container_list, 0, 1))
+        self.assertEqual(expected, observed)
+
+    def test_iter_task_to_expire_with_delay_reaping_is_async(self):
+        aco_dict = {
+            '.expiring_objects': {
+                self.past_time: [
+                    # tasks well past ready for execution
+                    {'name': self.past_time + '-a0/c0/o0',
+                     'content_type': 'application/async-deleted'},
+                    {'name': self.past_time + '-a1/c1/o1',
+                     'content_type': 'application/async-deleted'},
+                    {'name': self.past_time + '-a1/c2/o2',
+                     'content_type': 'application/async-deleted'},
+                ],
+                self.just_past_time: [
+                    # tasks only just ready for execution
+                    {'name': self.just_past_time + '-a0/c0/o0',
+                     'content_type': 'application/async-deleted'},
+                    {'name': self.just_past_time + '-a1/c1/o1',
+                     'content_type': 'application/async-deleted'},
+                    {'name': self.just_past_time + '-a1/c2/o2',
+                     'content_type': 'application/async-deleted'},
+                ],
+                self.future_time: [
+                    # tasks not yet ready for execution
+                    {'name': self.future_time + '-a0/c0/o0',
+                     'content_type': 'application/async-deleted'},
+                    {'name': self.future_time + '-a1/c1/o1',
+                     'content_type': 'application/async-deleted'},
+                    {'name': self.future_time + '-a1/c2/o2',
+                     'content_type': 'application/async-deleted'},
+                ],
+            }
+        }
+        fake_swift = FakeInternalClient(aco_dict)
+        # no accounts configured with delay_reaping
+        x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+                                  swift=fake_swift)
+        # ... we expect all past async tasks to yield
+        expected = [
+            self.make_task(self.past_time, target_path, is_async_delete=True)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                    'a1/c2/o2',
+                )
+            )
+        ] + [
+            self.make_task(self.just_past_time, target_path,
+                           is_async_delete=True)
+            for target_path in (
+                swob.wsgi_to_str(tgt) for tgt in (
+                    'a0/c0/o0',
+                    'a1/c1/o1',
+                    'a1/c2/o2',
+                )
+            )
+        ]
+        task_account_container_list = [
+            ('.expiring_objects', self.past_time),
+            ('.expiring_objects', self.just_past_time),
+        ]
+        observed = list(x.iter_task_to_expire(
+            task_account_container_list, 0, 1))
+        self.assertEqual(expected, observed)
+
+        # configure delay for account a1
+        self.conf['delay_reaping_a1'] = 300.0
+        x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+                                  swift=fake_swift)
+        # ... and we still expect all past async tasks to yield
+        observed = list(x.iter_task_to_expire(
+            task_account_container_list, 0, 1))
+        self.assertEqual(expected, observed)
+
+        # configure delays for all containers
+        self.conf['delay_reaping_a1/c0'] = 300.0
+        self.conf['delay_reaping_a1/c1'] = 300.0
+        self.conf['delay_reaping_a1/c2'] = 300.0
+        x = expirer.ObjectExpirer(self.conf, logger=self.logger,
+                                  swift=fake_swift)
+        # ... and we we still expect all past async tasks to yield
+        observed = list(x.iter_task_to_expire(
+            task_account_container_list, 0, 1))
+        self.assertEqual(expected, observed)
+
     def test_run_once_unicode_problem(self):
         requests = []