Add container and account reverse listings

This change adds the ability to tell the container or account server to
reverse their listings. This is done by sending a reverse=TRUE_VALUE,

Where TRUE_VALUE is one of the values true can be in common/utils:

  TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))

For example:

  curl -i -X GET -H "X-Auth-Token: $TOKEN" $STORAGE_URL/c/?reverse=on

I borrowed the swapping of the markers code from Kevin's old change,
thanks Kevin. And Tim Burke added some real nuggets of awesomeness.

DocImpact
Co-Authored-By: Kevin McDonald <kmcdonald@softlayer.com>
Co-Authored-By: Tim Burke <tim.burke@gmail.com>
Implements: blueprint reverse-object-listing

Change-Id: I5eb655360ac95042877da26d18707aebc11c02f6

doc/source/api/object_api_v1_overview.rst

@@ -128,7 +128,24 @@ If you have a large number of containers or objects, you can use query
 parameters to page through large lists of containers or objects. Use the
 *``marker``*, *``limit``*, and *``end_marker``* query parameters to
 control how many items are returned in a list and where the list starts
-or ends.
+or ends. If you want to page through in reverse order, you can use the query
+parameter *``reverse``*, noting that your marker and end_markers will be
+applied to a reverse listing should be switched. I.e, for a list of objects
+``[a, b, c, d, e]`` the non-reversed could be:
+
+.. code::
+
+  /v1/{account}/{container}/?marker=a&end_marker=d
+  b
+  c
+
+However, when reversed marker and end_marker are applied to a reversed list:
+
+.. code::
+
+  /v1/{account}/{container}/?marker=d&end_marker=a&reverse=on
+  c
+  b
 
 Object Storage HTTP requests have the following default constraints.
 Your service provider might use different default values.

swift/account/backend.py

@@ -366,7 +366,7 @@ class AccountBroker(DatabaseBroker):
             ''').fetchone())
 
     def list_containers_iter(self, limit, marker, end_marker, prefix,
-                             delimiter):
+                             delimiter, reverse=False):
         """
         Get a list of containers sorted by name starting at marker onward, up
         to limit entries. Entries will begin with the prefix and will not have
@@ -377,15 +377,21 @@ class AccountBroker(DatabaseBroker):
         :param end_marker: end marker query
         :param prefix: prefix query
         :param delimiter: delimiter for query
+        :param reverse: reverse the result order.
 
         :returns: list of tuples of (name, object_count, bytes_used, 0)
         """
         delim_force_gte = False
         (marker, end_marker, prefix, delimiter) = utf8encode(
             marker, end_marker, prefix, delimiter)
+        if reverse:
+            # Reverse the markers if we are reversing the listing.
+            marker, end_marker = end_marker, marker
         self._commit_puts_stale_ok()
         if delimiter and not prefix:
             prefix = ''
+        if prefix:
+            end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1)
         orig_marker = marker
         with self.get() as conn:
             results = []
@@ -395,9 +401,13 @@ class AccountBroker(DatabaseBroker):
                     FROM container
                     WHERE """
                 query_args = []
-                if end_marker:
+                if end_marker and (not prefix or end_marker < end_prefix):
                     query += ' name < ? AND'
                     query_args.append(end_marker)
+                elif prefix:
+                    query += ' name < ? AND'
+                    query_args.append(end_prefix)
+
                 if delim_force_gte:
                     query += ' name >= ? AND'
                     query_args.append(marker)
@@ -413,38 +423,40 @@ class AccountBroker(DatabaseBroker):
                     query += ' +deleted = 0'
                 else:
                     query += ' deleted = 0'
-                query += ' ORDER BY name LIMIT ?'
+                query += ' ORDER BY name %s LIMIT ?' % \
+                         ('DESC' if reverse else '')
                 query_args.append(limit - len(results))
                 curs = conn.execute(query, query_args)
                 curs.row_factory = None
 
-                if prefix is None:
+                # Delimiters without a prefix is ignored, further if there
-                    # A delimiter without a specified prefix is ignored
+                # is no delimiter then we can simply return the result as
+                # prefixes are now handled in the SQL statement.
+                if prefix is None or not delimiter:
                     return [r for r in curs]
-                if not delimiter:
-                    if not prefix:
-                        # It is possible to have a delimiter but no prefix
-                        # specified. As above, the prefix will be set to the
-                        # empty string, so avoid performing the extra work to
-                        # check against an empty prefix.
-                        return [r for r in curs]
-                    else:
-                        return [r for r in curs if r[0].startswith(prefix)]
 
                 # We have a delimiter and a prefix (possibly empty string) to
                 # handle
                 rowcount = 0
                 for row in curs:
                     rowcount += 1
-                    marker = name = row[0]
+                    name = row[0]
+                    if reverse:
+                        end_marker = name
+                    else:
+                        marker = name
+
                     if len(results) >= limit or not name.startswith(prefix):
                         curs.close()
                         return results
                     end = name.find(delimiter, len(prefix))
                     if end > 0:
-                        marker = name[:end] + chr(ord(delimiter) + 1)
+                        if reverse:
-                        # we want result to be inclusive of delim+1
+                            end_marker = name[:end + 1]
-                        delim_force_gte = True
+                        else:
+                            marker = name[:end] + chr(ord(delimiter) + 1)
+                            # we want result to be inclusive of delim+1
+                            delim_force_gte = True
                         dir_name = name[:end + 1]
                         if dir_name != orig_marker:
                             results.append([dir_name, 0, 0, 1])

swift/account/server.py

@@ -191,6 +191,7 @@ class AccountController(BaseStorageServer):
             return HTTPPreconditionFailed(body='Bad delimiter')
         limit = constraints.ACCOUNT_LISTING_LIMIT
         given_limit = get_param(req, 'limit')
+        reverse = config_true_value(get_param(req, 'reverse'))
         if given_limit and given_limit.isdigit():
             limit = int(given_limit)
             if limit > constraints.ACCOUNT_LISTING_LIMIT:
@@ -211,7 +212,7 @@ class AccountController(BaseStorageServer):
             return self._deleted_response(broker, req, HTTPNotFound)
         return account_listing_response(account, req, out_content_type, broker,
                                         limit, marker, end_marker, prefix,
-                                        delimiter)
+                                        delimiter, reverse)
 
     @public
     @replication

swift/account/utils.py

@@ -70,14 +70,14 @@ def get_response_headers(broker):
 
 def account_listing_response(account, req, response_content_type, broker=None,
                              limit='', marker='', end_marker='', prefix='',
-                             delimiter=''):
+                             delimiter='', reverse=False):
     if broker is None:
         broker = FakeAccountBroker()
 
     resp_headers = get_response_headers(broker)
 
     account_list = broker.list_containers_iter(limit, marker, end_marker,
-                                               prefix, delimiter)
+                                               prefix, delimiter, reverse)
     if response_content_type == 'application/json':
         data = []
         for (name, object_count, bytes_used, is_subdir) in account_list:

swift/container/backend.py

@@ -557,7 +557,7 @@ class ContainerBroker(DatabaseBroker):
             conn.commit()
 
     def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
-                          path=None, storage_policy_index=0):
+                          path=None, storage_policy_index=0, reverse=False):
         """
         Get a list of objects sorted by name starting at marker onward, up
         to limit entries.  Entries will begin with the prefix and will not
@@ -570,6 +570,7 @@ class ContainerBroker(DatabaseBroker):
         :param delimiter: delimiter for query
         :param path: if defined, will set the prefix and delimiter based on
                      the path
+        :param reverse: reverse the result order.
 
         :returns: list of tuples of (name, created_at, size, content_type,
                   etag)
@@ -578,6 +579,9 @@ class ContainerBroker(DatabaseBroker):
         (marker, end_marker, prefix, delimiter, path) = utf8encode(
             marker, end_marker, prefix, delimiter, path)
         self._commit_puts_stale_ok()
+        if reverse:
+            # Reverse the markers if we are reversing the listing.
+            marker, end_marker = end_marker, marker
         if path is not None:
             prefix = path
             if path:
@@ -585,6 +589,8 @@ class ContainerBroker(DatabaseBroker):
             delimiter = '/'
         elif delimiter and not prefix:
             prefix = ''
+        if prefix:
+            end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1)
         orig_marker = marker
         with self.get() as conn:
             results = []
@@ -592,9 +598,13 @@ class ContainerBroker(DatabaseBroker):
                 query = '''SELECT name, created_at, size, content_type, etag
                            FROM object WHERE'''
                 query_args = []
-                if end_marker:
+                if end_marker and (not prefix or end_marker < end_prefix):
                     query += ' name < ? AND'
                     query_args.append(end_marker)
+                elif prefix:
+                    query += ' name < ? AND'
+                    query_args.append(end_prefix)
+
                 if delim_force_gte:
                     query += ' name >= ? AND'
                     query_args.append(marker)
@@ -611,8 +621,8 @@ class ContainerBroker(DatabaseBroker):
                 else:
                     query += ' deleted = 0'
                 orig_tail_query = '''
-                    ORDER BY name LIMIT ?
+                    ORDER BY name %s LIMIT ?
-                '''
+                ''' % ('DESC' if reverse else '')
                 orig_tail_args = [limit - len(results)]
                 # storage policy filter
                 policy_tail_query = '''
@@ -633,25 +643,23 @@ class ContainerBroker(DatabaseBroker):
                                         tuple(query_args + tail_args))
                 curs.row_factory = None
 
-                if prefix is None:
+                # Delimiters without a prefix is ignored, further if there
-                    # A delimiter without a specified prefix is ignored
+                # is no delimiter then we can simply return the result as
+                # prefixes are now handled in the SQL statement.
+                if prefix is None or not delimiter:
                     return [r for r in curs]
-                if not delimiter:
-                    if not prefix:
-                        # It is possible to have a delimiter but no prefix
-                        # specified. As above, the prefix will be set to the
-                        # empty string, so avoid performing the extra work to
-                        # check against an empty prefix.
-                        return [r for r in curs]
-                    else:
-                        return [r for r in curs if r[0].startswith(prefix)]
 
                 # We have a delimiter and a prefix (possibly empty string) to
                 # handle
                 rowcount = 0
                 for row in curs:
                     rowcount += 1
-                    marker = name = row[0]
+                    name = row[0]
+                    if reverse:
+                        end_marker = name
+                    else:
+                        marker = name
+
                     if len(results) >= limit or not name.startswith(prefix):
                         curs.close()
                         return results
@@ -660,13 +668,19 @@ class ContainerBroker(DatabaseBroker):
                         if name == path:
                             continue
                         if end >= 0 and len(name) > end + len(delimiter):
-                            marker = name[:end] + chr(ord(delimiter) + 1)
+                            if reverse:
+                                end_marker = name[:end + 1]
+                            else:
+                                marker = name[:end] + chr(ord(delimiter) + 1)
                             curs.close()
                             break
                     elif end > 0:
-                        marker = name[:end] + chr(ord(delimiter) + 1)
+                        if reverse:
-                        # we want result to be inclusive of delim+1
+                            end_marker = name[:end + 1]
-                        delim_force_gte = True
+                        else:
+                            marker = name[:end] + chr(ord(delimiter) + 1)
+                            # we want result to be inclusive of delim+1
+                            delim_force_gte = True
                         dir_name = name[:end + 1]
                         if dir_name != orig_marker:
                             results.append([dir_name, '0', 0, None, ''])

swift/container/server.py

@@ -452,6 +452,7 @@ class ContainerController(BaseStorageServer):
         end_marker = get_param(req, 'end_marker')
         limit = constraints.CONTAINER_LISTING_LIMIT
         given_limit = get_param(req, 'limit')
+        reverse = config_true_value(get_param(req, 'reverse'))
         if given_limit and given_limit.isdigit():
             limit = int(given_limit)
             if limit > constraints.CONTAINER_LISTING_LIMIT:
@@ -471,7 +472,7 @@ class ContainerController(BaseStorageServer):
             return HTTPNotFound(request=req, headers=resp_headers)
         container_list = broker.list_objects_iter(
             limit, marker, end_marker, prefix, delimiter, path,
-            storage_policy_index=info['storage_policy_index'])
+            storage_policy_index=info['storage_policy_index'], reverse=reverse)
         return self.create_listing(req, out_content_type, info, resp_headers,
                                    broker.metadata, container_list, container)
 

test/functional/tests.py

@@ -327,6 +327,77 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
     set_up = False
 
 
+class TestAccountSortingEnv(object):
+    @classmethod
+    def setUp(cls):
+        cls.conn = Connection(tf.config)
+        cls.conn.authenticate()
+        cls.account = Account(cls.conn, tf.config.get('account',
+                                                      tf.config['username']))
+        cls.account.delete_containers()
+
+        postfix = Utils.create_name()
+        cls.cont_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz')
+        cls.cont_items = ['%s%s' % (x, postfix) for x in cls.cont_items]
+
+        for container in cls.cont_items:
+            c = cls.account.container(container)
+            if not c.create():
+                raise ResponseError(cls.conn.response)
+
+
+class TestAccountSorting(Base):
+    env = TestAccountSortingEnv
+    set_up = False
+
+    def testAccountContainerListSorting(self):
+        # name (byte order) sorting.
+        cont_list = sorted(self.env.cont_items)
+        cont_list.reverse()
+        cont_listing = self.env.account.containers(parms={'reverse': 'on'})
+        self.assert_status(200)
+        self.assertEqual(cont_list, cont_listing)
+
+    def testAccountContainerListSortingByPrefix(self):
+        cont_list = sorted(c for c in self.env.cont_items if c.startswith('a'))
+        cont_list.reverse()
+        cont_listing = self.env.account.containers(parms={
+            'reverse': 'on', 'prefix': 'a'})
+        self.assert_status(200)
+        self.assertEqual(cont_list, cont_listing)
+
+    def testAccountContainerListSortingByMarkersExclusive(self):
+        first_item = self.env.cont_items[3]  # 'b1' + postfix
+        last_item = self.env.cont_items[4]  # 'B2' + postfix
+
+        cont_list = sorted(c for c in self.env.cont_items
+                           if last_item < c < first_item)
+        cont_list.reverse()
+        cont_listing = self.env.account.containers(parms={
+            'reverse': 'on', 'marker': first_item, 'end_marker': last_item})
+        self.assert_status(200)
+        self.assertEqual(cont_list, cont_listing)
+
+    def testAccountContainerListSortingByMarkersInclusive(self):
+        first_item = self.env.cont_items[3]  # 'b1' + postfix
+        last_item = self.env.cont_items[4]  # 'B2' + postfix
+
+        cont_list = sorted(c for c in self.env.cont_items
+                           if last_item <= c <= first_item)
+        cont_list.reverse()
+        cont_listing = self.env.account.containers(parms={
+            'reverse': 'on', 'marker': first_item + '\x00',
+            'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)})
+        self.assert_status(200)
+        self.assertEqual(cont_list, cont_listing)
+
+    def testAccountContainerListSortingByReversedMarkers(self):
+        cont_listing = self.env.account.containers(parms={
+            'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'})
+        self.assert_status(204)
+        self.assertEqual([], cont_listing)
+
+
 class TestContainerEnv(object):
     @classmethod
     def setUp(cls):
@@ -647,6 +718,115 @@ class TestContainerUTF8(Base2, TestContainer):
     set_up = False
 
 
+class TestContainerSortingEnv(object):
+    @classmethod
+    def setUp(cls):
+        cls.conn = Connection(tf.config)
+        cls.conn.authenticate()
+        cls.account = Account(cls.conn, tf.config.get('account',
+                                                      tf.config['username']))
+        cls.account.delete_containers()
+
+        cls.container = cls.account.container(Utils.create_name())
+        if not cls.container.create():
+            raise ResponseError(cls.conn.response)
+
+        cls.file_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz')
+        cls.files = list()
+        cls.file_size = 128
+        for name in cls.file_items:
+            file_item = cls.container.file(name)
+            file_item.write_random(cls.file_size)
+            cls.files.append(file_item.name)
+
+
+class TestContainerSorting(Base):
+    env = TestContainerSortingEnv
+    set_up = False
+
+    def testContainerFileListSortingReversed(self):
+        file_list = list(sorted(self.env.file_items))
+        file_list.reverse()
+        cont_files = self.env.container.files(parms={'reverse': 'on'})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+    def testContainerFileSortingByPrefixReversed(self):
+        cont_list = sorted(c for c in self.env.file_items if c.startswith('a'))
+        cont_list.reverse()
+        cont_listing = self.env.container.files(parms={
+            'reverse': 'on', 'prefix': 'a'})
+        self.assert_status(200)
+        self.assertEqual(cont_list, cont_listing)
+
+    def testContainerFileSortingByMarkersExclusiveReversed(self):
+        first_item = self.env.file_items[3]  # 'b1' + postfix
+        last_item = self.env.file_items[4]  # 'B2' + postfix
+
+        cont_list = sorted(c for c in self.env.file_items
+                           if last_item < c < first_item)
+        cont_list.reverse()
+        cont_listing = self.env.container.files(parms={
+            'reverse': 'on', 'marker': first_item, 'end_marker': last_item})
+        self.assert_status(200)
+        self.assertEqual(cont_list, cont_listing)
+
+    def testContainerFileSortingByMarkersInclusiveReversed(self):
+        first_item = self.env.file_items[3]  # 'b1' + postfix
+        last_item = self.env.file_items[4]  # 'B2' + postfix
+
+        cont_list = sorted(c for c in self.env.file_items
+                           if last_item <= c <= first_item)
+        cont_list.reverse()
+        cont_listing = self.env.container.files(parms={
+            'reverse': 'on', 'marker': first_item + '\x00',
+            'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)})
+        self.assert_status(200)
+        self.assertEqual(cont_list, cont_listing)
+
+    def testContainerFileSortingByReversedMarkersReversed(self):
+        cont_listing = self.env.container.files(parms={
+            'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'})
+        self.assert_status(204)
+        self.assertEqual([], cont_listing)
+
+    def testContainerFileListSorting(self):
+        file_list = list(sorted(self.env.file_items))
+        cont_files = self.env.container.files()
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+        # Lets try again but with reverse is specifically turned off
+        cont_files = self.env.container.files(parms={'reverse': 'off'})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+        cont_files = self.env.container.files(parms={'reverse': 'false'})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+        cont_files = self.env.container.files(parms={'reverse': 'no'})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+        cont_files = self.env.container.files(parms={'reverse': ''})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+        # Lets try again but with a incorrect reverse values
+        cont_files = self.env.container.files(parms={'reverse': 'foo'})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+        cont_files = self.env.container.files(parms={'reverse': 'hai'})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+        cont_files = self.env.container.files(parms={'reverse': 'o=[]::::>'})
+        self.assert_status(200)
+        self.assertEqual(file_list, cont_files)
+
+
 class TestContainerPathsEnv(object):
     @classmethod
     def setUp(cls):

test/unit/account/test_backend.py

@@ -416,20 +416,48 @@ class TestAccountBroker(unittest.TestCase):
         self.assertEqual(listing[0][0], '0-0100')
         self.assertEqual(listing[-1][0], '0-0109')
 
+        listing = broker.list_containers_iter(10, '', None, '0-00', '-',
+                                              reverse=True)
+        self.assertEqual(len(listing), 10)
+        self.assertEqual(listing[0][0], '0-0099')
+        self.assertEqual(listing[-1][0], '0-0090')
+
         listing = broker.list_containers_iter(10, '', None, '0-', '-')
         self.assertEqual(len(listing), 10)
         self.assertEqual(listing[0][0], '0-0000')
         self.assertEqual(listing[-1][0], '0-0009')
 
+        listing = broker.list_containers_iter(10, '', None, '0-', '-',
+                                              reverse=True)
+        self.assertEqual(len(listing), 10)
+        self.assertEqual(listing[0][0], '0-0124')
+        self.assertEqual(listing[-1][0], '0-0115')
+
         listing = broker.list_containers_iter(10, '', None, '', '-')
         self.assertEqual(len(listing), 4)
         self.assertEqual([row[0] for row in listing],
                          ['0-', '1-', '2-', '3-'])
 
+        listing = broker.list_containers_iter(10, '', None, '', '-',
+                                              reverse=True)
+        self.assertEqual(len(listing), 4)
+        self.assertEqual([row[0] for row in listing],
+                         ['3-', '2-', '1-', '0-'])
+
         listing = broker.list_containers_iter(10, '2-', None, None, '-')
         self.assertEqual(len(listing), 1)
         self.assertEqual([row[0] for row in listing], ['3-'])
 
+        listing = broker.list_containers_iter(10, '2-', None, None, '-',
+                                              reverse=True)
+        self.assertEqual(len(listing), 2)
+        self.assertEqual([row[0] for row in listing], ['1-', '0-'])
+
+        listing = broker.list_containers_iter(10, '2.', None, None, '-',
+                                              reverse=True)
+        self.assertEqual(len(listing), 3)
+        self.assertEqual([row[0] for row in listing], ['2-', '1-', '0-'])
+
         listing = broker.list_containers_iter(10, '', None, '2', '-')
         self.assertEqual(len(listing), 1)
         self.assertEqual([row[0] for row in listing], ['2-'])
@@ -469,6 +497,147 @@ class TestAccountBroker(unittest.TestCase):
         self.assertEqual([row[0] for row in listing],
                          ['3-0049-', '3-0049-0049'])
 
+    def test_list_objects_iter_order_and_reverse(self):
+        # Test ContainerBroker.list_objects_iter
+        broker = AccountBroker(':memory:', account='a')
+        broker.initialize(Timestamp('1').internal, 0)
+
+        broker.put_container(
+            'c1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
+        broker.put_container(
+            'c10', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
+        broker.put_container(
+            'C1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
+        broker.put_container(
+            'c2', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
+        broker.put_container(
+            'c3', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
+        broker.put_container(
+            'C4', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
+
+        listing = broker.list_containers_iter(100, None, None, '', '',
+                                              reverse=False)
+        self.assertEqual([row[0] for row in listing],
+                         ['C1', 'C4', 'c1', 'c10', 'c2', 'c3'])
+        listing = broker.list_containers_iter(100, None, None, '', '',
+                                              reverse=True)
+        self.assertEqual([row[0] for row in listing],
+                         ['c3', 'c2', 'c10', 'c1', 'C4', 'C1'])
+        listing = broker.list_containers_iter(2, None, None, '', '',
+                                              reverse=True)
+        self.assertEqual([row[0] for row in listing],
+                         ['c3', 'c2'])
+        listing = broker.list_containers_iter(100, 'c2', 'C4', '', '',
+                                              reverse=True)
+        self.assertEqual([row[0] for row in listing],
+                         ['c10', 'c1'])
+
+    def test_reverse_prefix_delim(self):
+        expectations = [
+            {
+                'containers': [
+                    'topdir1-subdir1,0-c1',
+                    'topdir1-subdir1,1-c1',
+                    'topdir1-subdir1-c1',
+                ],
+                'params': {
+                    'prefix': 'topdir1-',
+                    'delimiter': '-',
+                },
+                'expected': [
+                    'topdir1-subdir1,0-',
+                    'topdir1-subdir1,1-',
+                    'topdir1-subdir1-',
+                ],
+            },
+            {
+                'containers': [
+                    'topdir1-subdir1,0-c1',
+                    'topdir1-subdir1,1-c1',
+                    'topdir1-subdir1-c1',
+                    'topdir1-subdir1.',
+                    'topdir1-subdir1.-c1',
+                ],
+                'params': {
+                    'prefix': 'topdir1-',
+                    'delimiter': '-',
+                },
+                'expected': [
+                    'topdir1-subdir1,0-',
+                    'topdir1-subdir1,1-',
+                    'topdir1-subdir1-',
+                    'topdir1-subdir1.',
+                    'topdir1-subdir1.-',
+                ],
+            },
+            {
+                'containers': [
+                    'topdir1-subdir1-c1',
+                    'topdir1-subdir1,0-c1',
+                    'topdir1-subdir1,1-c1',
+                ],
+                'params': {
+                    'prefix': 'topdir1-',
+                    'delimiter': '-',
+                    'reverse': True,
+                },
+                'expected': [
+                    'topdir1-subdir1-',
+                    'topdir1-subdir1,1-',
+                    'topdir1-subdir1,0-',
+                ],
+            },
+            {
+                'containers': [
+                    'topdir1-subdir1.-c1',
+                    'topdir1-subdir1.',
+                    'topdir1-subdir1-c1',
+                    'topdir1-subdir1-',
+                    'topdir1-subdir1,',
+                    'topdir1-subdir1,0-c1',
+                    'topdir1-subdir1,1-c1',
+                ],
+                'params': {
+                    'prefix': 'topdir1-',
+                    'delimiter': '-',
+                    'reverse': True,
+                },
+                'expected': [
+                    'topdir1-subdir1.-',
+                    'topdir1-subdir1.',
+                    'topdir1-subdir1-',
+                    'topdir1-subdir1,1-',
+                    'topdir1-subdir1,0-',
+                    'topdir1-subdir1,',
+                ],
+            },
+        ]
+        ts = make_timestamp_iter()
+        default_listing_params = {
+            'limit': 10000,
+            'marker': '',
+            'end_marker': None,
+            'prefix': None,
+            'delimiter': None,
+        }
+        failures = []
+        for expected in expectations:
+            broker = AccountBroker(':memory:', account='a')
+            broker.initialize(next(ts).internal, 0)
+            for name in expected['containers']:
+                broker.put_container(name, next(ts).internal, 0, 0, 0,
+                                     POLICIES.default.idx)
+            params = default_listing_params.copy()
+            params.update(expected['params'])
+            listing = list(c[0] for c in broker.list_containers_iter(**params))
+            if listing != expected['expected']:
+                expected['listing'] = listing
+                failures.append(
+                    "With containers %(containers)r, the params %(params)r "
+                    "produced %(listing)r instead of %(expected)r" % expected)
+        self.assertFalse(failures, "Found the following failures:\n%s" %
+                         '\n'.join(failures))
+
     def test_double_check_trailing_delimiter(self):
         # Test AccountBroker.list_containers_iter for an
         # account that has an odd container with a trailing delimiter

test/unit/container/test_backend.py

@@ -34,7 +34,8 @@ from swift.common.storage_policy import POLICIES
 
 import mock
 
-from test.unit import patch_policies, with_tempdir
+from test.unit import (patch_policies, with_tempdir, make_timestamp_iter,
+                       EMPTY_ETAG)
 from test.unit.common.test_db import TestExampleBroker
 
 
@@ -773,6 +774,12 @@ class TestContainerBroker(unittest.TestCase):
         self.assertEqual(listing[0][0], '1/0075')
         self.assertEqual(listing[-1][0], '2/0004')
 
+        listing = broker.list_objects_iter(55, '2/0005', None, None, '',
+                                           reverse=True)
+        self.assertEqual(len(listing), 55)
+        self.assertEqual(listing[0][0], '2/0004')
+        self.assertEqual(listing[-1][0], '1/0075')
+
         listing = broker.list_objects_iter(10, '', None, '0/01', '')
         self.assertEqual(len(listing), 10)
         self.assertEqual(listing[0][0], '0/0100')
@@ -783,17 +790,34 @@ class TestContainerBroker(unittest.TestCase):
         self.assertEqual(listing[0][0], '0/0000')
         self.assertEqual(listing[-1][0], '0/0009')
 
+        listing = broker.list_objects_iter(10, '', None, '0/', '/',
+                                           reverse=True)
+        self.assertEqual(len(listing), 10)
+        self.assertEqual(listing[0][0], '0/0124')
+        self.assertEqual(listing[-1][0], '0/0115')
+
         # Same as above, but using the path argument.
         listing = broker.list_objects_iter(10, '', None, None, '', '0')
         self.assertEqual(len(listing), 10)
         self.assertEqual(listing[0][0], '0/0000')
         self.assertEqual(listing[-1][0], '0/0009')
 
+        listing = broker.list_objects_iter(10, '', None, None, '', '0',
+                                           reverse=True)
+        self.assertEqual(len(listing), 10)
+        self.assertEqual(listing[0][0], '0/0124')
+        self.assertEqual(listing[-1][0], '0/0115')
+
         listing = broker.list_objects_iter(10, '', None, '', '/')
         self.assertEqual(len(listing), 4)
         self.assertEqual([row[0] for row in listing],
                          ['0/', '1/', '2/', '3/'])
 
+        listing = broker.list_objects_iter(10, '', None, '', '/', reverse=True)
+        self.assertEqual(len(listing), 4)
+        self.assertEqual([row[0] for row in listing],
+                         ['3/', '2/', '1/', '0/'])
+
         listing = broker.list_objects_iter(10, '2', None, None, '/')
         self.assertEqual(len(listing), 2)
         self.assertEqual([row[0] for row in listing], ['2/', '3/'])
@@ -802,6 +826,16 @@ class TestContainerBroker(unittest.TestCase):
         self.assertEqual(len(listing), 1)
         self.assertEqual([row[0] for row in listing], ['3/'])
 
+        listing = broker.list_objects_iter(10, '2/', None, None, '/',
+                                           reverse=True)
+        self.assertEqual(len(listing), 2)
+        self.assertEqual([row[0] for row in listing], ['1/', '0/'])
+
+        listing = broker.list_objects_iter(10, '20', None, None, '/',
+                                           reverse=True)
+        self.assertEqual(len(listing), 3)
+        self.assertEqual([row[0] for row in listing], ['2/', '1/', '0/'])
+
         listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/')
         self.assertEqual(len(listing), 10)
         self.assertEqual(listing[0][0], '2/0051')
@@ -852,6 +886,113 @@ class TestContainerBroker(unittest.TestCase):
         self.assertEqual(len(listing), 2)
         self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001'])
 
+    def test_reverse_prefix_delim(self):
+        expectations = [
+            {
+                'objects': [
+                    'topdir1/subdir1.0/obj1',
+                    'topdir1/subdir1.1/obj1',
+                    'topdir1/subdir1/obj1',
+                ],
+                'params': {
+                    'prefix': 'topdir1/',
+                    'delimiter': '/',
+                },
+                'expected': [
+                    'topdir1/subdir1.0/',
+                    'topdir1/subdir1.1/',
+                    'topdir1/subdir1/',
+                ],
+            },
+            {
+                'objects': [
+                    'topdir1/subdir1.0/obj1',
+                    'topdir1/subdir1.1/obj1',
+                    'topdir1/subdir1/obj1',
+                    'topdir1/subdir10',
+                    'topdir1/subdir10/obj1',
+                ],
+                'params': {
+                    'prefix': 'topdir1/',
+                    'delimiter': '/',
+                },
+                'expected': [
+                    'topdir1/subdir1.0/',
+                    'topdir1/subdir1.1/',
+                    'topdir1/subdir1/',
+                    'topdir1/subdir10',
+                    'topdir1/subdir10/',
+                ],
+            },
+            {
+                'objects': [
+                    'topdir1/subdir1/obj1',
+                    'topdir1/subdir1.0/obj1',
+                    'topdir1/subdir1.1/obj1',
+                ],
+                'params': {
+                    'prefix': 'topdir1/',
+                    'delimiter': '/',
+                    'reverse': True,
+                },
+                'expected': [
+                    'topdir1/subdir1/',
+                    'topdir1/subdir1.1/',
+                    'topdir1/subdir1.0/',
+                ],
+            },
+            {
+                'objects': [
+                    'topdir1/subdir10/obj1',
+                    'topdir1/subdir10',
+                    'topdir1/subdir1/obj1',
+                    'topdir1/subdir1.0/obj1',
+                    'topdir1/subdir1.1/obj1',
+                ],
+                'params': {
+                    'prefix': 'topdir1/',
+                    'delimiter': '/',
+                    'reverse': True,
+                },
+                'expected': [
+                    'topdir1/subdir10/',
+                    'topdir1/subdir10',
+                    'topdir1/subdir1/',
+                    'topdir1/subdir1.1/',
+                    'topdir1/subdir1.0/',
+                ],
+            },
+        ]
+        ts = make_timestamp_iter()
+        default_listing_params = {
+            'limit': 10000,
+            'marker': '',
+            'end_marker': None,
+            'prefix': None,
+            'delimiter': None,
+        }
+        obj_create_params = {
+            'size': 0,
+            'content_type': 'application/test',
+            'etag': EMPTY_ETAG,
+        }
+        failures = []
+        for expected in expectations:
+            broker = ContainerBroker(':memory:', account='a', container='c')
+            broker.initialize(next(ts).internal, 0)
+            for name in expected['objects']:
+                broker.put_object(name, next(ts).internal, **obj_create_params)
+            params = default_listing_params.copy()
+            params.update(expected['params'])
+            listing = list(o[0] for o in broker.list_objects_iter(**params))
+            if listing != expected['expected']:
+                expected['listing'] = listing
+                failures.append(
+                    "With objects %(objects)r, the params %(params)r "
+                    "produced %(listing)r instead of %(expected)r" % expected)
+        self.assertFalse(failures, "Found the following failures:\n%s" %
+                         '\n'.join(failures))
+
     def test_list_objects_iter_non_slash(self):
         # Test ContainerBroker.list_objects_iter using a
         # delimiter that is not a slash
@@ -1006,6 +1147,47 @@ class TestContainerBroker(unittest.TestCase):
         self.assertEqual([row[0] for row in listing],
                          ['/pets/fish/a', '/pets/fish/b'])
 
+    def test_list_objects_iter_order_and_reverse(self):
+        # Test ContainerBroker.list_objects_iter
+        broker = ContainerBroker(':memory:', account='a', container='c')
+        broker.initialize(Timestamp('1').internal, 0)
+
+        broker.put_object(
+            'o1', Timestamp(0).internal, 0,
+            'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+        broker.put_object(
+            'o10', Timestamp(0).internal, 0,
+            'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+        broker.put_object(
+            'O1', Timestamp(0).internal, 0,
+            'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+        broker.put_object(
+            'o2', Timestamp(0).internal, 0,
+            'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+        broker.put_object(
+            'o3', Timestamp(0).internal, 0,
+            'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+        broker.put_object(
+            'O4', Timestamp(0).internal, 0,
+            'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
+
+        listing = broker.list_objects_iter(100, None, None, '', '',
+                                           reverse=False)
+        self.assertEqual([row[0] for row in listing],
+                         ['O1', 'O4', 'o1', 'o10', 'o2', 'o3'])
+        listing = broker.list_objects_iter(100, None, None, '', '',
+                                           reverse=True)
+        self.assertEqual([row[0] for row in listing],
+                         ['o3', 'o2', 'o10', 'o1', 'O4', 'O1'])
+        listing = broker.list_objects_iter(2, None, None, '', '',
+                                           reverse=True)
+        self.assertEqual([row[0] for row in listing],
+                         ['o3', 'o2'])
+        listing = broker.list_objects_iter(100, 'o2', 'O4', '', '',
+                                           reverse=True)
+        self.assertEqual([row[0] for row in listing],
+                         ['o10', 'o1'])
+
     def test_double_check_trailing_delimiter(self):
         # Test ContainerBroker.list_objects_iter for a
         # container that has an odd file with a trailing delimiter