added end_marker query parameter to container and object listings

swift/account/reaper.py

@@ -224,7 +224,8 @@ class AccountReaper(Daemon):
             marker = ''
             while True:
                 containers = \
-                    list(broker.list_containers_iter(1000, marker, None, None))
+                    list(broker.list_containers_iter(1000, marker, None, None,
+                                                     None))
                 if not containers:
                     break
                 try:

swift/account/server.py

@@ -200,6 +200,7 @@ class AccountController(object):
                     return  HTTPPreconditionFailed(request=req,
                         body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT)
             marker = get_param(req, 'marker', '')
+            end_marker = get_param(req, 'end_marker')
             query_format = get_param(req, 'format')
         except UnicodeDecodeError, err:
             return HTTPBadRequest(body='parameters not utf8',
@@ -210,8 +211,8 @@ class AccountController(object):
                                 ['text/plain', 'application/json',
                                  'application/xml', 'text/xml'],
                                 default_match='text/plain')
-        account_list = broker.list_containers_iter(limit, marker, prefix,
+        account_list = broker.list_containers_iter(limit, marker, end_marker,
-                                                  delimiter)
+                                                   prefix, delimiter)
         if out_content_type == 'application/json':
             json_pattern = ['"name":%s', '"count":%s', '"bytes":%s']
             json_pattern = '{' + ','.join(json_pattern) + '}'

swift/common/db.py

@@ -940,8 +940,8 @@ class ContainerBroker(DatabaseBroker):
                     rv.append(row['name'])
         return list(set(rv))
 
-    def list_objects_iter(self, limit, marker, prefix, delimiter, path=None,
+    def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
-                          format=None):
+                          path=None, format=None):
         """
         Get a list of objects sorted by name starting at marker onward, up
         to limit entries.  Entries will begin with the prefix and will not
@@ -949,6 +949,7 @@ class ContainerBroker(DatabaseBroker):
 
         :param limit: maximum number of entries to get
         :param marker: marker query
+        :param end_marker: end marker query
         :param prefix: prefix query
         :param delimeter: delimeter for query
         :param path: if defined, will set the prefix and delimter based on
@@ -977,6 +978,9 @@ class ContainerBroker(DatabaseBroker):
                 query = '''SELECT name, created_at, size, content_type, etag
                            FROM object WHERE'''
                 query_args = []
+                if end_marker:
+                    query += ' name <= ? AND'
+                    query_args.append(end_marker)
                 if marker and marker >= prefix:
                     query += ' name > ? AND'
                     query_args.append(marker)
@@ -1440,7 +1444,8 @@ class AccountBroker(DatabaseBroker):
                     rv.append(row['name'])
         return list(set(rv))
 
-    def list_containers_iter(self, limit, marker, prefix, delimiter):
+    def list_containers_iter(self, limit, marker, end_marker, prefix,
+                             delimiter):
         """
         Get a list of containerss sorted by name starting at marker onward, up
         to limit entries.  Entries will begin with the prefix and will not
@@ -1448,6 +1453,7 @@ class AccountBroker(DatabaseBroker):
 
         :param limit: maximum number of entries to get
         :param marker: marker query
+        :param end_marker: end marker query
         :param prefix: prefix query
         :param delimeter: delimeter for query
 
@@ -1469,6 +1475,9 @@ class AccountBroker(DatabaseBroker):
                     FROM container
                     WHERE deleted = 0 AND """
                 query_args = []
+                if end_marker:
+                    query += ' name <= ? AND'
+                    query_args.append(end_marker)
                 if marker and marker >= prefix:
                     query += ' name > ? AND'
                     query_args.append(marker)

swift/container/server.py

@@ -267,6 +267,7 @@ class ContainerController(object):
                 # delimiters can be made more flexible later
                 return HTTPPreconditionFailed(body='Bad delimiter')
             marker = get_param(req, 'marker', '')
+            end_marker = get_param(req, 'end_marker')
             limit = CONTAINER_LISTING_LIMIT
             given_limit = get_param(req, 'limit')
             if given_limit and given_limit.isdigit():
@@ -284,8 +285,8 @@ class ContainerController(object):
                                 ['text/plain', 'application/json',
                                  'application/xml', 'text/xml'],
                                 default_match='text/plain')
-        container_list = broker.list_objects_iter(limit, marker, prefix,
+        container_list = broker.list_objects_iter(limit, marker, end_marker,
-                                                  delimiter, path)
+                                                  prefix, delimiter, path)
         if out_content_type == 'application/json':
             json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s',
                             '"content_type":%s, "last_modified":"%s"']

test/unit/common/test_db.py

@@ -978,52 +978,57 @@ class TestContainerBroker(unittest.TestCase):
                               normalize_timestamp(time()), 0, 'text/plain',
                               'd41d8cd98f00b204e9800998ecf8427e')
 
-        listing = broker.list_objects_iter(100, '', None, '')
+        listing = broker.list_objects_iter(100, '', None, None, '')
         self.assertEquals(len(listing), 100)
         self.assertEquals(listing[0][0], '0/0000')
         self.assertEquals(listing[-1][0], '0/0099')
 
-        listing = broker.list_objects_iter(100, '0/0099', None, '')
+        listing = broker.list_objects_iter(100, '', '0/0050', None, '')
+        self.assertEquals(len(listing), 51)
+        self.assertEquals(listing[0][0], '0/0000')
+        self.assertEquals(listing[-1][0], '0/0050')
+
+        listing = broker.list_objects_iter(100, '0/0099', None, None, '')
         self.assertEquals(len(listing), 100)
         self.assertEquals(listing[0][0], '0/0100')
         self.assertEquals(listing[-1][0], '1/0074')
 
-        listing = broker.list_objects_iter(55, '1/0074', None, '')
+        listing = broker.list_objects_iter(55, '1/0074', None, None, '')
         self.assertEquals(len(listing), 55)
         self.assertEquals(listing[0][0], '1/0075')
         self.assertEquals(listing[-1][0], '2/0004')
 
-        listing = broker.list_objects_iter(10, '', '0/01', '')
+        listing = broker.list_objects_iter(10, '', None, '0/01', '')
         self.assertEquals(len(listing), 10)
         self.assertEquals(listing[0][0], '0/0100')
         self.assertEquals(listing[-1][0], '0/0109')
 
-        listing = broker.list_objects_iter(10, '', '0/', '/')
+        listing = broker.list_objects_iter(10, '', None, '0/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals(listing[0][0], '0/0000')
         self.assertEquals(listing[-1][0], '0/0009')
 
-        listing = broker.list_objects_iter(10, '', '', '/')
+        listing = broker.list_objects_iter(10, '', None, '', '/')
         self.assertEquals(len(listing), 4)
         self.assertEquals([row[0] for row in listing],
                           ['0/', '1/', '2/', '3/'])
 
-        listing = broker.list_objects_iter(10, '2', None, '/')
+        listing = broker.list_objects_iter(10, '2', None, None, '/')
         self.assertEquals(len(listing), 2)
         self.assertEquals([row[0] for row in listing], ['2/', '3/'])
 
-        listing = broker.list_objects_iter(10, '2/', None, '/')
+        listing = broker.list_objects_iter(10, '2/',None,  None, '/')
         self.assertEquals(len(listing), 1)
         self.assertEquals([row[0] for row in listing], ['3/'])
 
-        listing = broker.list_objects_iter(10, '2/0050', '2/', '/')
+        listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals(listing[0][0], '2/0051')
         self.assertEquals(listing[1][0], '2/0051/')
         self.assertEquals(listing[2][0], '2/0052')
         self.assertEquals(listing[-1][0], '2/0059')
 
-        listing = broker.list_objects_iter(10, '3/0045', '3/', '/')
+        listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
                            ['3/0045/', '3/0046', '3/0046/', '3/0047',
@@ -1032,33 +1037,34 @@ class TestContainerBroker(unittest.TestCase):
 
         broker.put_object('3/0049/', normalize_timestamp(time()), 0,
                           'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
-        listing = broker.list_objects_iter(10, '3/0048', None, None)
+        listing = broker.list_objects_iter(10, '3/0048', None, None, None)
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
             ['3/0048/0049', '3/0049', '3/0049/',
             '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049',
             '3/0052', '3/0052/0049'])
 
-        listing = broker.list_objects_iter(10, '3/0048', '3/', '/')
+        listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
             ['3/0048/', '3/0049', '3/0049/', '3/0050',
             '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053'])
 
-        listing = broker.list_objects_iter(10, None, '3/0049/', '/')
+        listing = broker.list_objects_iter(10, None, None, '3/0049/', '/')
         self.assertEquals(len(listing), 2)
         self.assertEquals([row[0] for row in listing],
             ['3/0049/', '3/0049/0049'])
 
-        listing = broker.list_objects_iter(10, None, None, None, '3/0049')
+        listing = broker.list_objects_iter(10, None, None, None, None,
+                                           '3/0049')
         self.assertEquals(len(listing), 1)
         self.assertEquals([row[0] for row in listing], ['3/0049/0049'])
 
-        listing = broker.list_objects_iter(2, None, '3/', '/')
+        listing = broker.list_objects_iter(2, None, None, '3/', '/')
         self.assertEquals(len(listing), 2)
         self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/'])
 
-        listing = broker.list_objects_iter(2, None, None, None, '3')
+        listing = broker.list_objects_iter(2, None, None, None, None, '3')
         self.assertEquals(len(listing), 2)
         self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001'])
 
@@ -1082,11 +1088,11 @@ class TestContainerBroker(unittest.TestCase):
 
         #def list_objects_iter(self, limit, marker, prefix, delimiter, path=None,
         #                          format=None):
-        listing = broker.list_objects_iter(100, None, '/pets/f', '/')
+        listing = broker.list_objects_iter(100, None, None, '/pets/f', '/')
         self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt'])
-        listing = broker.list_objects_iter(100, None, '/pets/fish', '/')
+        listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/')
         self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt'])
-        listing = broker.list_objects_iter(100, None, '/pets/fish/', '/')
+        listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/')
         self.assertEquals([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b'])
 
     def test_double_check_trailing_delimiter(self):
@@ -1114,19 +1120,19 @@ class TestContainerBroker(unittest.TestCase):
                           'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
         broker.put_object('c', normalize_timestamp(time()), 0,
                           'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
-        listing = broker.list_objects_iter(15, None, None, None)
+        listing = broker.list_objects_iter(15, None, None, None, None)
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
             ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c'])
-        listing = broker.list_objects_iter(15, None, '', '/')
+        listing = broker.list_objects_iter(15, None, None, '', '/')
         self.assertEquals(len(listing), 5)
         self.assertEquals([row[0] for row in listing],
             ['a', 'a/', 'b', 'b/', 'c'])
-        listing = broker.list_objects_iter(15, None, 'a/', '/')
+        listing = broker.list_objects_iter(15, None, None, 'a/', '/')
         self.assertEquals(len(listing), 4)
         self.assertEquals([row[0] for row in listing],
             ['a/', 'a/a', 'a/a/', 'a/b'])
-        listing = broker.list_objects_iter(15, None, 'b/', '/')
+        listing = broker.list_objects_iter(15, None, None, 'b/', '/')
         self.assertEquals(len(listing), 2)
         self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])
 
@@ -1646,57 +1652,62 @@ class TestAccountBroker(unittest.TestCase):
             broker.put_container('3/%04d/0049' % cont,
                                  normalize_timestamp(time()), 0, 0, 0)
 
-        listing = broker.list_containers_iter(100, '', None, '')
+        listing = broker.list_containers_iter(100, '', None, None, '')
         self.assertEquals(len(listing), 100)
         self.assertEquals(listing[0][0], '0/0000')
         self.assertEquals(listing[-1][0], '0/0099')
 
-        listing = broker.list_containers_iter(100, '0/0099', None, '')
+        listing = broker.list_containers_iter(100, '', '0/0050', None, '')
+        self.assertEquals(len(listing), 51)
+        self.assertEquals(listing[0][0], '0/0000')
+        self.assertEquals(listing[-1][0], '0/0050')
+
+        listing = broker.list_containers_iter(100, '0/0099', None, None, '')
         self.assertEquals(len(listing), 100)
         self.assertEquals(listing[0][0], '0/0100')
         self.assertEquals(listing[-1][0], '1/0074')
 
-        listing = broker.list_containers_iter(55, '1/0074', None, '')
+        listing = broker.list_containers_iter(55, '1/0074', None, None, '')
         self.assertEquals(len(listing), 55)
         self.assertEquals(listing[0][0], '1/0075')
         self.assertEquals(listing[-1][0], '2/0004')
 
-        listing = broker.list_containers_iter(10, '', '0/01', '')
+        listing = broker.list_containers_iter(10, '', None, '0/01', '')
         self.assertEquals(len(listing), 10)
         self.assertEquals(listing[0][0], '0/0100')
         self.assertEquals(listing[-1][0], '0/0109')
 
-        listing = broker.list_containers_iter(10, '', '0/01', '/')
+        listing = broker.list_containers_iter(10, '', None, '0/01', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals(listing[0][0], '0/0100')
         self.assertEquals(listing[-1][0], '0/0109')
 
-        listing = broker.list_containers_iter(10, '', '0/', '/')
+        listing = broker.list_containers_iter(10, '', None, '0/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals(listing[0][0], '0/0000')
         self.assertEquals(listing[-1][0], '0/0009')
 
-        listing = broker.list_containers_iter(10, '', '', '/')
+        listing = broker.list_containers_iter(10, '', None, '', '/')
         self.assertEquals(len(listing), 4)
         self.assertEquals([row[0] for row in listing],
                           ['0/', '1/', '2/', '3/'])
 
-        listing = broker.list_containers_iter(10, '2/', None, '/')
+        listing = broker.list_containers_iter(10, '2/', None, None, '/')
         self.assertEquals(len(listing), 1)
         self.assertEquals([row[0] for row in listing], ['3/'])
 
-        listing = broker.list_containers_iter(10, '', '2', '/')
+        listing = broker.list_containers_iter(10, '', None, '2', '/')
         self.assertEquals(len(listing), 1)
         self.assertEquals([row[0] for row in listing], ['2/'])
 
-        listing = broker.list_containers_iter(10, '2/0050', '2/', '/')
+        listing = broker.list_containers_iter(10, '2/0050', None, '2/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals(listing[0][0], '2/0051')
         self.assertEquals(listing[1][0], '2/0051/')
         self.assertEquals(listing[2][0], '2/0052')
         self.assertEquals(listing[-1][0], '2/0059')
 
-        listing = broker.list_containers_iter(10, '3/0045', '3/', '/')
+        listing = broker.list_containers_iter(10, '3/0045', None, '3/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
                            ['3/0045/', '3/0046', '3/0046/', '3/0047',
@@ -1704,21 +1715,21 @@ class TestAccountBroker(unittest.TestCase):
                             '3/0049/', '3/0050'])
 
         broker.put_container('3/0049/', normalize_timestamp(time()), 0, 0, 0)
-        listing = broker.list_containers_iter(10, '3/0048', None, None)
+        listing = broker.list_containers_iter(10, '3/0048', None, None, None)
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
                            ['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049',
                             '3/0050', '3/0050/0049', '3/0051', '3/0051/0049',
                             '3/0052', '3/0052/0049'])
 
-        listing = broker.list_containers_iter(10, '3/0048', '3/', '/')
+        listing = broker.list_containers_iter(10, '3/0048', None, '3/', '/')
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
                            ['3/0048/', '3/0049', '3/0049/', '3/0050',
                             '3/0050/', '3/0051', '3/0051/', '3/0052',
                             '3/0052/', '3/0053'])
 
-        listing = broker.list_containers_iter(10, None, '3/0049/', '/')
+        listing = broker.list_containers_iter(10, None, None, '3/0049/', '/')
         self.assertEquals(len(listing), 2)
         self.assertEquals([row[0] for row in listing],
                           ['3/0049/', '3/0049/0049'])
@@ -1738,20 +1749,20 @@ class TestAccountBroker(unittest.TestCase):
         broker.put_container('b/a', normalize_timestamp(time()), 0, 0, 0)
         broker.put_container('b/b', normalize_timestamp(time()), 0, 0, 0)
         broker.put_container('c', normalize_timestamp(time()), 0, 0, 0)
-        listing = broker.list_containers_iter(15, None, None, None)
+        listing = broker.list_containers_iter(15, None, None, None, None)
         self.assertEquals(len(listing), 10)
         self.assertEquals([row[0] for row in listing],
                            ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b',
                             'b/a', 'b/b', 'c'])
-        listing = broker.list_containers_iter(15, None, '', '/')
+        listing = broker.list_containers_iter(15, None, None, '', '/')
         self.assertEquals(len(listing), 5)
         self.assertEquals([row[0] for row in listing],
                           ['a', 'a/', 'b', 'b/', 'c'])
-        listing = broker.list_containers_iter(15, None, 'a/', '/')
+        listing = broker.list_containers_iter(15, None, None, 'a/', '/')
         self.assertEquals(len(listing), 4)
         self.assertEquals([row[0] for row in listing],
                           ['a/', 'a/a', 'a/a/', 'a/b'])
-        listing = broker.list_containers_iter(15, None, 'b/', '/')
+        listing = broker.list_containers_iter(15, None, None, 'b/', '/')
         self.assertEquals(len(listing), 2)
         self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])