diff --git a/swift/account/reaper.py b/swift/account/reaper.py index 87fa026ebd..814f7551e5 100644 --- a/swift/account/reaper.py +++ b/swift/account/reaper.py @@ -224,7 +224,8 @@ class AccountReaper(Daemon): marker = '' while True: containers = \ - list(broker.list_containers_iter(1000, marker, None, None)) + list(broker.list_containers_iter(1000, marker, None, None, + None)) if not containers: break try: diff --git a/swift/account/server.py b/swift/account/server.py index bd10670b9e..67a67c4854 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -200,6 +200,7 @@ class AccountController(object): return HTTPPreconditionFailed(request=req, body='Maximum limit is %d' % ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') + end_marker = get_param(req, 'end_marker') query_format = get_param(req, 'format') except UnicodeDecodeError, err: return HTTPBadRequest(body='parameters not utf8', @@ -210,8 +211,8 @@ class AccountController(object): ['text/plain', 'application/json', 'application/xml', 'text/xml'], default_match='text/plain') - account_list = broker.list_containers_iter(limit, marker, prefix, - delimiter) + account_list = broker.list_containers_iter(limit, marker, end_marker, + prefix, delimiter) if out_content_type == 'application/json': json_pattern = ['"name":%s', '"count":%s', '"bytes":%s'] json_pattern = '{' + ','.join(json_pattern) + '}' diff --git a/swift/common/db.py b/swift/common/db.py index 3a04de3cad..c19c62910f 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -940,8 +940,8 @@ class ContainerBroker(DatabaseBroker): rv.append(row['name']) return list(set(rv)) - def list_objects_iter(self, limit, marker, prefix, delimiter, path=None, - format=None): + def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter, + path=None, format=None): """ Get a list of objects sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not @@ -949,6 +949,7 @@ class ContainerBroker(DatabaseBroker): :param limit: maximum number of entries to get :param marker: marker query + :param end_marker: end marker query :param prefix: prefix query :param delimeter: delimeter for query :param path: if defined, will set the prefix and delimter based on @@ -977,6 +978,9 @@ class ContainerBroker(DatabaseBroker): query = '''SELECT name, created_at, size, content_type, etag FROM object WHERE''' query_args = [] + if end_marker: + query += ' name <= ? AND' + query_args.append(end_marker) if marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) @@ -1440,7 +1444,8 @@ class AccountBroker(DatabaseBroker): rv.append(row['name']) return list(set(rv)) - def list_containers_iter(self, limit, marker, prefix, delimiter): + def list_containers_iter(self, limit, marker, end_marker, prefix, + delimiter): """ Get a list of containerss sorted by name starting at marker onward, up to limit entries. Entries will begin with the prefix and will not @@ -1448,6 +1453,7 @@ class AccountBroker(DatabaseBroker): :param limit: maximum number of entries to get :param marker: marker query + :param end_marker: end marker query :param prefix: prefix query :param delimeter: delimeter for query @@ -1469,6 +1475,9 @@ class AccountBroker(DatabaseBroker): FROM container WHERE deleted = 0 AND """ query_args = [] + if end_marker: + query += ' name <= ? AND' + query_args.append(end_marker) if marker and marker >= prefix: query += ' name > ? AND' query_args.append(marker) diff --git a/swift/container/server.py b/swift/container/server.py index 6dbbc940e0..45ed00fd4d 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -267,6 +267,7 @@ class ContainerController(object): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') marker = get_param(req, 'marker', '') + end_marker = get_param(req, 'end_marker') limit = CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') if given_limit and given_limit.isdigit(): @@ -284,8 +285,8 @@ class ContainerController(object): ['text/plain', 'application/json', 'application/xml', 'text/xml'], default_match='text/plain') - container_list = broker.list_objects_iter(limit, marker, prefix, - delimiter, path) + container_list = broker.list_objects_iter(limit, marker, end_marker, + prefix, delimiter, path) if out_content_type == 'application/json': json_pattern = ['"name":%s', '"hash":"%s"', '"bytes":%s', '"content_type":%s, "last_modified":"%s"'] diff --git a/test/unit/common/test_db.py b/test/unit/common/test_db.py index 55a776a4e9..8ab3a96d6e 100644 --- a/test/unit/common/test_db.py +++ b/test/unit/common/test_db.py @@ -978,52 +978,57 @@ class TestContainerBroker(unittest.TestCase): normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - listing = broker.list_objects_iter(100, '', None, '') + listing = broker.list_objects_iter(100, '', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0099') - listing = broker.list_objects_iter(100, '0/0099', None, '') + listing = broker.list_objects_iter(100, '', '0/0050', None, '') + self.assertEquals(len(listing), 51) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0050') + + listing = broker.list_objects_iter(100, '0/0099', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[-1][0], '1/0074') - listing = broker.list_objects_iter(55, '1/0074', None, '') + listing = broker.list_objects_iter(55, '1/0074', None, None, '') self.assertEquals(len(listing), 55) self.assertEquals(listing[0][0], '1/0075') self.assertEquals(listing[-1][0], '2/0004') - listing = broker.list_objects_iter(10, '', '0/01', '') + listing = broker.list_objects_iter(10, '', None, '0/01', '') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[-1][0], '0/0109') - listing = broker.list_objects_iter(10, '', '0/', '/') + listing = broker.list_objects_iter(10, '', None, '0/', '/') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0009') - listing = broker.list_objects_iter(10, '', '', '/') + listing = broker.list_objects_iter(10, '', None, '', '/') self.assertEquals(len(listing), 4) self.assertEquals([row[0] for row in listing], ['0/', '1/', '2/', '3/']) - listing = broker.list_objects_iter(10, '2', None, '/') + listing = broker.list_objects_iter(10, '2', None, None, '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['2/', '3/']) - listing = broker.list_objects_iter(10, '2/', None, '/') + listing = broker.list_objects_iter(10, '2/',None, None, '/') self.assertEquals(len(listing), 1) self.assertEquals([row[0] for row in listing], ['3/']) - listing = broker.list_objects_iter(10, '2/0050', '2/', '/') + listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '2/0051') self.assertEquals(listing[1][0], '2/0051/') self.assertEquals(listing[2][0], '2/0052') self.assertEquals(listing[-1][0], '2/0059') - listing = broker.list_objects_iter(10, '3/0045', '3/', '/') + listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/') self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3/0045/', '3/0046', '3/0046/', '3/0047', @@ -1032,33 +1037,34 @@ class TestContainerBroker(unittest.TestCase): broker.put_object('3/0049/', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - listing = broker.list_objects_iter(10, '3/0048', None, None) + listing = broker.list_objects_iter(10, '3/0048', None, None, None) self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', '3/0052', '3/0052/0049']) - listing = broker.list_objects_iter(10, '3/0048', '3/', '/') + listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/') self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3/0048/', '3/0049', '3/0049/', '3/0050', '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053']) - listing = broker.list_objects_iter(10, None, '3/0049/', '/') + listing = broker.list_objects_iter(10, None, None, '3/0049/', '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['3/0049/', '3/0049/0049']) - listing = broker.list_objects_iter(10, None, None, None, '3/0049') + listing = broker.list_objects_iter(10, None, None, None, None, + '3/0049') self.assertEquals(len(listing), 1) self.assertEquals([row[0] for row in listing], ['3/0049/0049']) - listing = broker.list_objects_iter(2, None, '3/', '/') + listing = broker.list_objects_iter(2, None, None, '3/', '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/']) - listing = broker.list_objects_iter(2, None, None, None, '3') + listing = broker.list_objects_iter(2, None, None, None, None, '3') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001']) @@ -1082,11 +1088,11 @@ class TestContainerBroker(unittest.TestCase): #def list_objects_iter(self, limit, marker, prefix, delimiter, path=None, # format=None): - listing = broker.list_objects_iter(100, None, '/pets/f', '/') + listing = broker.list_objects_iter(100, None, None, '/pets/f', '/') self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) - listing = broker.list_objects_iter(100, None, '/pets/fish', '/') + listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/') self.assertEquals([row[0] for row in listing], ['/pets/fish/', '/pets/fish_info.txt']) - listing = broker.list_objects_iter(100, None, '/pets/fish/', '/') + listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/') self.assertEquals([row[0] for row in listing], ['/pets/fish/a', '/pets/fish/b']) def test_double_check_trailing_delimiter(self): @@ -1114,19 +1120,19 @@ class TestContainerBroker(unittest.TestCase): 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') broker.put_object('c', normalize_timestamp(time()), 0, 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e') - listing = broker.list_objects_iter(15, None, None, None) + listing = broker.list_objects_iter(15, None, None, None, None) self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c']) - listing = broker.list_objects_iter(15, None, '', '/') + listing = broker.list_objects_iter(15, None, None, '', '/') self.assertEquals(len(listing), 5) self.assertEquals([row[0] for row in listing], ['a', 'a/', 'b', 'b/', 'c']) - listing = broker.list_objects_iter(15, None, 'a/', '/') + listing = broker.list_objects_iter(15, None, None, 'a/', '/') self.assertEquals(len(listing), 4) self.assertEquals([row[0] for row in listing], ['a/', 'a/a', 'a/a/', 'a/b']) - listing = broker.list_objects_iter(15, None, 'b/', '/') + listing = broker.list_objects_iter(15, None, None, 'b/', '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['b/a', 'b/b']) @@ -1646,57 +1652,62 @@ class TestAccountBroker(unittest.TestCase): broker.put_container('3/%04d/0049' % cont, normalize_timestamp(time()), 0, 0, 0) - listing = broker.list_containers_iter(100, '', None, '') + listing = broker.list_containers_iter(100, '', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0099') - listing = broker.list_containers_iter(100, '0/0099', None, '') + listing = broker.list_containers_iter(100, '', '0/0050', None, '') + self.assertEquals(len(listing), 51) + self.assertEquals(listing[0][0], '0/0000') + self.assertEquals(listing[-1][0], '0/0050') + + listing = broker.list_containers_iter(100, '0/0099', None, None, '') self.assertEquals(len(listing), 100) self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[-1][0], '1/0074') - listing = broker.list_containers_iter(55, '1/0074', None, '') + listing = broker.list_containers_iter(55, '1/0074', None, None, '') self.assertEquals(len(listing), 55) self.assertEquals(listing[0][0], '1/0075') self.assertEquals(listing[-1][0], '2/0004') - listing = broker.list_containers_iter(10, '', '0/01', '') + listing = broker.list_containers_iter(10, '', None, '0/01', '') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[-1][0], '0/0109') - listing = broker.list_containers_iter(10, '', '0/01', '/') + listing = broker.list_containers_iter(10, '', None, '0/01', '/') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0100') self.assertEquals(listing[-1][0], '0/0109') - listing = broker.list_containers_iter(10, '', '0/', '/') + listing = broker.list_containers_iter(10, '', None, '0/', '/') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '0/0000') self.assertEquals(listing[-1][0], '0/0009') - listing = broker.list_containers_iter(10, '', '', '/') + listing = broker.list_containers_iter(10, '', None, '', '/') self.assertEquals(len(listing), 4) self.assertEquals([row[0] for row in listing], ['0/', '1/', '2/', '3/']) - listing = broker.list_containers_iter(10, '2/', None, '/') + listing = broker.list_containers_iter(10, '2/', None, None, '/') self.assertEquals(len(listing), 1) self.assertEquals([row[0] for row in listing], ['3/']) - listing = broker.list_containers_iter(10, '', '2', '/') + listing = broker.list_containers_iter(10, '', None, '2', '/') self.assertEquals(len(listing), 1) self.assertEquals([row[0] for row in listing], ['2/']) - listing = broker.list_containers_iter(10, '2/0050', '2/', '/') + listing = broker.list_containers_iter(10, '2/0050', None, '2/', '/') self.assertEquals(len(listing), 10) self.assertEquals(listing[0][0], '2/0051') self.assertEquals(listing[1][0], '2/0051/') self.assertEquals(listing[2][0], '2/0052') self.assertEquals(listing[-1][0], '2/0059') - listing = broker.list_containers_iter(10, '3/0045', '3/', '/') + listing = broker.list_containers_iter(10, '3/0045', None, '3/', '/') self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3/0045/', '3/0046', '3/0046/', '3/0047', @@ -1704,21 +1715,21 @@ class TestAccountBroker(unittest.TestCase): '3/0049/', '3/0050']) broker.put_container('3/0049/', normalize_timestamp(time()), 0, 0, 0) - listing = broker.list_containers_iter(10, '3/0048', None, None) + listing = broker.list_containers_iter(10, '3/0048', None, None, None) self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3/0048/0049', '3/0049', '3/0049/', '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049', '3/0052', '3/0052/0049']) - listing = broker.list_containers_iter(10, '3/0048', '3/', '/') + listing = broker.list_containers_iter(10, '3/0048', None, '3/', '/') self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['3/0048/', '3/0049', '3/0049/', '3/0050', '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053']) - listing = broker.list_containers_iter(10, None, '3/0049/', '/') + listing = broker.list_containers_iter(10, None, None, '3/0049/', '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['3/0049/', '3/0049/0049']) @@ -1738,20 +1749,20 @@ class TestAccountBroker(unittest.TestCase): broker.put_container('b/a', normalize_timestamp(time()), 0, 0, 0) broker.put_container('b/b', normalize_timestamp(time()), 0, 0, 0) broker.put_container('c', normalize_timestamp(time()), 0, 0, 0) - listing = broker.list_containers_iter(15, None, None, None) + listing = broker.list_containers_iter(15, None, None, None, None) self.assertEquals(len(listing), 10) self.assertEquals([row[0] for row in listing], ['a', 'a/', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b', 'b/a', 'b/b', 'c']) - listing = broker.list_containers_iter(15, None, '', '/') + listing = broker.list_containers_iter(15, None, None, '', '/') self.assertEquals(len(listing), 5) self.assertEquals([row[0] for row in listing], ['a', 'a/', 'b', 'b/', 'c']) - listing = broker.list_containers_iter(15, None, 'a/', '/') + listing = broker.list_containers_iter(15, None, None, 'a/', '/') self.assertEquals(len(listing), 4) self.assertEquals([row[0] for row in listing], ['a/', 'a/a', 'a/a/', 'a/b']) - listing = broker.list_containers_iter(15, None, 'b/', '/') + listing = broker.list_containers_iter(15, None, None, 'b/', '/') self.assertEquals(len(listing), 2) self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])