openstack/swift
Code Issues Proposed changes

Add container and account reverse listings

Browse Source 
This change adds the ability to tell the container or account server to
reverse their listings. This is done by sending a reverse=TRUE_VALUE,

Where TRUE_VALUE is one of the values true can be in common/utils:

  TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))

For example:

  curl -i -X GET -H "X-Auth-Token: $TOKEN" $STORAGE_URL/c/?reverse=on

I borrowed the swapping of the markers code from Kevin's old change,
thanks Kevin. And Tim Burke added some real nuggets of awesomeness.

DocImpact
Co-Authored-By: Kevin McDonald <kmcdonald@softlayer.com>
Co-Authored-By: Tim Burke <tim.burke@gmail.com>
Implements: blueprint reverse-object-listing

Change-Id: I5eb655360ac95042877da26d18707aebc11c02f6
This commit is contained in:
Matthew Oliver 2014-09-11 16:51:51 +10:00 committed by Alistair Coles
parent fdc8828e85
commit 7c1e6cd583
9 changed files with 620 additions and 44 deletions

19
doc/source/api/object_api_v1_overview.rst

@ -128,7 +128,24 @@ If you have a large number of containers or objects, you can use query
parameters to page through large lists of containers or objects. Use the
*``marker``*, *``limit``*, and *``end_marker``* query parameters to
control how many items are returned in a list and where the list starts
or ends.
or ends. If you want to page through in reverse order, you can use the query
parameter *``reverse``*, noting that your marker and end_markers will be
applied to a reverse listing should be switched. I.e, for a list of objects
``[a, b, c, d, e]`` the non-reversed could be:
.. code::
/v1/{account}/{container}/?marker=a&end_marker=d
b
c
However, when reversed marker and end_marker are applied to a reversed list:
.. code::
/v1/{account}/{container}/?marker=d&end_marker=a&reverse=on
c
b
Object Storage HTTP requests have the following default constraints.
Your service provider might use different default values.

48
swift/account/backend.py

@ -366,7 +366,7 @@ class AccountBroker(DatabaseBroker):
''').fetchone())
def list_containers_iter(self, limit, marker, end_marker, prefix,
delimiter):
delimiter, reverse=False):
"""
Get a list of containers sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not have
@ -377,15 +377,21 @@ class AccountBroker(DatabaseBroker):
:param end_marker: end marker query
:param prefix: prefix query
:param delimiter: delimiter for query
:param reverse: reverse the result order.
:returns: list of tuples of (name, object_count, bytes_used, 0)
"""
delim_force_gte = False
(marker, end_marker, prefix, delimiter) = utf8encode(
marker, end_marker, prefix, delimiter)
if reverse:
# Reverse the markers if we are reversing the listing.
marker, end_marker = end_marker, marker
self._commit_puts_stale_ok()
if delimiter and not prefix:
prefix = ''
if prefix:
end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1)
orig_marker = marker
with self.get() as conn:
results = []
@ -395,9 +401,13 @@ class AccountBroker(DatabaseBroker):
FROM container
WHERE """
query_args = []
if end_marker:
if end_marker and (not prefix or end_marker < end_prefix):
query += ' name < ? AND'
query_args.append(end_marker)
elif prefix:
query += ' name < ? AND'
query_args.append(end_prefix)
if delim_force_gte:
query += ' name >= ? AND'
query_args.append(marker)
@ -413,38 +423,40 @@ class AccountBroker(DatabaseBroker):
query += ' +deleted = 0'
else:
query += ' deleted = 0'
query += ' ORDER BY name LIMIT ?'
query += ' ORDER BY name %s LIMIT ?' % \
('DESC' if reverse else '')
query_args.append(limit - len(results))
curs = conn.execute(query, query_args)
curs.row_factory = None
if prefix is None:
# A delimiter without a specified prefix is ignored
# Delimiters without a prefix is ignored, further if there
# is no delimiter then we can simply return the result as
# prefixes are now handled in the SQL statement.
if prefix is None or not delimiter:
return [r for r in curs]
if not delimiter:
if not prefix:
# It is possible to have a delimiter but no prefix
# specified. As above, the prefix will be set to the
# empty string, so avoid performing the extra work to
# check against an empty prefix.
return [r for r in curs]
else:
return [r for r in curs if r[0].startswith(prefix)]
# We have a delimiter and a prefix (possibly empty string) to
# handle
rowcount = 0
for row in curs:
rowcount += 1
marker = name = row[0]
name = row[0]
if reverse:
end_marker = name
else:
marker = name
if len(results) >= limit or not name.startswith(prefix):
curs.close()
return results
end = name.find(delimiter, len(prefix))
if end > 0:
marker = name[:end] + chr(ord(delimiter) + 1)
# we want result to be inclusive of delim+1
delim_force_gte = True
if reverse:
end_marker = name[:end + 1]
else:
marker = name[:end] + chr(ord(delimiter) + 1)
# we want result to be inclusive of delim+1
delim_force_gte = True
dir_name = name[:end + 1]
if dir_name != orig_marker:
results.append([dir_name, 0, 0, 1])

3
swift/account/server.py

@ -191,6 +191,7 @@ class AccountController(BaseStorageServer):
return HTTPPreconditionFailed(body='Bad delimiter')
limit = constraints.ACCOUNT_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.ACCOUNT_LISTING_LIMIT:
@ -211,7 +212,7 @@ class AccountController(BaseStorageServer):
return self._deleted_response(broker, req, HTTPNotFound)
return account_listing_response(account, req, out_content_type, broker,
limit, marker, end_marker, prefix,
delimiter)
delimiter, reverse)
@public
@replication

4
swift/account/utils.py

@ -70,14 +70,14 @@ def get_response_headers(broker):
def account_listing_response(account, req, response_content_type, broker=None,
limit='', marker='', end_marker='', prefix='',
delimiter=''):
delimiter='', reverse=False):
if broker is None:
broker = FakeAccountBroker()
resp_headers = get_response_headers(broker)
account_list = broker.list_containers_iter(limit, marker, end_marker,
prefix, delimiter)
prefix, delimiter, reverse)
if response_content_type == 'application/json':
data = []
for (name, object_count, bytes_used, is_subdir) in account_list:

54
swift/container/backend.py

@ -557,7 +557,7 @@ class ContainerBroker(DatabaseBroker):
conn.commit()
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
path=None, storage_policy_index=0):
path=None, storage_policy_index=0, reverse=False):
"""
Get a list of objects sorted by name starting at marker onward, up
to limit entries. Entries will begin with the prefix and will not
@ -570,6 +570,7 @@ class ContainerBroker(DatabaseBroker):
:param delimiter: delimiter for query
:param path: if defined, will set the prefix and delimiter based on
the path
:param reverse: reverse the result order.
:returns: list of tuples of (name, created_at, size, content_type,
etag)
@ -578,6 +579,9 @@ class ContainerBroker(DatabaseBroker):
(marker, end_marker, prefix, delimiter, path) = utf8encode(
marker, end_marker, prefix, delimiter, path)
self._commit_puts_stale_ok()
if reverse:
# Reverse the markers if we are reversing the listing.
marker, end_marker = end_marker, marker
if path is not None:
prefix = path
if path:
@ -585,6 +589,8 @@ class ContainerBroker(DatabaseBroker):
delimiter = '/'
elif delimiter and not prefix:
prefix = ''
if prefix:
end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1)
orig_marker = marker
with self.get() as conn:
results = []
@ -592,9 +598,13 @@ class ContainerBroker(DatabaseBroker):
query = '''SELECT name, created_at, size, content_type, etag
FROM object WHERE'''
query_args = []
if end_marker:
if end_marker and (not prefix or end_marker < end_prefix):
query += ' name < ? AND'
query_args.append(end_marker)
elif prefix:
query += ' name < ? AND'
query_args.append(end_prefix)
if delim_force_gte:
query += ' name >= ? AND'
query_args.append(marker)
@ -611,8 +621,8 @@ class ContainerBroker(DatabaseBroker):
else:
query += ' deleted = 0'
orig_tail_query = '''
ORDER BY name LIMIT ?
'''
ORDER BY name %s LIMIT ?
''' % ('DESC' if reverse else '')
orig_tail_args = [limit - len(results)]
# storage policy filter
policy_tail_query = '''
@ -633,25 +643,23 @@ class ContainerBroker(DatabaseBroker):
tuple(query_args + tail_args))
curs.row_factory = None
if prefix is None:
# A delimiter without a specified prefix is ignored
# Delimiters without a prefix is ignored, further if there
# is no delimiter then we can simply return the result as
# prefixes are now handled in the SQL statement.
if prefix is None or not delimiter:
return [r for r in curs]
if not delimiter:
if not prefix:
# It is possible to have a delimiter but no prefix
# specified. As above, the prefix will be set to the
# empty string, so avoid performing the extra work to
# check against an empty prefix.
return [r for r in curs]
else:
return [r for r in curs if r[0].startswith(prefix)]
# We have a delimiter and a prefix (possibly empty string) to
# handle
rowcount = 0
for row in curs:
rowcount += 1
marker = name = row[0]
name = row[0]
if reverse:
end_marker = name
else:
marker = name
if len(results) >= limit or not name.startswith(prefix):
curs.close()
return results
@ -660,13 +668,19 @@ class ContainerBroker(DatabaseBroker):
if name == path:
continue
if end >= 0 and len(name) > end + len(delimiter):
marker = name[:end] + chr(ord(delimiter) + 1)
if reverse:
end_marker = name[:end + 1]
else:
marker = name[:end] + chr(ord(delimiter) + 1)
curs.close()
break
elif end > 0:
marker = name[:end] + chr(ord(delimiter) + 1)
# we want result to be inclusive of delim+1
delim_force_gte = True
if reverse:
end_marker = name[:end + 1]
else:
marker = name[:end] + chr(ord(delimiter) + 1)
# we want result to be inclusive of delim+1
delim_force_gte = True
dir_name = name[:end + 1]
if dir_name != orig_marker:
results.append([dir_name, '0', 0, None, ''])

3
swift/container/server.py

@ -452,6 +452,7 @@ class ContainerController(BaseStorageServer):
end_marker = get_param(req, 'end_marker')
limit = constraints.CONTAINER_LISTING_LIMIT
given_limit = get_param(req, 'limit')
reverse = config_true_value(get_param(req, 'reverse'))
if given_limit and given_limit.isdigit():
limit = int(given_limit)
if limit > constraints.CONTAINER_LISTING_LIMIT:
@ -471,7 +472,7 @@ class ContainerController(BaseStorageServer):
return HTTPNotFound(request=req, headers=resp_headers)
container_list = broker.list_objects_iter(
limit, marker, end_marker, prefix, delimiter, path,
storage_policy_index=info['storage_policy_index'])
storage_policy_index=info['storage_policy_index'], reverse=reverse)
return self.create_listing(req, out_content_type, info, resp_headers,
broker.metadata, container_list, container)

180
test/functional/tests.py

@ -327,6 +327,77 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
set_up = False
class TestAccountSortingEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
postfix = Utils.create_name()
cls.cont_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz')
cls.cont_items = ['%s%s' % (x, postfix) for x in cls.cont_items]
for container in cls.cont_items:
c = cls.account.container(container)
if not c.create():
raise ResponseError(cls.conn.response)
class TestAccountSorting(Base):
env = TestAccountSortingEnv
set_up = False
def testAccountContainerListSorting(self):
# name (byte order) sorting.
cont_list = sorted(self.env.cont_items)
cont_list.reverse()
cont_listing = self.env.account.containers(parms={'reverse': 'on'})
self.assert_status(200)
self.assertEqual(cont_list, cont_listing)
def testAccountContainerListSortingByPrefix(self):
cont_list = sorted(c for c in self.env.cont_items if c.startswith('a'))
cont_list.reverse()
cont_listing = self.env.account.containers(parms={
'reverse': 'on', 'prefix': 'a'})
self.assert_status(200)
self.assertEqual(cont_list, cont_listing)
def testAccountContainerListSortingByMarkersExclusive(self):
first_item = self.env.cont_items[3] # 'b1' + postfix
last_item = self.env.cont_items[4] # 'B2' + postfix
cont_list = sorted(c for c in self.env.cont_items
if last_item < c < first_item)
cont_list.reverse()
cont_listing = self.env.account.containers(parms={
'reverse': 'on', 'marker': first_item, 'end_marker': last_item})
self.assert_status(200)
self.assertEqual(cont_list, cont_listing)
def testAccountContainerListSortingByMarkersInclusive(self):
first_item = self.env.cont_items[3] # 'b1' + postfix
last_item = self.env.cont_items[4] # 'B2' + postfix
cont_list = sorted(c for c in self.env.cont_items
if last_item <= c <= first_item)
cont_list.reverse()
cont_listing = self.env.account.containers(parms={
'reverse': 'on', 'marker': first_item + '\x00',
'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)})
self.assert_status(200)
self.assertEqual(cont_list, cont_listing)
def testAccountContainerListSortingByReversedMarkers(self):
cont_listing = self.env.account.containers(parms={
'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'})
self.assert_status(204)
self.assertEqual([], cont_listing)
class TestContainerEnv(object):
@classmethod
def setUp(cls):
@ -647,6 +718,115 @@ class TestContainerUTF8(Base2, TestContainer):
set_up = False
class TestContainerSortingEnv(object):
@classmethod
def setUp(cls):
cls.conn = Connection(tf.config)
cls.conn.authenticate()
cls.account = Account(cls.conn, tf.config.get('account',
tf.config['username']))
cls.account.delete_containers()
cls.container = cls.account.container(Utils.create_name())
if not cls.container.create():
raise ResponseError(cls.conn.response)
cls.file_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz')
cls.files = list()
cls.file_size = 128
for name in cls.file_items:
file_item = cls.container.file(name)
file_item.write_random(cls.file_size)
cls.files.append(file_item.name)
class TestContainerSorting(Base):
env = TestContainerSortingEnv
set_up = False
def testContainerFileListSortingReversed(self):
file_list = list(sorted(self.env.file_items))
file_list.reverse()
cont_files = self.env.container.files(parms={'reverse': 'on'})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
def testContainerFileSortingByPrefixReversed(self):
cont_list = sorted(c for c in self.env.file_items if c.startswith('a'))
cont_list.reverse()
cont_listing = self.env.container.files(parms={
'reverse': 'on', 'prefix': 'a'})
self.assert_status(200)
self.assertEqual(cont_list, cont_listing)
def testContainerFileSortingByMarkersExclusiveReversed(self):
first_item = self.env.file_items[3] # 'b1' + postfix
last_item = self.env.file_items[4] # 'B2' + postfix
cont_list = sorted(c for c in self.env.file_items
if last_item < c < first_item)
cont_list.reverse()
cont_listing = self.env.container.files(parms={
'reverse': 'on', 'marker': first_item, 'end_marker': last_item})
self.assert_status(200)
self.assertEqual(cont_list, cont_listing)
def testContainerFileSortingByMarkersInclusiveReversed(self):
first_item = self.env.file_items[3] # 'b1' + postfix
last_item = self.env.file_items[4] # 'B2' + postfix
cont_list = sorted(c for c in self.env.file_items
if last_item <= c <= first_item)
cont_list.reverse()
cont_listing = self.env.container.files(parms={
'reverse': 'on', 'marker': first_item + '\x00',
'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)})
self.assert_status(200)
self.assertEqual(cont_list, cont_listing)
def testContainerFileSortingByReversedMarkersReversed(self):
cont_listing = self.env.container.files(parms={
'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'})
self.assert_status(204)
self.assertEqual([], cont_listing)
def testContainerFileListSorting(self):
file_list = list(sorted(self.env.file_items))
cont_files = self.env.container.files()
self.assert_status(200)
self.assertEqual(file_list, cont_files)
# Lets try again but with reverse is specifically turned off
cont_files = self.env.container.files(parms={'reverse': 'off'})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
cont_files = self.env.container.files(parms={'reverse': 'false'})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
cont_files = self.env.container.files(parms={'reverse': 'no'})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
cont_files = self.env.container.files(parms={'reverse': ''})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
# Lets try again but with a incorrect reverse values
cont_files = self.env.container.files(parms={'reverse': 'foo'})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
cont_files = self.env.container.files(parms={'reverse': 'hai'})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
cont_files = self.env.container.files(parms={'reverse': 'o=[]::::>'})
self.assert_status(200)
self.assertEqual(file_list, cont_files)
class TestContainerPathsEnv(object):
@classmethod
def setUp(cls):

169
test/unit/account/test_backend.py

@ -416,20 +416,48 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = broker.list_containers_iter(10, '', None, '0-00', '-',
reverse=True)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0099')
self.assertEqual(listing[-1][0], '0-0090')
listing = broker.list_containers_iter(10, '', None, '0-', '-')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0009')
listing = broker.list_containers_iter(10, '', None, '0-', '-',
reverse=True)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0124')
self.assertEqual(listing[-1][0], '0-0115')
listing = broker.list_containers_iter(10, '', None, '', '-')
self.assertEqual(len(listing), 4)
self.assertEqual([row[0] for row in listing],
['0-', '1-', '2-', '3-'])
listing = broker.list_containers_iter(10, '', None, '', '-',
reverse=True)
self.assertEqual(len(listing), 4)
self.assertEqual([row[0] for row in listing],
['3-', '2-', '1-', '0-'])
listing = broker.list_containers_iter(10, '2-', None, None, '-')
self.assertEqual(len(listing), 1)
self.assertEqual([row[0] for row in listing], ['3-'])
listing = broker.list_containers_iter(10, '2-', None, None, '-',
reverse=True)
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing], ['1-', '0-'])
listing = broker.list_containers_iter(10, '2.', None, None, '-',
reverse=True)
self.assertEqual(len(listing), 3)
self.assertEqual([row[0] for row in listing], ['2-', '1-', '0-'])
listing = broker.list_containers_iter(10, '', None, '2', '-')
self.assertEqual(len(listing), 1)
self.assertEqual([row[0] for row in listing], ['2-'])
@ -469,6 +497,147 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing],
['3-0049-', '3-0049-0049'])
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
broker = AccountBroker(':memory:', account='a')
broker.initialize(Timestamp('1').internal, 0)
broker.put_container(
'c1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
broker.put_container(
'c10', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
broker.put_container(
'C1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
broker.put_container(
'c2', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
broker.put_container(
'c3', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
broker.put_container(
'C4', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
listing = broker.list_containers_iter(100, None, None, '', '',
reverse=False)
self.assertEqual([row[0] for row in listing],
['C1', 'C4', 'c1', 'c10', 'c2', 'c3'])
listing = broker.list_containers_iter(100, None, None, '', '',
reverse=True)
self.assertEqual([row[0] for row in listing],
['c3', 'c2', 'c10', 'c1', 'C4', 'C1'])
listing = broker.list_containers_iter(2, None, None, '', '',
reverse=True)
self.assertEqual([row[0] for row in listing],
['c3', 'c2'])
listing = broker.list_containers_iter(100, 'c2', 'C4', '', '',
reverse=True)
self.assertEqual([row[0] for row in listing],
['c10', 'c1'])
def test_reverse_prefix_delim(self):
expectations = [
{
'containers': [
'topdir1-subdir1,0-c1',
'topdir1-subdir1,1-c1',
'topdir1-subdir1-c1',
],
'params': {
'prefix': 'topdir1-',
'delimiter': '-',
},
'expected': [
'topdir1-subdir1,0-',
'topdir1-subdir1,1-',
'topdir1-subdir1-',
],
},
{
'containers': [
'topdir1-subdir1,0-c1',
'topdir1-subdir1,1-c1',
'topdir1-subdir1-c1',
'topdir1-subdir1.',
'topdir1-subdir1.-c1',
],
'params': {
'prefix': 'topdir1-',
'delimiter': '-',
},
'expected': [
'topdir1-subdir1,0-',
'topdir1-subdir1,1-',
'topdir1-subdir1-',
'topdir1-subdir1.',
'topdir1-subdir1.-',
],
},
{
'containers': [
'topdir1-subdir1-c1',
'topdir1-subdir1,0-c1',
'topdir1-subdir1,1-c1',
],
'params': {
'prefix': 'topdir1-',
'delimiter': '-',
'reverse': True,
},
'expected': [
'topdir1-subdir1-',
'topdir1-subdir1,1-',
'topdir1-subdir1,0-',
],
},
{
'containers': [
'topdir1-subdir1.-c1',
'topdir1-subdir1.',
'topdir1-subdir1-c1',
'topdir1-subdir1-',
'topdir1-subdir1,',
'topdir1-subdir1,0-c1',
'topdir1-subdir1,1-c1',
],
'params': {
'prefix': 'topdir1-',
'delimiter': '-',
'reverse': True,
},
'expected': [
'topdir1-subdir1.-',
'topdir1-subdir1.',
'topdir1-subdir1-',
'topdir1-subdir1,1-',
'topdir1-subdir1,0-',
'topdir1-subdir1,',
],
},
]
ts = make_timestamp_iter()
default_listing_params = {
'limit': 10000,
'marker': '',
'end_marker': None,
'prefix': None,
'delimiter': None,
}
failures = []
for expected in expectations:
broker = AccountBroker(':memory:', account='a')
broker.initialize(next(ts).internal, 0)
for name in expected['containers']:
broker.put_container(name, next(ts).internal, 0, 0, 0,
POLICIES.default.idx)
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(c[0] for c in broker.list_containers_iter(**params))
if listing != expected['expected']:
expected['listing'] = listing
failures.append(
"With containers %(containers)r, the params %(params)r "
"produced %(listing)r instead of %(expected)r" % expected)
self.assertFalse(failures, "Found the following failures:\n%s" %
'\n'.join(failures))
def test_double_check_trailing_delimiter(self):
# Test AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter

184
test/unit/container/test_backend.py

@ -34,7 +34,8 @@ from swift.common.storage_policy import POLICIES
import mock
from test.unit import patch_policies, with_tempdir
from test.unit import (patch_policies, with_tempdir, make_timestamp_iter,
EMPTY_ETAG)
from test.unit.common.test_db import TestExampleBroker
@ -773,6 +774,12 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(listing[0][0], '1/0075')
self.assertEqual(listing[-1][0], '2/0004')
listing = broker.list_objects_iter(55, '2/0005', None, None, '',
reverse=True)
self.assertEqual(len(listing), 55)
self.assertEqual(listing[0][0], '2/0004')
self.assertEqual(listing[-1][0], '1/0075')
listing = broker.list_objects_iter(10, '', None, '0/01', '')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0/0100')
@ -783,17 +790,34 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(listing[0][0], '0/0000')
self.assertEqual(listing[-1][0], '0/0009')
listing = broker.list_objects_iter(10, '', None, '0/', '/',
reverse=True)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0/0124')
self.assertEqual(listing[-1][0], '0/0115')
# Same as above, but using the path argument.
listing = broker.list_objects_iter(10, '', None, None, '', '0')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0/0000')
self.assertEqual(listing[-1][0], '0/0009')
listing = broker.list_objects_iter(10, '', None, None, '', '0',
reverse=True)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0/0124')
self.assertEqual(listing[-1][0], '0/0115')
listing = broker.list_objects_iter(10, '', None, '', '/')
self.assertEqual(len(listing), 4)
self.assertEqual([row[0] for row in listing],
['0/', '1/', '2/', '3/'])
listing = broker.list_objects_iter(10, '', None, '', '/', reverse=True)
self.assertEqual(len(listing), 4)
self.assertEqual([row[0] for row in listing],
['3/', '2/', '1/', '0/'])
listing = broker.list_objects_iter(10, '2', None, None, '/')
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing], ['2/', '3/'])
@ -802,6 +826,16 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(len(listing), 1)
self.assertEqual([row[0] for row in listing], ['3/'])
listing = broker.list_objects_iter(10, '2/', None, None, '/',
reverse=True)
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing], ['1/', '0/'])
listing = broker.list_objects_iter(10, '20', None, None, '/',
reverse=True)
self.assertEqual(len(listing), 3)
self.assertEqual([row[0] for row in listing], ['2/', '1/', '0/'])
listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/')
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '2/0051')
@ -852,6 +886,113 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(len(listing), 2)
self.assertEqual([row[0] for row in listing], ['3/0000', '3/0001'])
def test_reverse_prefix_delim(self):
expectations = [
{
'objects': [
'topdir1/subdir1.0/obj1',
'topdir1/subdir1.1/obj1',
'topdir1/subdir1/obj1',
],
'params': {
'prefix': 'topdir1/',
'delimiter': '/',
},
'expected': [
'topdir1/subdir1.0/',
'topdir1/subdir1.1/',
'topdir1/subdir1/',
],
},
{
'objects': [
'topdir1/subdir1.0/obj1',
'topdir1/subdir1.1/obj1',
'topdir1/subdir1/obj1',
'topdir1/subdir10',
'topdir1/subdir10/obj1',
],
'params': {
'prefix': 'topdir1/',
'delimiter': '/',
},
'expected': [
'topdir1/subdir1.0/',
'topdir1/subdir1.1/',
'topdir1/subdir1/',
'topdir1/subdir10',
'topdir1/subdir10/',
],
},
{
'objects': [
'topdir1/subdir1/obj1',
'topdir1/subdir1.0/obj1',
'topdir1/subdir1.1/obj1',
],
'params': {
'prefix': 'topdir1/',
'delimiter': '/',
'reverse': True,
},
'expected': [
'topdir1/subdir1/',
'topdir1/subdir1.1/',
'topdir1/subdir1.0/',
],
},
{
'objects': [
'topdir1/subdir10/obj1',
'topdir1/subdir10',
'topdir1/subdir1/obj1',
'topdir1/subdir1.0/obj1',
'topdir1/subdir1.1/obj1',
],
'params': {
'prefix': 'topdir1/',
'delimiter': '/',
'reverse': True,
},
'expected': [
'topdir1/subdir10/',
'topdir1/subdir10',
'topdir1/subdir1/',
'topdir1/subdir1.1/',
'topdir1/subdir1.0/',
],
},
]
ts = make_timestamp_iter()
default_listing_params = {
'limit': 10000,
'marker': '',
'end_marker': None,
'prefix': None,
'delimiter': None,
}
obj_create_params = {
'size': 0,
'content_type': 'application/test',
'etag': EMPTY_ETAG,
}
failures = []
for expected in expectations:
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(next(ts).internal, 0)
for name in expected['objects']:
broker.put_object(name, next(ts).internal, **obj_create_params)
params = default_listing_params.copy()
params.update(expected['params'])
listing = list(o[0] for o in broker.list_objects_iter(**params))
if listing != expected['expected']:
expected['listing'] = listing
failures.append(
"With objects %(objects)r, the params %(params)r "
"produced %(listing)r instead of %(expected)r" % expected)
self.assertFalse(failures, "Found the following failures:\n%s" %
'\n'.join(failures))
def test_list_objects_iter_non_slash(self):
# Test ContainerBroker.list_objects_iter using a
# delimiter that is not a slash
@ -1006,6 +1147,47 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual([row[0] for row in listing],
['/pets/fish/a', '/pets/fish/b'])
def test_list_objects_iter_order_and_reverse(self):
# Test ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(Timestamp('1').internal, 0)
broker.put_object(
'o1', Timestamp(0).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object(
'o10', Timestamp(0).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object(
'O1', Timestamp(0).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object(
'o2', Timestamp(0).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object(
'o3', Timestamp(0).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object(
'O4', Timestamp(0).internal, 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(100, None, None, '', '',
reverse=False)
self.assertEqual([row[0] for row in listing],
['O1', 'O4', 'o1', 'o10', 'o2', 'o3'])
listing = broker.list_objects_iter(100, None, None, '', '',
reverse=True)
self.assertEqual([row[0] for row in listing],
['o3', 'o2', 'o10', 'o1', 'O4', 'O1'])
listing = broker.list_objects_iter(2, None, None, '', '',
reverse=True)
self.assertEqual([row[0] for row in listing],
['o3', 'o2'])
listing = broker.list_objects_iter(100, 'o2', 'O4', '', '',
reverse=True)
self.assertEqual([row[0] for row in listing],
['o10', 'o1'])
def test_double_check_trailing_delimiter(self):
# Test ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter