Add non-slash delimiter tests

Also expand the logic following the select statement in
list_objects_iter and list_containers_iter to document the state
following the prefix and delimiter tests.

Also remove doc strings which obscure test names when run with
"./.unittests" script.

Performance Note:

The following simple python program illustrates the performance
advantage of not attempting to invoke startswith() method of a string
when we know the argument is an empty string.

    import time

    curs = []
    for i in range(10000):
        curs.append(['nameofthing%d'%(i), 1, 2, 3, 4])

    beg = time.time()
    for i in range(1000):
        x = [r for r in curs]
    end = time.time()
    print "without: %.2f" % (end - beg)

    beg = time.time()
    for i in range(1000):
        y = [r for r in curs if r[0].startswith('')]
    end = time.time()
    print "with: %.2f" % (end - beg)

    # Output:
    # $ python ~/test_startswith.py
    # without: 0.60
    # with: 2.73

Change-Id: I1ac72cd7932953b0efe7db8ff591829b985b8d73
Signed-off-by: Peter Portante <peter.portante@redhat.com>
This commit is contained in:
Peter Portante 2013-06-04 16:45:42 -04:00
parent 4077252f23
commit 483c5ec4c6
2 changed files with 246 additions and 28 deletions

View File

@ -1116,9 +1116,20 @@ class ContainerBroker(DatabaseBroker):
curs.row_factory = None
if prefix is None:
# A delimiter without a specified prefix is ignored
return [r for r in curs]
if not delimiter:
return [r for r in curs if r[0].startswith(prefix)]
if not prefix:
# It is possible to have a delimiter but no prefix
# specified. As above, the prefix will be set to the
# empty string, so avoid performing the extra work to
# check against an empty prefix.
return [r for r in curs]
else:
return [r for r in curs if r[0].startswith(prefix)]
# We have a delimiter and a prefix (possibly empty string) to
# handle
rowcount = 0
for row in curs:
rowcount += 1
@ -1589,9 +1600,20 @@ class AccountBroker(DatabaseBroker):
curs.row_factory = None
if prefix is None:
# A delimiter without a specified prefix is ignored
return [r for r in curs]
if not delimiter:
return [r for r in curs if r[0].startswith(prefix)]
if not prefix:
# It is possible to have a delimiter but no prefix
# specified. As above, the prefix will be set to the
# empty string, so avoid performing the extra work to
# check against an empty prefix.
return [r for r in curs]
else:
return [r for r in curs if r[0].startswith(prefix)]
# We have a delimiter and a prefix (possibly empty string) to
# handle
rowcount = 0
for row in curs:
rowcount += 1

View File

@ -611,7 +611,7 @@ class TestContainerBroker(unittest.TestCase):
""" Tests for swift.common.db.ContainerBroker """
def test_creation(self):
""" Test swift.common.db.ContainerBroker.__init__ """
# Test swift.common.db.ContainerBroker.__init__
broker = ContainerBroker(':memory:', account='a', container='c')
self.assertEqual(broker.db_file, ':memory:')
broker.initialize(normalize_timestamp('1'))
@ -621,8 +621,8 @@ class TestContainerBroker(unittest.TestCase):
self.assertEqual(curs.fetchall()[0][0], 1)
def test_exception(self):
""" Test swift.common.db.ContainerBroker throwing a conn away after
unhandled exception """
# Test swift.common.db.ContainerBroker throwing a conn away after
# unhandled exception
first_conn = None
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
@ -637,7 +637,7 @@ class TestContainerBroker(unittest.TestCase):
self.assert_(broker.conn is None)
def test_empty(self):
""" Test swift.common.db.ContainerBroker.empty """
# Test swift.common.db.ContainerBroker.empty
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
self.assert_(broker.empty())
@ -709,7 +709,7 @@ class TestContainerBroker(unittest.TestCase):
broker.delete_db(normalize_timestamp(time()))
def test_delete_object(self):
""" Test swift.common.db.ContainerBroker.delete_object """
# Test swift.common.db.ContainerBroker.delete_object
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain',
@ -732,7 +732,7 @@ class TestContainerBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 1)
def test_put_object(self):
""" Test swift.common.db.ContainerBroker.put_object """
# Test swift.common.db.ContainerBroker.put_object
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
@ -931,7 +931,7 @@ class TestContainerBroker(unittest.TestCase):
"SELECT deleted FROM object").fetchone()[0], 0)
def test_get_info(self):
""" Test swift.common.db.ContainerBroker.get_info """
# Test swift.common.db.ContainerBroker.get_info
broker = ContainerBroker(':memory:', account='test1',
container='test2')
broker.initialize(normalize_timestamp('1'))
@ -1063,7 +1063,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEquals(info['reported_bytes_used'], 1123)
def test_list_objects_iter(self):
""" Test swift.common.db.ContainerBroker.list_objects_iter """
# Test swift.common.db.ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
for obj1 in xrange(4):
@ -1111,6 +1111,12 @@ class TestContainerBroker(unittest.TestCase):
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0009')
# Same as above, but using the path argument.
listing = broker.list_objects_iter(10, '', None, None, '', '0')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0/0000')
self.assertEquals(listing[-1][0], '0/0009')
listing = broker.list_objects_iter(10, '', None, '', '/')
self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing],
@ -1171,8 +1177,123 @@ class TestContainerBroker(unittest.TestCase):
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001'])
def test_list_objects_iter_non_slash(self):
# Test swift.common.db.ContainerBroker.list_objects_iter using a
# delimiter that is not a slash
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
for obj1 in xrange(4):
for obj2 in xrange(125):
broker.put_object('%d:%04d' % (obj1, obj2),
normalize_timestamp(time()), 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
for obj in xrange(125):
broker.put_object('2:0051:%04d' % obj,
normalize_timestamp(time()), 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
for obj in xrange(125):
broker.put_object('3:%04d:0049' % obj,
normalize_timestamp(time()), 0, 'text/plain',
'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(100, '', None, None, '')
self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0:0000')
self.assertEquals(listing[-1][0], '0:0099')
listing = broker.list_objects_iter(100, '', '0:0050', None, '')
self.assertEquals(len(listing), 50)
self.assertEquals(listing[0][0], '0:0000')
self.assertEquals(listing[-1][0], '0:0049')
listing = broker.list_objects_iter(100, '0:0099', None, None, '')
self.assertEquals(len(listing), 100)
self.assertEquals(listing[0][0], '0:0100')
self.assertEquals(listing[-1][0], '1:0074')
listing = broker.list_objects_iter(55, '1:0074', None, None, '')
self.assertEquals(len(listing), 55)
self.assertEquals(listing[0][0], '1:0075')
self.assertEquals(listing[-1][0], '2:0004')
listing = broker.list_objects_iter(10, '', None, '0:01', '')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0:0100')
self.assertEquals(listing[-1][0], '0:0109')
listing = broker.list_objects_iter(10, '', None, '0:', ':')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '0:0000')
self.assertEquals(listing[-1][0], '0:0009')
# Same as above, but using the path argument, so nothing should be
# returned since path uses a '/' as a delimiter.
listing = broker.list_objects_iter(10, '', None, None, '', '0')
self.assertEquals(len(listing), 0)
listing = broker.list_objects_iter(10, '', None, '', ':')
self.assertEquals(len(listing), 4)
self.assertEquals([row[0] for row in listing],
['0:', '1:', '2:', '3:'])
listing = broker.list_objects_iter(10, '2', None, None, ':')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['2:', '3:'])
listing = broker.list_objects_iter(10, '2:', None, None, ':')
self.assertEquals(len(listing), 1)
self.assertEquals([row[0] for row in listing], ['3:'])
listing = broker.list_objects_iter(10, '2:0050', None, '2:', ':')
self.assertEquals(len(listing), 10)
self.assertEquals(listing[0][0], '2:0051')
self.assertEquals(listing[1][0], '2:0051:')
self.assertEquals(listing[2][0], '2:0052')
self.assertEquals(listing[-1][0], '2:0059')
listing = broker.list_objects_iter(10, '3:0045', None, '3:', ':')
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3:0045:', '3:0046', '3:0046:', '3:0047',
'3:0047:', '3:0048', '3:0048:', '3:0049',
'3:0049:', '3:0050'])
broker.put_object('3:0049:', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(10, '3:0048', None, None, None)
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3:0048:0049', '3:0049', '3:0049:',
'3:0049:0049', '3:0050', '3:0050:0049', '3:0051', '3:0051:0049',
'3:0052', '3:0052:0049'])
listing = broker.list_objects_iter(10, '3:0048', None, '3:', ':')
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['3:0048:', '3:0049', '3:0049:', '3:0050',
'3:0050:', '3:0051', '3:0051:', '3:0052', '3:0052:', '3:0053'])
listing = broker.list_objects_iter(10, None, None, '3:0049:', ':')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing],
['3:0049:', '3:0049:0049'])
# Same as above, but using the path argument, so nothing should be
# returned since path uses a '/' as a delimiter.
listing = broker.list_objects_iter(10, None, None, None, None,
'3:0049')
self.assertEquals(len(listing), 0)
listing = broker.list_objects_iter(2, None, None, '3:', ':')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['3:0000', '3:0000:'])
listing = broker.list_objects_iter(2, None, None, None, None, '3')
self.assertEquals(len(listing), 0)
def test_list_objects_iter_prefix_delim(self):
""" Test swift.common.db.ContainerBroker.list_objects_iter """
# Test swift.common.db.ContainerBroker.list_objects_iter
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
@ -1202,8 +1323,8 @@ class TestContainerBroker(unittest.TestCase):
['/pets/fish/a', '/pets/fish/b'])
def test_double_check_trailing_delimiter(self):
""" Test swift.common.db.ContainerBroker.list_objects_iter for a
container that has an odd file with a trailing delimiter """
# Test swift.common.db.ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
broker.put_object('a', normalize_timestamp(time()), 0,
@ -1276,6 +1397,81 @@ class TestContainerBroker(unittest.TestCase):
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])
def test_double_check_trailing_delimiter_non_slash(self):
# Test swift.common.db.ContainerBroker.list_objects_iter for a
# container that has an odd file with a trailing delimiter
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
broker.put_object('a', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('a:', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('a:a', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('a:a:a', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('a:a:b', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('a:b', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('b', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('b:a', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('b:b', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('c', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('a:0', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('0', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('0:', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('00', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('0:0', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('0:00', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('0:1', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('0:1:', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('0:1:0', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('1', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('1:', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
broker.put_object('1:0', normalize_timestamp(time()), 0,
'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')
listing = broker.list_objects_iter(25, None, None, None, None)
self.assertEquals(len(listing), 22)
self.assertEquals([row[0] for row in listing],
['0', '00', '0:', '0:0', '0:00', '0:1', '0:1:', '0:1:0', '1', '1:',
'1:0', 'a', 'a:', 'a:0', 'a:a', 'a:a:a', 'a:a:b', 'a:b', 'b', 'b:a',
'b:b', 'c'])
listing = broker.list_objects_iter(25, None, None, '', ':')
self.assertEquals(len(listing), 10)
self.assertEquals([row[0] for row in listing],
['0', '00', '0:', '1', '1:', 'a', 'a:', 'b', 'b:', 'c'])
listing = broker.list_objects_iter(25, None, None, 'a:', ':')
self.assertEquals(len(listing), 5)
self.assertEquals([row[0] for row in listing],
['a:', 'a:0', 'a:a', 'a:a:', 'a:b'])
listing = broker.list_objects_iter(25, None, None, '0:', ':')
self.assertEquals(len(listing), 5)
self.assertEquals([row[0] for row in listing],
['0:', '0:0', '0:00', '0:1', '0:1:'])
listing = broker.list_objects_iter(25, None, None, '0:1:', ':')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing],
['0:1:', '0:1:0'])
listing = broker.list_objects_iter(25, None, None, 'b:', ':')
self.assertEquals(len(listing), 2)
self.assertEquals([row[0] for row in listing], ['b:a', 'b:b'])
def test_chexor(self):
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
@ -1296,7 +1492,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEquals(broker.get_info()['hash'], hashc)
def test_newid(self):
"""test DatabaseBroker.newid"""
# test DatabaseBroker.newid
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
id = broker.get_info()['id']
@ -1304,7 +1500,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertNotEquals(id, broker.get_info()['id'])
def test_get_items_since(self):
"""test DatabaseBroker.get_items_since"""
# test DatabaseBroker.get_items_since
broker = ContainerBroker(':memory:', account='a', container='c')
broker.initialize(normalize_timestamp('1'))
broker.put_object('a', normalize_timestamp(1), 0,
@ -1317,7 +1513,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEquals(items[0]['name'], 'b')
def test_sync_merging(self):
""" exercise the DatabaseBroker sync functions a bit """
# exercise the DatabaseBroker sync functions a bit
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1.initialize(normalize_timestamp('1'))
broker2 = ContainerBroker(':memory:', account='a', container='c')
@ -1352,7 +1548,7 @@ class TestContainerBroker(unittest.TestCase):
sorted([rec['name'] for rec in items]))
def test_merge_items_overwrite(self):
"""test DatabaseBroker.merge_items"""
# test DatabaseBroker.merge_items
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1.initialize(normalize_timestamp('1'))
id = broker1.get_info()['id']
@ -1377,7 +1573,7 @@ class TestContainerBroker(unittest.TestCase):
self.assertEquals(rec['created_at'], normalize_timestamp(3))
def test_merge_items_post_overwrite_out_of_order(self):
"""test DatabaseBroker.merge_items"""
# test DatabaseBroker.merge_items
broker1 = ContainerBroker(':memory:', account='a', container='c')
broker1.initialize(normalize_timestamp('1'))
id = broker1.get_info()['id']
@ -1573,7 +1769,7 @@ class TestAccountBroker(unittest.TestCase):
""" Tests for swift.common.db.AccountBroker """
def test_creation(self):
""" Test swift.common.db.AccountBroker.__init__ """
# Test swift.common.db.AccountBroker.__init__
broker = AccountBroker(':memory:', account='a')
self.assertEqual(broker.db_file, ':memory:')
got_exc = False
@ -1590,8 +1786,8 @@ class TestAccountBroker(unittest.TestCase):
self.assertEqual(curs.fetchall()[0][0], 1)
def test_exception(self):
""" Test swift.common.db.AccountBroker throwing a conn away after
exception """
# Test swift.common.db.AccountBroker throwing a conn away after
# exception
first_conn = None
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
@ -1606,7 +1802,7 @@ class TestAccountBroker(unittest.TestCase):
self.assert_(broker.conn is None)
def test_empty(self):
""" Test swift.common.db.AccountBroker.empty """
# Test swift.common.db.AccountBroker.empty
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
self.assert_(broker.empty())
@ -1685,7 +1881,7 @@ class TestAccountBroker(unittest.TestCase):
# self.assert_('a' not in containers)
def test_delete_container(self):
""" Test swift.common.db.AccountBroker.delete_container """
# Test swift.common.db.AccountBroker.delete_container
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('o', normalize_timestamp(time()), 0, 0, 0)
@ -1707,7 +1903,7 @@ class TestAccountBroker(unittest.TestCase):
"WHERE deleted = 1").fetchone()[0], 1)
def test_put_container(self):
""" Test swift.common.db.AccountBroker.put_container """
# Test swift.common.db.AccountBroker.put_container
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
@ -1808,7 +2004,7 @@ class TestAccountBroker(unittest.TestCase):
"SELECT deleted FROM container").fetchone()[0], 0)
def test_get_info(self):
""" Test swift.common.db.AccountBroker.get_info """
# Test swift.common.db.AccountBroker.get_info
broker = AccountBroker(':memory:', account='test1')
broker.initialize(normalize_timestamp('1'))
@ -1844,7 +2040,7 @@ class TestAccountBroker(unittest.TestCase):
self.assertEquals(info['container_count'], 0)
def test_list_containers_iter(self):
""" Test swift.common.db.AccountBroker.list_containers_iter """
# Test swift.common.db.AccountBroker.list_containers_iter
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
for cont1 in xrange(4):
@ -1942,8 +2138,8 @@ class TestAccountBroker(unittest.TestCase):
['3-0049-', '3-0049-0049'])
def test_double_check_trailing_delimiter(self):
""" Test swift.common.db.AccountBroker.list_containers_iter for an
account that has an odd file with a trailing delimiter """
# Test swift.common.db.AccountBroker.list_containers_iter for an
# account that has an odd container with a trailing delimiter
broker = AccountBroker(':memory:', account='a')
broker.initialize(normalize_timestamp('1'))
broker.put_container('a', normalize_timestamp(time()), 0, 0, 0)