openstack/swift
Code Issues Proposed changes

Refactor timestamp iter in db backend tests

Browse Source 
Change-Id: I61d044a6d778e158af73a3f4d9fc75d642af0beb
This commit is contained in:
Clay Gerrard
2019-10-25 14:34:41 -05:00
parent 739c370410
commit b75d593935
4 changed files with 255 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/unit/account/test_auditor.py
View File

@@ -170,7 +170,7 @@ class TestAuditorRealBrokerMigration(
for i in range(num_containers):
name = 'test-container-%02d' % i
policy = next(policies)
self.broker.put_container(name, next(self.ts),
self.broker.put_container(name, next(self.ts).internal,
0, 0, 0, int(policy))
per_policy_container_counts[int(policy)] += 1

95
test/unit/account/test_backend.py
View File

@@ -47,6 +47,10 @@ from test.unit.common import test_db
class TestAccountBroker(unittest.TestCase):
"""Tests for AccountBroker"""
def setUp(self):
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
def test_creation(self):
# Test AccountBroker.__init__
broker = AccountBroker(':memory:', account='a')
@@ -176,28 +180,26 @@ class TestAccountBroker(unittest.TestCase):
broker.reclaim(Timestamp.now().internal, time())
def test_delete_db_status(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
start = next(ts)
start = next(self.ts)
broker = AccountBroker(':memory:', account='a')
broker.initialize(start)
broker.initialize(start.internal)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assertEqual(info['put_timestamp'], start.internal)
self.assertGreaterEqual(Timestamp(info['created_at']), start)
self.assertEqual(info['delete_timestamp'], '0')
if self.__class__ == TestAccountBrokerBeforeMetadata:
self.assertEqual(info['status_changed_at'], '0')
else:
self.assertEqual(info['status_changed_at'],
Timestamp(start).internal)
self.assertEqual(info['status_changed_at'], start.internal)
# delete it
delete_timestamp = next(ts)
broker.delete_db(delete_timestamp)
delete_timestamp = next(self.ts)
broker.delete_db(delete_timestamp.internal)
info = broker.get_info()
self.assertEqual(info['put_timestamp'], Timestamp(start).internal)
self.assertEqual(info['put_timestamp'], start.internal)
self.assertGreaterEqual(Timestamp(info['created_at']), start)
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], delete_timestamp)
self.assertEqual(info['delete_timestamp'], delete_timestamp.internal)
self.assertEqual(info['status_changed_at'], delete_timestamp.internal)
def test_delete_container(self):
# Test AccountBroker.delete_container
@@ -644,7 +646,6 @@ class TestAccountBroker(unittest.TestCase):
],
},
]
ts = make_timestamp_iter()
default_listing_params = {
'limit': 10000,
'marker': '',
@@ -655,9 +656,9 @@ class TestAccountBroker(unittest.TestCase):
failures = []
for expected in expectations:
broker = AccountBroker(':memory:', account='a')
broker.initialize(next(ts).internal, 0)
broker.initialize(next(self.ts).internal, 0)
for name in expected['containers']:
broker.put_container(name, next(ts).internal, 0, 0, 0,
broker.put_container(name, next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx)
params = default_listing_params.copy()
params.update(expected['params'])
@@ -874,9 +875,8 @@ class TestAccountBroker(unittest.TestCase):
StoragePolicy(2, 'two', False),
StoragePolicy(3, 'three', False)])
def test_get_policy_stats(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(':memory:', account='a')
broker.initialize(next(ts))
broker.initialize(next(self.ts).internal)
# check empty policy_stats
self.assertTrue(broker.empty())
policy_stats = broker.get_policy_stats()
@@ -885,9 +885,9 @@ class TestAccountBroker(unittest.TestCase):
# add some empty containers
for policy in POLICIES:
container_name = 'c-%s' % policy.name
put_timestamp = next(ts)
put_timestamp = next(self.ts)
broker.put_container(container_name,
put_timestamp, 0,
put_timestamp.internal, 0,
0, 0,
policy.idx)
policy_stats = broker.get_policy_stats()
@@ -900,10 +900,10 @@ class TestAccountBroker(unittest.TestCase):
# update the containers object & byte count
for policy in POLICIES:
container_name = 'c-%s' % policy.name
put_timestamp = next(ts)
put_timestamp = next(self.ts)
count = policy.idx * 100 # good as any integer
broker.put_container(container_name,
put_timestamp, 0,
put_timestamp.internal, 0,
count, count,
policy.idx)
@@ -926,9 +926,9 @@ class TestAccountBroker(unittest.TestCase):
# now delete the containers one by one
for policy in POLICIES:
container_name = 'c-%s' % policy.name
delete_timestamp = next(ts)
delete_timestamp = next(self.ts)
broker.put_container(container_name,
0, delete_timestamp,
0, delete_timestamp.internal,
0, 0,
policy.idx)
@@ -942,16 +942,15 @@ class TestAccountBroker(unittest.TestCase):
@patch_policies([StoragePolicy(0, 'zero', False),
StoragePolicy(1, 'one', True)])
def test_policy_stats_tracking(self):
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(':memory:', account='a')
broker.initialize(next(ts))
broker.initialize(next(self.ts).internal)
# policy 0
broker.put_container('con1', next(ts), 0, 12, 2798641, 0)
broker.put_container('con1', next(ts), 0, 13, 8156441, 0)
broker.put_container('con1', next(self.ts).internal, 0, 12, 2798641, 0)
broker.put_container('con1', next(self.ts).internal, 0, 13, 8156441, 0)
# policy 1
broker.put_container('con2', next(ts), 0, 7, 5751991, 1)
broker.put_container('con2', next(ts), 0, 8, 6085379, 1)
broker.put_container('con2', next(self.ts).internal, 0, 7, 5751991, 1)
broker.put_container('con2', next(self.ts).internal, 0, 8, 6085379, 1)
stats = broker.get_policy_stats()
self.assertEqual(len(stats), 2)
@@ -1051,6 +1050,8 @@ class TestAccountBrokerBeforeMetadata(TestAccountBroker):
"""
def setUp(self):
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_account_stat_table = \
AccountBroker.create_account_stat_table
AccountBroker.create_account_stat_table = \
@@ -1136,6 +1137,8 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
"""
def setUp(self):
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self._imported_create_container_table = \
AccountBroker.create_container_table
AccountBroker.create_container_table = \
@@ -1348,16 +1351,14 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
@with_tempdir
def test_half_upgraded_database(self, tempdir):
db_path = os.path.join(tempdir, 'account.db')
ts = itertools.count()
ts = (Timestamp(t).internal for t in itertools.count(int(time())))
broker = AccountBroker(db_path, account='a')
broker.initialize(next(ts))
broker.initialize(next(self.ts).internal)
self.assertTrue(broker.empty())
# add a container (to pending file)
broker.put_container('c', next(ts), 0, 0, 0,
broker.put_container('c', next(self.ts).internal, 0, 0, 0,
POLICIES.default.idx)
real_get = broker.get
@@ -1410,16 +1411,14 @@ class TestAccountBrokerBeforeSPI(TestAccountBroker):
@with_tempdir
def test_pre_storage_policy_replication(self, tempdir):
ts = make_timestamp_iter()
# make and two account database "replicas"
old_broker = AccountBroker(os.path.join(tempdir, 'old_account.db'),
account='a')
old_broker.initialize(next(ts).internal)
old_broker.initialize(next(self.ts).internal)
new_broker = AccountBroker(os.path.join(tempdir, 'new_account.db'),
account='a')
new_broker.initialize(next(ts).internal)
timestamp = next(ts).internal
new_broker.initialize(next(self.ts).internal)
timestamp = next(self.ts).internal
# manually insert an existing row to avoid migration for old database
with old_broker.get() as conn:
@@ -1581,12 +1580,13 @@ class AccountBrokerPreTrackContainerCountSetup(object):
self.assertUnmigrated(broker)
self.tempdir = mkdtemp()
self.ts = (Timestamp(t).internal for t in itertools.count(int(time())))
# tests seem to assume x-timestamp was set by the proxy before "now"
self.ts = make_timestamp_iter(offset=-1)
self.db_path = os.path.join(self.tempdir, 'sda', 'accounts',
'0', '0', '0', 'test.db')
self.broker = AccountBroker(self.db_path, account='a')
self.broker.initialize(next(self.ts))
self.broker.initialize(next(self.ts).internal)
# Common sanity-check that our starting, pre-migration state correctly
# does not have the container_count column.
@@ -1630,7 +1630,7 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
for i in range(num_containers):
name = 'test-container-%02d' % i
policy = next(policies)
self.broker.put_container(name, next(self.ts),
self.broker.put_container(name, next(self.ts).internal,
0, 0, 0, int(policy))
per_policy_container_counts[int(policy)] += 1
@@ -1717,7 +1717,7 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
for i in range(num_containers):
name = 'test-container-%02d' % i
policy = next(policies)
self.broker.put_container(name, next(self.ts),
self.broker.put_container(name, next(self.ts).internal,
0, 0, 0, int(policy))
# keep track of stub container policies
container_policy_map[name] = policy
@@ -1726,7 +1726,7 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
for i in range(0, num_containers, 2):
name = 'test-container-%02d' % i
policy = container_policy_map[name]
self.broker.put_container(name, 0, next(self.ts),
self.broker.put_container(name, 0, next(self.ts).internal,
0, 0, int(policy))
total_container_count = self.broker.get_info()['container_count']
@@ -1747,12 +1747,12 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
# add a few container entries
for i in range(num_containers):
name = 'test-container-%02d' % i
self.broker.put_container(name, next(self.ts),
self.broker.put_container(name, next(self.ts).internal,
0, 0, 0, int(policy))
# delete about half of the containers
for i in range(0, num_containers, 2):
name = 'test-container-%02d' % i
self.broker.put_container(name, 0, next(self.ts),
self.broker.put_container(name, 0, next(self.ts).internal,
0, 0, int(policy))
total_container_count = self.broker.get_info()['container_count']
@@ -1770,14 +1770,15 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
with patch_policies(legacy_only=True):
# add a container for the legacy policy
policy = POLICIES[0]
self.broker.put_container('test-legacy-container', next(self.ts),
0, 0, 0, int(policy))
self.broker.put_container('test-legacy-container',
next(self.ts).internal, 0, 0, 0,
int(policy))
# now create an impossible situation by adding a container for a
# policy index that doesn't exist
non_existent_policy_index = int(policy) + 1
self.broker.put_container('test-non-existent-policy',
next(self.ts), 0, 0, 0,
next(self.ts).internal, 0, 0, 0,
non_existent_policy_index)
total_container_count = self.broker.get_info()['container_count']

73
test/unit/common/test_db.py
View File

@@ -46,7 +46,7 @@ from swift.common.utils import normalize_timestamp, mkdirs, Timestamp
from swift.common.exceptions import LockTimeout
from swift.common.swob import HTTPException
from test.unit import with_tempdir
from test.unit import with_tempdir, make_timestamp_iter
class TestHelperFunctions(unittest.TestCase):
@@ -327,17 +327,16 @@ class TestExampleBroker(unittest.TestCase):
policy = 0
def setUp(self):
self.ts = (Timestamp(t).internal for t in
itertools.count(int(time.time())))
self.ts = make_timestamp_iter()
def test_delete_db(self):
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(next(self.ts))
broker.delete_db(next(self.ts))
broker.initialize(next(self.ts).internal)
broker.delete_db(next(self.ts).internal)
self.assertTrue(broker.is_deleted())
def test_merge_timestamps_simple_delete(self):
put_timestamp = next(self.ts)
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp)
created_at = broker.get_info()['created_at']
@@ -348,7 +347,7 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(info['delete_timestamp'], '0')
self.assertEqual(info['status_changed_at'], put_timestamp)
# delete
delete_timestamp = next(self.ts)
delete_timestamp = next(self.ts).internal
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assertTrue(broker.is_deleted())
info = broker.get_info()
@@ -364,7 +363,7 @@ class TestExampleBroker(unittest.TestCase):
broker.delete_test('test', timestamp)
def test_merge_timestamps_delete_with_objects(self):
put_timestamp = next(self.ts)
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
@@ -375,11 +374,11 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(info['delete_timestamp'], '0')
self.assertEqual(info['status_changed_at'], put_timestamp)
# add object
self.put_item(broker, next(self.ts))
self.put_item(broker, next(self.ts).internal)
self.assertEqual(broker.get_info()[
'%s_count' % broker.db_contains_type], 1)
# delete
delete_timestamp = next(self.ts)
delete_timestamp = next(self.ts).internal
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assertFalse(broker.is_deleted())
info = broker.get_info()
@@ -389,18 +388,18 @@ class TestExampleBroker(unittest.TestCase):
# status is unchanged
self.assertEqual(info['status_changed_at'], put_timestamp)
# count is causing status to hold on
self.delete_item(broker, next(self.ts))
self.delete_item(broker, next(self.ts).internal)
self.assertEqual(broker.get_info()[
'%s_count' % broker.db_contains_type], 0)
self.assertTrue(broker.is_deleted())
def test_merge_timestamps_simple_recreate(self):
put_timestamp = next(self.ts)
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
virgin_status_changed_at = broker.get_info()['status_changed_at']
created_at = broker.get_info()['created_at']
delete_timestamp = next(self.ts)
delete_timestamp = next(self.ts).internal
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assertTrue(broker.is_deleted())
info = broker.get_info()
@@ -411,7 +410,7 @@ class TestExampleBroker(unittest.TestCase):
self.assertTrue(orig_status_changed_at >
Timestamp(virgin_status_changed_at))
# recreate
recreate_timestamp = next(self.ts)
recreate_timestamp = next(self.ts).internal
status_changed_at = time.time()
with patch('swift.common.db.time.time', new=lambda: status_changed_at):
broker.merge_timestamps(created_at, recreate_timestamp, '0')
@@ -423,12 +422,12 @@ class TestExampleBroker(unittest.TestCase):
self.assertTrue(info['status_changed_at'], status_changed_at)
def test_merge_timestamps_recreate_with_objects(self):
put_timestamp = next(self.ts)
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
# delete
delete_timestamp = next(self.ts)
delete_timestamp = next(self.ts).internal
broker.merge_timestamps(created_at, put_timestamp, delete_timestamp)
self.assertTrue(broker.is_deleted())
info = broker.get_info()
@@ -439,12 +438,12 @@ class TestExampleBroker(unittest.TestCase):
self.assertTrue(Timestamp(orig_status_changed_at) >=
Timestamp(put_timestamp))
# add object
self.put_item(broker, next(self.ts))
self.put_item(broker, next(self.ts).internal)
count_key = '%s_count' % broker.db_contains_type
self.assertEqual(broker.get_info()[count_key], 1)
self.assertFalse(broker.is_deleted())
# recreate
recreate_timestamp = next(self.ts)
recreate_timestamp = next(self.ts).internal
broker.merge_timestamps(created_at, recreate_timestamp, '0')
self.assertFalse(broker.is_deleted())
info = broker.get_info()
@@ -453,30 +452,31 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(info['delete_timestamp'], delete_timestamp)
self.assertEqual(info['status_changed_at'], orig_status_changed_at)
# count is not causing status to hold on
self.delete_item(broker, next(self.ts))
self.delete_item(broker, next(self.ts).internal)
self.assertFalse(broker.is_deleted())
def test_merge_timestamps_update_put_no_status_change(self):
put_timestamp = next(self.ts)
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
info = broker.get_info()
orig_status_changed_at = info['status_changed_at']
created_at = info['created_at']
new_put_timestamp = next(self.ts)
new_put_timestamp = next(self.ts).internal
broker.merge_timestamps(created_at, new_put_timestamp, '0')
info = broker.get_info()
self.assertEqual(new_put_timestamp, info['put_timestamp'])
self.assertEqual(orig_status_changed_at, info['status_changed_at'])
def test_merge_timestamps_update_delete_no_status_change(self):
put_timestamp = next(self.ts)
put_timestamp = next(self.ts).internal
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(put_timestamp, storage_policy_index=int(self.policy))
created_at = broker.get_info()['created_at']
broker.merge_timestamps(created_at, put_timestamp, next(self.ts))
broker.merge_timestamps(created_at, put_timestamp,
next(self.ts).internal)
orig_status_changed_at = broker.get_info()['status_changed_at']
new_delete_timestamp = next(self.ts)
new_delete_timestamp = next(self.ts).internal
broker.merge_timestamps(created_at, put_timestamp,
new_delete_timestamp)
info = broker.get_info()
@@ -485,13 +485,14 @@ class TestExampleBroker(unittest.TestCase):
def test_get_max_row(self):
broker = self.broker_class(':memory:', account='a', container='c')
broker.initialize(next(self.ts), storage_policy_index=int(self.policy))
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
self.assertEqual(-1, broker.get_max_row())
self.put_item(broker, next(self.ts))
self.put_item(broker, next(self.ts).internal)
self.assertEqual(1, broker.get_max_row())
self.delete_item(broker, next(self.ts))
self.delete_item(broker, next(self.ts).internal)
self.assertEqual(2, broker.get_max_row())
self.put_item(broker, next(self.ts))
self.put_item(broker, next(self.ts).internal)
self.assertEqual(3, broker.get_max_row())
def test_get_info(self):
@@ -540,7 +541,7 @@ class TestExampleBroker(unittest.TestCase):
def test_put_timestamp(self):
broker = self.broker_class(':memory:', account='a', container='c')
orig_put_timestamp = next(self.ts)
orig_put_timestamp = next(self.ts).internal
broker.initialize(orig_put_timestamp,
storage_policy_index=int(self.policy))
self.assertEqual(broker.get_info()['put_timestamp'],
@@ -550,7 +551,7 @@ class TestExampleBroker(unittest.TestCase):
self.assertEqual(broker.get_info()['put_timestamp'],
orig_put_timestamp)
# put_timestamp newer - gets newer
newer_put_timestamp = next(self.ts)
newer_put_timestamp = next(self.ts).internal
broker.update_put_timestamp(newer_put_timestamp)
self.assertEqual(broker.get_info()['put_timestamp'],
newer_put_timestamp)
@@ -561,7 +562,7 @@ class TestExampleBroker(unittest.TestCase):
def test_status_changed_at(self):
broker = self.broker_class(':memory:', account='test', container='c')
put_timestamp = next(self.ts)
put_timestamp = next(self.ts).internal
created_at = time.time()
with patch('swift.common.db.time.time', new=lambda: created_at):
broker.initialize(put_timestamp,
@@ -570,13 +571,13 @@ class TestExampleBroker(unittest.TestCase):
put_timestamp)
self.assertEqual(broker.get_info()['created_at'],
Timestamp(created_at).internal)
status_changed_at = next(self.ts)
status_changed_at = next(self.ts).internal
broker.update_status_changed_at(status_changed_at)
self.assertEqual(broker.get_info()['status_changed_at'],
status_changed_at)
# save the old and get a new status_changed_at
old_status_changed_at, status_changed_at = \
status_changed_at, next(self.ts)
status_changed_at, next(self.ts).internal
broker.update_status_changed_at(status_changed_at)
self.assertEqual(broker.get_info()['status_changed_at'],
status_changed_at)
@@ -603,9 +604,9 @@ class TestExampleBroker(unittest.TestCase):
def test_commit_pending(self, tempdir):
broker = self.broker_class(os.path.join(tempdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts),
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
self.put_item(broker, next(self.ts))
self.put_item(broker, next(self.ts).internal)
qry = 'select * from %s_stat' % broker.db_type
with broker.get() as conn:
rows = [dict(x) for x in conn.execute(qry)]
@@ -619,7 +620,7 @@ class TestExampleBroker(unittest.TestCase):
def test_maybe_get(self, tempdir):
broker = self.broker_class(os.path.join(tempdir, 'test.db'),
account='a', container='c')
broker.initialize(next(self.ts),
broker.initialize(next(self.ts).internal,
storage_policy_index=int(self.policy))
qry = 'select account from %s_stat' % broker.db_type
with broker.maybe_get(None) as conn:

367
test/unit/container/test_backend.py
View File

File diff suppressed because it is too large Load Diff