openstack/swift
Code Issues Proposed changes

container-server unit tests: use self.ts everywhere

Browse Source 
The setUp method creates a timestamp iterator, so let's use it
consistently in all the tests.

Change-Id: Ibd06b243c6db93380b99227ac79157269a64b28a
This commit is contained in:
Alistair Coles
2024-01-04 16:43:26 +00:00
parent 3cbf01b12d
commit f2c6c19411
1 changed files with 103 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
test/unit/container/test_server.py
View File

@@ -1618,36 +1618,33 @@ class TestContainerController(unittest.TestCase):
[mock.call(os.path.join(self.testdir, 'sda1'))]) [mock.call(os.path.join(self.testdir, 'sda1'))])
def test_UPDATE(self): def test_UPDATE(self):
ts_iter = make_timestamp_iter()
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', '/sda1/p/a/c',
environ={'REQUEST_METHOD': 'PUT'}, environ={'REQUEST_METHOD': 'PUT'},
headers={'X-Timestamp': next(ts_iter).internal}) headers={'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 201) self.assertEqual(resp.status_int, 201)
ts_iter = make_timestamp_iter()
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', '/sda1/p/a/c',
environ={'REQUEST_METHOD': 'UPDATE'}, environ={'REQUEST_METHOD': 'UPDATE'},
headers={'X-Timestamp': next(ts_iter).internal}, headers={'X-Timestamp': next(self.ts).internal},
body='[invalid json') body='[invalid json')
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 400) self.assertEqual(resp.status_int, 400)
ts_iter = make_timestamp_iter()
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', '/sda1/p/a/c',
environ={'REQUEST_METHOD': 'GET'}, environ={'REQUEST_METHOD': 'GET'},
headers={'X-Timestamp': next(ts_iter).internal}) headers={'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 204) self.assertEqual(resp.status_int, 204)
obj_ts = next(ts_iter) obj_ts = next(self.ts)
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', '/sda1/p/a/c',
environ={'REQUEST_METHOD': 'UPDATE'}, environ={'REQUEST_METHOD': 'UPDATE'},
headers={'X-Timestamp': next(ts_iter).internal}, headers={'X-Timestamp': next(self.ts).internal},
body=json.dumps([ body=json.dumps([
{'name': 'some obj', 'deleted': 0, {'name': 'some obj', 'deleted': 0,
'created_at': obj_ts.internal, 'created_at': obj_ts.internal,
@@ -1655,12 +1652,12 @@ class TestContainerController(unittest.TestCase):
'storage_policy_index': POLICIES.default.idx, 'storage_policy_index': POLICIES.default.idx,
'content_type': 'foo/bar'}, 'content_type': 'foo/bar'},
{'name': 'some tombstone', 'deleted': 1, {'name': 'some tombstone', 'deleted': 1,
'created_at': next(ts_iter).internal, 'created_at': next(self.ts).internal,
'etag': 'noetag', 'size': 0, 'etag': 'noetag', 'size': 0,
'storage_policy_index': POLICIES.default.idx, 'storage_policy_index': POLICIES.default.idx,
'content_type': 'application/deleted'}, 'content_type': 'application/deleted'},
{'name': 'wrong policy', 'deleted': 0, {'name': 'wrong policy', 'deleted': 0,
'created_at': next(ts_iter).internal, 'created_at': next(self.ts).internal,
'etag': 'whatever', 'size': 6789, 'etag': 'whatever', 'size': 6789,
'storage_policy_index': 1, 'storage_policy_index': 1,
'content_type': 'foo/bar'}, 'content_type': 'foo/bar'},
@@ -1671,7 +1668,7 @@ class TestContainerController(unittest.TestCase):
req = Request.blank( req = Request.blank(
'/sda1/p/a/c?format=json', '/sda1/p/a/c?format=json',
environ={'REQUEST_METHOD': 'GET'}, environ={'REQUEST_METHOD': 'GET'},
headers={'X-Timestamp': next(ts_iter).internal}) headers={'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200) self.assertEqual(resp.status_int, 200)
self.assertEqual(json.loads(resp.body), [ self.assertEqual(json.loads(resp.body), [
@@ -1680,20 +1677,19 @@ class TestContainerController(unittest.TestCase):
]) ])
def test_UPDATE_autocreate(self): def test_UPDATE_autocreate(self):
ts_iter = make_timestamp_iter()
req = Request.blank( req = Request.blank(
'/sda1/p/.a/c', '/sda1/p/.a/c',
environ={'REQUEST_METHOD': 'GET'}, environ={'REQUEST_METHOD': 'GET'},
headers={'X-Timestamp': next(ts_iter).internal}) headers={'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 404) self.assertEqual(resp.status_int, 404)
obj_ts = next(ts_iter) obj_ts = next(self.ts)
req = Request.blank( req = Request.blank(
'/sda1/p/.a/c', '/sda1/p/.a/c',
environ={'REQUEST_METHOD': 'UPDATE'}, environ={'REQUEST_METHOD': 'UPDATE'},
headers={ headers={
'X-Timestamp': next(ts_iter).internal, 'X-Timestamp': next(self.ts).internal,
'X-Backend-Storage-Policy-Index': str(POLICIES.default.idx)}, 'X-Backend-Storage-Policy-Index': str(POLICIES.default.idx)},
body=json.dumps([ body=json.dumps([
{'name': 'some obj', 'deleted': 0, {'name': 'some obj', 'deleted': 0,
@@ -1702,12 +1698,12 @@ class TestContainerController(unittest.TestCase):
'storage_policy_index': POLICIES.default.idx, 'storage_policy_index': POLICIES.default.idx,
'content_type': 'foo/bar'}, 'content_type': 'foo/bar'},
{'name': 'some tombstone', 'deleted': 1, {'name': 'some tombstone', 'deleted': 1,
'created_at': next(ts_iter).internal, 'created_at': next(self.ts).internal,
'etag': 'noetag', 'size': 0, 'etag': 'noetag', 'size': 0,
'storage_policy_index': POLICIES.default.idx, 'storage_policy_index': POLICIES.default.idx,
'content_type': 'application/deleted'}, 'content_type': 'application/deleted'},
{'name': 'wrong policy', 'deleted': 0, {'name': 'wrong policy', 'deleted': 0,
'created_at': next(ts_iter).internal, 'created_at': next(self.ts).internal,
'etag': 'whatever', 'size': 6789, 'etag': 'whatever', 'size': 6789,
'storage_policy_index': 1, 'storage_policy_index': 1,
'content_type': 'foo/bar'}, 'content_type': 'foo/bar'},
@@ -1718,7 +1714,7 @@ class TestContainerController(unittest.TestCase):
req = Request.blank( req = Request.blank(
'/sda1/p/.a/c?format=json', '/sda1/p/.a/c?format=json',
environ={'REQUEST_METHOD': 'GET'}, environ={'REQUEST_METHOD': 'GET'},
headers={'X-Timestamp': next(ts_iter).internal}) headers={'X-Timestamp': next(self.ts).internal})
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual(resp.status_int, 200) self.assertEqual(resp.status_int, 200)
self.assertEqual(json.loads(resp.body), [ self.assertEqual(json.loads(resp.body), [
@@ -2481,9 +2477,8 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(resp.status_int, 412) self.assertEqual(resp.status_int, 412)
def test_PUT_shard_range_autocreates_shard_container(self): def test_PUT_shard_range_autocreates_shard_container(self):
ts_iter = make_timestamp_iter() shard_range = ShardRange('.shards_a/shard_c', next(self.ts))
shard_range = ShardRange('.shards_a/shard_c', next(ts_iter)) put_timestamp = next(self.ts).internal
put_timestamp = next(ts_iter).internal
headers = {'X-Backend-Record-Type': 'shard', headers = {'X-Backend-Record-Type': 'shard',
'X-Timestamp': put_timestamp, 'X-Timestamp': put_timestamp,
'X-Container-Sysmeta-Test': 'set', 'X-Container-Sysmeta-Test': 'set',
@@ -2498,7 +2493,7 @@ class TestContainerController(unittest.TestCase):
# PUT shard range to non-existent container with autocreate flag, # PUT shard range to non-existent container with autocreate flag,
# missing storage policy # missing storage policy
headers['X-Timestamp'] = next(ts_iter).internal headers['X-Timestamp'] = next(self.ts).internal
headers['X-Backend-Auto-Create'] = 't' headers['X-Backend-Auto-Create'] = 't'
req = Request.blank( req = Request.blank(
'/sda1/p/.shards_a/shard_c', method='PUT', headers=headers, '/sda1/p/.shards_a/shard_c', method='PUT', headers=headers,
@@ -2509,7 +2504,7 @@ class TestContainerController(unittest.TestCase):
resp.body) resp.body)
# PUT shard range to non-existent container with autocreate flag # PUT shard range to non-existent container with autocreate flag
headers['X-Timestamp'] = next(ts_iter).internal headers['X-Timestamp'] = next(self.ts).internal
policy_index = random.choice(POLICIES).idx policy_index = random.choice(POLICIES).idx
headers['X-Backend-Storage-Policy-Index'] = str(policy_index) headers['X-Backend-Storage-Policy-Index'] = str(policy_index)
req = Request.blank( req = Request.blank(
@@ -2519,7 +2514,7 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(201, resp.status_int) self.assertEqual(201, resp.status_int)
# repeat PUT of shard range to autocreated container - 202 response # repeat PUT of shard range to autocreated container - 202 response
headers['X-Timestamp'] = next(ts_iter).internal headers['X-Timestamp'] = next(self.ts).internal
headers.pop('X-Backend-Storage-Policy-Index') # no longer required headers.pop('X-Backend-Storage-Policy-Index') # no longer required
req = Request.blank( req = Request.blank(
'/sda1/p/.shards_a/shard_c', method='PUT', headers=headers, '/sda1/p/.shards_a/shard_c', method='PUT', headers=headers,
@@ -2528,17 +2523,16 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(202, resp.status_int) self.assertEqual(202, resp.status_int)
# regular PUT to autocreated container - 202 response # regular PUT to autocreated container - 202 response
headers['X-Timestamp'] = next(ts_iter).internal headers['X-Timestamp'] = next(self.ts).internal
req = Request.blank( req = Request.blank(
'/sda1/p/.shards_a/shard_c', method='PUT', '/sda1/p/.shards_a/shard_c', method='PUT',
headers={'X-Timestamp': next(ts_iter).internal}, headers={'X-Timestamp': next(self.ts).internal},
body=json.dumps([dict(shard_range)])) body=json.dumps([dict(shard_range)]))
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
self.assertEqual(202, resp.status_int) self.assertEqual(202, resp.status_int)
def test_PUT_shard_range_to_deleted_container(self): def test_PUT_shard_range_to_deleted_container(self):
ts_iter = make_timestamp_iter() put_time = next(self.ts).internal
put_time = next(ts_iter).internal
# create a container, get it to sharded state and then delete it # create a container, get it to sharded state and then delete it
req = Request.blank('/sda1/p/a/c', method='PUT', req = Request.blank('/sda1/p/a/c', method='PUT',
headers={'X-Timestamp': put_time}) headers={'X-Timestamp': put_time})
@@ -2546,11 +2540,11 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(201, resp.status_int) self.assertEqual(201, resp.status_int)
broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c') broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c')
broker.enable_sharding(next(ts_iter)) broker.enable_sharding(next(self.ts))
self.assertTrue(broker.set_sharding_state()) self.assertTrue(broker.set_sharding_state())
self.assertTrue(broker.set_sharded_state()) self.assertTrue(broker.set_sharded_state())
delete_time = next(ts_iter).internal delete_time = next(self.ts).internal
req = Request.blank('/sda1/p/a/c', method='DELETE', req = Request.blank('/sda1/p/a/c', method='DELETE',
headers={'X-Timestamp': delete_time}) headers={'X-Timestamp': delete_time})
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
@@ -2563,10 +2557,10 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(404, resp.status_int) self.assertEqual(404, resp.status_int)
# shard range PUT is accepted but container remains deleted # shard range PUT is accepted but container remains deleted
shard_range = ShardRange('.shards_a/shard_c', next(ts_iter), shard_range = ShardRange('.shards_a/shard_c', next(self.ts),
state=ShardRange.ACTIVE) state=ShardRange.ACTIVE)
headers = {'X-Backend-Record-Type': 'shard', headers = {'X-Backend-Record-Type': 'shard',
'X-Timestamp': next(ts_iter).internal, 'X-Timestamp': next(self.ts).internal,
'X-Container-Sysmeta-Test': 'set', 'X-Container-Sysmeta-Test': 'set',
'X-Container-Meta-Test': 'persisted'} 'X-Container-Meta-Test': 'persisted'}
@@ -2582,7 +2576,7 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(404, resp.status_int) self.assertEqual(404, resp.status_int)
# unless shard range has non-zero stats, then container is revived # unless shard range has non-zero stats, then container is revived
shard_range.update_meta(99, 1234, meta_timestamp=next(ts_iter)) shard_range.update_meta(99, 1234, meta_timestamp=next(self.ts))
req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers, req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers,
body=json.dumps([dict(shard_range)])) body=json.dumps([dict(shard_range)]))
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
@@ -2596,19 +2590,18 @@ class TestContainerController(unittest.TestCase):
self.assertEqual('99', resp.headers['X-Container-Object-Count']) self.assertEqual('99', resp.headers['X-Container-Object-Count'])
def test_PUT_shard_range_json_in_body(self): def test_PUT_shard_range_json_in_body(self):
ts_iter = make_timestamp_iter() oldest_ts = next(self.ts) # used for stale shard range PUT later
oldest_ts = next(ts_iter) # used for stale shard range PUT later
shard_bounds = [('', 'ham', ShardRange.ACTIVE), shard_bounds = [('', 'ham', ShardRange.ACTIVE),
('ham', 'salami', ShardRange.ACTIVE), ('ham', 'salami', ShardRange.ACTIVE),
('salami', '', ShardRange.CREATED)] ('salami', '', ShardRange.CREATED)]
shard_ranges = [ shard_ranges = [
ShardRange('.shards_a/_%s' % upper, next(ts_iter), ShardRange('.shards_a/_%s' % upper, next(self.ts),
lower, upper, lower, upper,
i * 100, i * 1000, meta_timestamp=next(ts_iter), i * 100, i * 1000, meta_timestamp=next(self.ts),
state=state, state_timestamp=next(ts_iter)) state=state, state_timestamp=next(self.ts))
for i, (lower, upper, state) in enumerate(shard_bounds)] for i, (lower, upper, state) in enumerate(shard_bounds)]
put_timestamp = next(ts_iter).internal put_timestamp = next(self.ts).internal
headers = {'X-Backend-Record-Type': 'shard', headers = {'X-Backend-Record-Type': 'shard',
'X-Timestamp': put_timestamp, 'X-Timestamp': put_timestamp,
'X-Container-Sysmeta-Test': 'set', 'X-Container-Sysmeta-Test': 'set',
@@ -2647,7 +2640,7 @@ class TestContainerController(unittest.TestCase):
# empty json dict # empty json dict
body = json.dumps({}) body = json.dumps({})
headers['X-Timestamp'] = next(ts_iter).internal headers['X-Timestamp'] = next(self.ts).internal
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
@@ -2658,12 +2651,12 @@ class TestContainerController(unittest.TestCase):
broker.get_shard_ranges()) broker.get_shard_ranges())
self.assertEqual(put_timestamp, broker.get_info()['put_timestamp']) self.assertEqual(put_timestamp, broker.get_info()['put_timestamp'])
older_ts = next(ts_iter) # used for stale shard range PUT later older_ts = next(self.ts) # used for stale shard range PUT later
# updated and new shard ranges # updated and new shard ranges
shard_ranges[1].bytes_used += 100 shard_ranges[1].bytes_used += 100
shard_ranges[1].meta_timestamp = next(ts_iter) shard_ranges[1].meta_timestamp = next(self.ts)
body = json.dumps([dict(sr) for sr in shard_ranges[1:]]) body = json.dumps([dict(sr) for sr in shard_ranges[1:]])
headers['X-Timestamp'] = next(ts_iter).internal headers['X-Timestamp'] = next(self.ts).internal
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
@@ -2682,7 +2675,7 @@ class TestContainerController(unittest.TestCase):
stale_shard_range.state = ShardRange.CREATED stale_shard_range.state = ShardRange.CREATED
stale_shard_range.state_timestamp = oldest_ts stale_shard_range.state_timestamp = oldest_ts
body = json.dumps([dict(stale_shard_range)]) body = json.dumps([dict(stale_shard_range)])
headers['X-Timestamp'] = next(ts_iter).internal headers['X-Timestamp'] = next(self.ts).internal
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
resp = req.get_response(self.controller) resp = req.get_response(self.controller)
@@ -2695,7 +2688,7 @@ class TestContainerController(unittest.TestCase):
# deleted shard range # deleted shard range
shard_ranges[0].deleted = 1 shard_ranges[0].deleted = 1
shard_ranges[0].timestamp = next(ts_iter) shard_ranges[0].timestamp = next(self.ts)
body = json.dumps([dict(shard_ranges[0])]) body = json.dumps([dict(shard_ranges[0])])
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
@@ -2708,7 +2701,7 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(put_timestamp, broker.get_info()['put_timestamp']) self.assertEqual(put_timestamp, broker.get_info()['put_timestamp'])
def check_bad_body(body): def check_bad_body(body):
bad_put_timestamp = next(ts_iter).internal bad_put_timestamp = next(self.ts).internal
headers['X-Timestamp'] = bad_put_timestamp headers['X-Timestamp'] = bad_put_timestamp
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
@@ -2725,7 +2718,7 @@ class TestContainerController(unittest.TestCase):
check_bad_body('') check_bad_body('')
check_bad_body('["not a shard range"]') check_bad_body('["not a shard range"]')
check_bad_body('[[]]') check_bad_body('[[]]')
bad_shard_range = dict(ShardRange('a/c', next(ts_iter))) bad_shard_range = dict(ShardRange('a/c', next(self.ts)))
bad_shard_range.pop('timestamp') bad_shard_range.pop('timestamp')
check_bad_body(json.dumps([bad_shard_range])) check_bad_body(json.dumps([bad_shard_range]))
@@ -2733,7 +2726,7 @@ class TestContainerController(unittest.TestCase):
# body ignored # body ignored
body = json.dumps([dict(sr) for sr in shard_ranges]) body = json.dumps([dict(sr) for sr in shard_ranges])
# note, regular PUT so put timestamp is updated # note, regular PUT so put timestamp is updated
put_timestamp = next(ts_iter).internal put_timestamp = next(self.ts).internal
headers['X-Timestamp'] = put_timestamp headers['X-Timestamp'] = put_timestamp
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
@@ -2744,21 +2737,20 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(put_timestamp, broker.get_info()['put_timestamp']) self.assertEqual(put_timestamp, broker.get_info()['put_timestamp'])
check_not_shard_record_type({'X-Backend-Record-Type': 'object', check_not_shard_record_type({'X-Backend-Record-Type': 'object',
'X-Timestamp': next(ts_iter).internal}) 'X-Timestamp': next(self.ts).internal})
check_not_shard_record_type({'X-Timestamp': next(ts_iter).internal}) check_not_shard_record_type({'X-Timestamp': next(self.ts).internal})
def test_PUT_GET_shard_ranges(self): def test_PUT_GET_shard_ranges(self):
# make a container # make a container
ts_iter = make_timestamp_iter()
ts_now = Timestamp.now() # used when mocking Timestamp.now() ts_now = Timestamp.now() # used when mocking Timestamp.now()
ts_put = next(ts_iter) ts_put = next(self.ts)
headers = {'X-Timestamp': ts_put.normal} headers = {'X-Timestamp': ts_put.normal}
req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers) req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers)
self.assertEqual(201, req.get_response(self.controller).status_int) self.assertEqual(201, req.get_response(self.controller).status_int)
# PUT some objects # PUT some objects
objects = [{'name': 'obj_%d' % i, objects = [{'name': 'obj_%d' % i,
'x-timestamp': next(ts_iter).normal, 'x-timestamp': next(self.ts).normal,
'x-content-type': 'text/plain', 'x-content-type': 'text/plain',
'x-etag': 'etag_%d' % i, 'x-etag': 'etag_%d' % i,
'x-size': 1024 * i 'x-size': 1024 * i
@@ -2777,10 +2769,10 @@ class TestContainerController(unittest.TestCase):
('yoghurt', '', ShardRange.FOUND), ('yoghurt', '', ShardRange.FOUND),
] ]
shard_ranges = [ shard_ranges = [
ShardRange('.sharded_a/_%s' % upper, next(ts_iter), ShardRange('.sharded_a/_%s' % upper, next(self.ts),
lower, upper, lower, upper,
i * 100, i * 1000, meta_timestamp=next(ts_iter), i * 100, i * 1000, meta_timestamp=next(self.ts),
state=state, state_timestamp=next(ts_iter)) state=state, state_timestamp=next(self.ts))
for i, (lower, upper, state) in enumerate(shard_bounds)] for i, (lower, upper, state) in enumerate(shard_bounds)]
for shard_range in shard_ranges: for shard_range in shard_ranges:
self._put_shard_range(shard_range) self._put_shard_range(shard_range)
@@ -3013,7 +3005,7 @@ class TestContainerController(unittest.TestCase):
params='&states=listing&reverse=true&marker=egg') params='&states=listing&reverse=true&marker=egg')
self.assertIsNone( self.assertIsNone(
resp.headers.get('X-Backend-Override-Shard-Name-Filter')) resp.headers.get('X-Backend-Override-Shard-Name-Filter'))
ts_epoch = next(ts_iter) ts_epoch = next(self.ts)
broker.enable_sharding(ts_epoch) broker.enable_sharding(ts_epoch)
self.assertTrue(broker.set_sharding_state()) self.assertTrue(broker.set_sharding_state())
check_shard_GET( check_shard_GET(
@@ -3032,7 +3024,7 @@ class TestContainerController(unittest.TestCase):
# params and return listing shard ranges for entire namespace if # params and return listing shard ranges for entire namespace if
# X-Backend-Override-Shard-Name-Filter == 'sharded' # X-Backend-Override-Shard-Name-Filter == 'sharded'
self.assertTrue(broker.set_sharded_state()) self.assertTrue(broker.set_sharded_state())
ts_now = next(ts_iter) ts_now = next(self.ts)
with mock_timestamp_now(ts_now): with mock_timestamp_now(ts_now):
extra_shard_range = broker.get_own_shard_range() extra_shard_range = broker.get_own_shard_range()
extra_shard_range.lower = shard_ranges[2].upper extra_shard_range.lower = shard_ranges[2].upper
@@ -3065,7 +3057,7 @@ class TestContainerController(unittest.TestCase):
# delete a shard range # delete a shard range
shard_range = shard_ranges[1] shard_range = shard_ranges[1]
shard_range.set_deleted(timestamp=next(ts_iter)) shard_range.set_deleted(timestamp=next(self.ts))
self._put_shard_range(shard_range) self._put_shard_range(shard_range)
self._assert_shard_ranges_equal(shard_ranges[:1] + shard_ranges[2:], self._assert_shard_ranges_equal(shard_ranges[:1] + shard_ranges[2:],
@@ -3091,32 +3083,31 @@ class TestContainerController(unittest.TestCase):
# make a shrunk shard container with two acceptors that overlap with # make a shrunk shard container with two acceptors that overlap with
# the shard's namespace # the shard's namespace
shard_path = '.shards_a/c_f' shard_path = '.shards_a/c_f'
ts_iter = make_timestamp_iter()
ts_now = Timestamp.now() # used when mocking Timestamp.now() ts_now = Timestamp.now() # used when mocking Timestamp.now()
own_shard_range = ShardRange(shard_path, next(ts_iter), own_shard_range = ShardRange(shard_path, next(self.ts),
'b', 'f', 100, 1000, 'b', 'f', 100, 1000,
meta_timestamp=next(ts_iter), meta_timestamp=next(self.ts),
state=ShardRange.SHRUNK, state=ShardRange.SHRUNK,
state_timestamp=next(ts_iter), state_timestamp=next(self.ts),
epoch=next(ts_iter)) epoch=next(self.ts))
shard_ranges = [] shard_ranges = []
for lower, upper in (('a', 'd'), ('d', 'g')): for lower, upper in (('a', 'd'), ('d', 'g')):
shard_ranges.append( shard_ranges.append(
ShardRange('.shards_a/c_%s' % upper, next(ts_iter), ShardRange('.shards_a/c_%s' % upper, next(self.ts),
lower, upper, 100, 1000, lower, upper, 100, 1000,
meta_timestamp=next(ts_iter), meta_timestamp=next(self.ts),
state=ShardRange.ACTIVE, state=ShardRange.ACTIVE,
state_timestamp=next(ts_iter))) state_timestamp=next(self.ts)))
# create container # create container
headers = {'X-Timestamp': next(ts_iter).normal} headers = {'X-Timestamp': next(self.ts).normal}
req = Request.blank( req = Request.blank(
'/sda1/p/%s' % shard_path, method='PUT', headers=headers) '/sda1/p/%s' % shard_path, method='PUT', headers=headers)
self.assertIn( self.assertIn(
req.get_response(self.controller).status_int, (201, 202)) req.get_response(self.controller).status_int, (201, 202))
# PUT the acceptor shard ranges and own shard range # PUT the acceptor shard ranges and own shard range
headers = {'X-Timestamp': next(ts_iter).normal, headers = {'X-Timestamp': next(self.ts).normal,
'X-Container-Sysmeta-Shard-Root': 'a/c', 'X-Container-Sysmeta-Shard-Root': 'a/c',
'X-Backend-Record-Type': 'shard'} 'X-Backend-Record-Type': 'shard'}
body = json.dumps( body = json.dumps(
@@ -3185,27 +3176,26 @@ class TestContainerController(unittest.TestCase):
def test_GET_shard_ranges_using_state_aliases(self): def test_GET_shard_ranges_using_state_aliases(self):
# make a shard container # make a shard container
ts_iter = make_timestamp_iter()
shard_ranges = [] shard_ranges = []
lower = '' lower = ''
for state in sorted(ShardRange.STATES.keys()): for state in sorted(ShardRange.STATES.keys()):
upper = str(state) upper = str(state)
shard_ranges.append( shard_ranges.append(
ShardRange('.shards_a/c_%s' % upper, next(ts_iter), ShardRange('.shards_a/c_%s' % upper, next(self.ts),
lower, upper, state * 100, state * 1000, lower, upper, state * 100, state * 1000,
meta_timestamp=next(ts_iter), meta_timestamp=next(self.ts),
state=state, state_timestamp=next(ts_iter))) state=state, state_timestamp=next(self.ts)))
lower = upper lower = upper
def do_test(root_path, path, params, expected_states): def do_test(root_path, path, params, expected_states):
expected = [ expected = [
sr for sr in shard_ranges if sr.state in expected_states] sr for sr in shard_ranges if sr.state in expected_states]
own_shard_range = ShardRange(path, next(ts_iter), '', '', own_shard_range = ShardRange(path, next(self.ts), '', '',
state=ShardRange.ACTIVE) state=ShardRange.ACTIVE)
expected.append(own_shard_range.copy(lower=expected[-1].upper)) expected.append(own_shard_range.copy(lower=expected[-1].upper))
expected = [dict(sr, last_modified=sr.timestamp.isoformat) expected = [dict(sr, last_modified=sr.timestamp.isoformat)
for sr in expected] for sr in expected]
headers = {'X-Timestamp': next(ts_iter).normal} headers = {'X-Timestamp': next(self.ts).normal}
# create container # create container
req = Request.blank( req = Request.blank(
@@ -3213,7 +3203,7 @@ class TestContainerController(unittest.TestCase):
self.assertIn( self.assertIn(
req.get_response(self.controller).status_int, (201, 202)) req.get_response(self.controller).status_int, (201, 202))
# PUT some shard ranges # PUT some shard ranges
headers = {'X-Timestamp': next(ts_iter).normal, headers = {'X-Timestamp': next(self.ts).normal,
'X-Container-Sysmeta-Shard-Root': root_path, 'X-Container-Sysmeta-Shard-Root': root_path,
'X-Backend-Record-Type': 'shard'} 'X-Backend-Record-Type': 'shard'}
body = json.dumps( body = json.dumps(
@@ -3259,26 +3249,25 @@ class TestContainerController(unittest.TestCase):
def test_GET_shard_ranges_include_deleted(self): def test_GET_shard_ranges_include_deleted(self):
# make a shard container # make a shard container
ts_iter = make_timestamp_iter()
ts_now = Timestamp.now() # used when mocking Timestamp.now() ts_now = Timestamp.now() # used when mocking Timestamp.now()
shard_ranges = [] shard_ranges = []
lower = '' lower = ''
for state in sorted(ShardRange.STATES.keys()): for state in sorted(ShardRange.STATES.keys()):
upper = str(state) upper = str(state)
shard_ranges.append( shard_ranges.append(
ShardRange('.shards_a/c_%s' % upper, next(ts_iter), ShardRange('.shards_a/c_%s' % upper, next(self.ts),
lower, upper, state * 100, state * 1000, lower, upper, state * 100, state * 1000,
meta_timestamp=next(ts_iter), meta_timestamp=next(self.ts),
state=state, state_timestamp=next(ts_iter))) state=state, state_timestamp=next(self.ts)))
lower = upper lower = upper
# create container # create container
headers = {'X-Timestamp': next(ts_iter).normal} headers = {'X-Timestamp': next(self.ts).normal}
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers) '/sda1/p/a/c', method='PUT', headers=headers)
self.assertIn( self.assertIn(
req.get_response(self.controller).status_int, (201, 202)) req.get_response(self.controller).status_int, (201, 202))
# PUT some shard ranges # PUT some shard ranges
headers = {'X-Timestamp': next(ts_iter).normal, headers = {'X-Timestamp': next(self.ts).normal,
'X-Backend-Record-Type': 'shard'} 'X-Backend-Record-Type': 'shard'}
body = json.dumps([dict(sr) for sr in shard_ranges]) body = json.dumps([dict(sr) for sr in shard_ranges])
req = Request.blank( req = Request.blank(
@@ -3303,10 +3292,10 @@ class TestContainerController(unittest.TestCase):
do_test(False, shard_ranges) do_test(False, shard_ranges)
do_test(True, shard_ranges) do_test(True, shard_ranges)
headers = {'X-Timestamp': next(ts_iter).normal, headers = {'X-Timestamp': next(self.ts).normal,
'X-Backend-Record-Type': 'shard'} 'X-Backend-Record-Type': 'shard'}
for sr in shard_ranges[::2]: for sr in shard_ranges[::2]:
sr.set_deleted(timestamp=next(ts_iter)) sr.set_deleted(timestamp=next(self.ts))
body = json.dumps([dict(sr) for sr in shard_ranges]) body = json.dumps([dict(sr) for sr in shard_ranges])
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
@@ -3317,10 +3306,10 @@ class TestContainerController(unittest.TestCase):
do_test(False, shard_ranges[1::2]) do_test(False, shard_ranges[1::2])
do_test(True, shard_ranges) do_test(True, shard_ranges)
headers = {'X-Timestamp': next(ts_iter).normal, headers = {'X-Timestamp': next(self.ts).normal,
'X-Backend-Record-Type': 'shard'} 'X-Backend-Record-Type': 'shard'}
for sr in shard_ranges[1::2]: for sr in shard_ranges[1::2]:
sr.set_deleted(timestamp=next(ts_iter)) sr.set_deleted(timestamp=next(self.ts))
body = json.dumps([dict(sr) for sr in shard_ranges]) body = json.dumps([dict(sr) for sr in shard_ranges])
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers, body=body) '/sda1/p/a/c', method='PUT', headers=headers, body=body)
@@ -3331,26 +3320,25 @@ class TestContainerController(unittest.TestCase):
def test_GET_shard_ranges_errors(self): def test_GET_shard_ranges_errors(self):
# verify that x-backend-record-type is not included in error responses # verify that x-backend-record-type is not included in error responses
ts_iter = make_timestamp_iter()
ts_now = Timestamp.now() # used when mocking Timestamp.now() ts_now = Timestamp.now() # used when mocking Timestamp.now()
shard_ranges = [] shard_ranges = []
lower = '' lower = ''
for state in sorted(ShardRange.STATES.keys()): for state in sorted(ShardRange.STATES.keys()):
upper = str(state) upper = str(state)
shard_ranges.append( shard_ranges.append(
ShardRange('.shards_a/c_%s' % upper, next(ts_iter), ShardRange('.shards_a/c_%s' % upper, next(self.ts),
lower, upper, state * 100, state * 1000, lower, upper, state * 100, state * 1000,
meta_timestamp=next(ts_iter), meta_timestamp=next(self.ts),
state=state, state_timestamp=next(ts_iter))) state=state, state_timestamp=next(self.ts)))
lower = upper lower = upper
# create container # create container
headers = {'X-Timestamp': next(ts_iter).normal} headers = {'X-Timestamp': next(self.ts).normal}
req = Request.blank( req = Request.blank(
'/sda1/p/a/c', method='PUT', headers=headers) '/sda1/p/a/c', method='PUT', headers=headers)
self.assertIn( self.assertIn(
req.get_response(self.controller).status_int, (201, 202)) req.get_response(self.controller).status_int, (201, 202))
# PUT some shard ranges # PUT some shard ranges
headers = {'X-Timestamp': next(ts_iter).normal, headers = {'X-Timestamp': next(self.ts).normal,
'X-Backend-Record-Type': 'shard'} 'X-Backend-Record-Type': 'shard'}
body = json.dumps([dict(sr) for sr in shard_ranges]) body = json.dumps([dict(sr) for sr in shard_ranges])
req = Request.blank( req = Request.blank(
@@ -3381,7 +3369,7 @@ class TestContainerController(unittest.TestCase):
# delete the container # delete the container
req = Request.blank('/sda1/p/a/c', method='DELETE', req = Request.blank('/sda1/p/a/c', method='DELETE',
headers={'X-Timestamp': next(ts_iter).normal}) headers={'X-Timestamp': next(self.ts).normal})
self.assertEqual(204, req.get_response(self.controller).status_int) self.assertEqual(204, req.get_response(self.controller).status_int)
do_test({'states': 'bad'}, 404) do_test({'states': 'bad'}, 404)
@@ -3486,14 +3474,13 @@ class TestContainerController(unittest.TestCase):
def test_GET_auto_record_type(self): def test_GET_auto_record_type(self):
# make a container # make a container
ts_iter = make_timestamp_iter()
ts_now = Timestamp.now() # used when mocking Timestamp.now() ts_now = Timestamp.now() # used when mocking Timestamp.now()
headers = {'X-Timestamp': next(ts_iter).normal} headers = {'X-Timestamp': next(self.ts).normal}
req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers) req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers)
self.assertEqual(201, req.get_response(self.controller).status_int) self.assertEqual(201, req.get_response(self.controller).status_int)
# PUT some objects # PUT some objects
objects = [{'name': 'obj_%d' % i, objects = [{'name': 'obj_%d' % i,
'x-timestamp': next(ts_iter).normal, 'x-timestamp': next(self.ts).normal,
'x-content-type': 'text/plain', 'x-content-type': 'text/plain',
'x-etag': 'etag_%d' % i, 'x-etag': 'etag_%d' % i,
'x-size': 1024 * i 'x-size': 1024 * i
@@ -3508,10 +3495,10 @@ class TestContainerController(unittest.TestCase):
shard_bounds = [('', 'm', ShardRange.CLEAVED), shard_bounds = [('', 'm', ShardRange.CLEAVED),
('m', '', ShardRange.CREATED)] ('m', '', ShardRange.CREATED)]
shard_ranges = [ shard_ranges = [
ShardRange('.sharded_a/_%s' % upper, next(ts_iter), ShardRange('.sharded_a/_%s' % upper, next(self.ts),
lower, upper, lower, upper,
i * 100, i * 1000, meta_timestamp=next(ts_iter), i * 100, i * 1000, meta_timestamp=next(self.ts),
state=state, state_timestamp=next(ts_iter)) state=state, state_timestamp=next(self.ts))
for i, (lower, upper, state) in enumerate(shard_bounds)] for i, (lower, upper, state) in enumerate(shard_bounds)]
for shard_range in shard_ranges: for shard_range in shard_ranges:
self._put_shard_range(shard_range) self._put_shard_range(shard_range)
@@ -3578,7 +3565,7 @@ class TestContainerController(unittest.TestCase):
self.assertEqual(headers, resp.headers) self.assertEqual(headers, resp.headers)
# move to sharding state # move to sharding state
broker.enable_sharding(next(ts_iter)) broker.enable_sharding(next(self.ts))
self.assertTrue(broker.set_sharding_state()) self.assertTrue(broker.set_sharding_state())
req = Request.blank('/sda1/p/a/c?format=json', method='GET', req = Request.blank('/sda1/p/a/c?format=json', method='GET',
headers={'X-Backend-Record-Type': 'auto'}) headers={'X-Backend-Record-Type': 'auto'})
@@ -3633,14 +3620,13 @@ class TestContainerController(unittest.TestCase):
def test_PUT_GET_to_sharding_container(self): def test_PUT_GET_to_sharding_container(self):
broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c') broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c')
ts_iter = make_timestamp_iter() headers = {'X-Timestamp': next(self.ts).normal}
headers = {'X-Timestamp': next(ts_iter).normal}
req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers) req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers)
self.assertEqual(201, req.get_response(self.controller).status_int) self.assertEqual(201, req.get_response(self.controller).status_int)
def do_update(name, timestamp=None, headers=None): def do_update(name, timestamp=None, headers=None):
# Make a PUT request to container controller to update an object # Make a PUT request to container controller to update an object
timestamp = timestamp or next(ts_iter) timestamp = timestamp or next(self.ts)
headers = headers or {} headers = headers or {}
headers.update({'X-Timestamp': timestamp.internal, headers.update({'X-Timestamp': timestamp.internal,
'X-Size': 17, 'X-Size': 17,
@@ -3674,7 +3660,7 @@ class TestContainerController(unittest.TestCase):
assert_broker_rows(broker, ['unsharded'], 1) assert_broker_rows(broker, ['unsharded'], 1)
# move container to sharding state # move container to sharding state
broker.enable_sharding(next(ts_iter)) broker.enable_sharding(next(self.ts))
self.assertTrue(broker.set_sharding_state()) self.assertTrue(broker.set_sharding_state())
assert_broker_rows(broker.get_brokers()[0], ['unsharded'], 1) assert_broker_rows(broker.get_brokers()[0], ['unsharded'], 1)
assert_broker_rows(broker.get_brokers()[1], [], 1) assert_broker_rows(broker.get_brokers()[1], [], 1)
@@ -3714,14 +3700,13 @@ class TestContainerController(unittest.TestCase):
def _check_object_update_redirected_to_shard(self, method): def _check_object_update_redirected_to_shard(self, method):
expected_status = 204 if method == 'DELETE' else 201 expected_status = 204 if method == 'DELETE' else 201
broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c') broker = self.controller._get_container_broker('sda1', 'p', 'a', 'c')
ts_iter = make_timestamp_iter() headers = {'X-Timestamp': next(self.ts).normal}
headers = {'X-Timestamp': next(ts_iter).normal}
req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers) req = Request.blank('/sda1/p/a/c', method='PUT', headers=headers)
self.assertEqual(201, req.get_response(self.controller).status_int) self.assertEqual(201, req.get_response(self.controller).status_int)
def do_update(name, timestamp=None, headers=None): def do_update(name, timestamp=None, headers=None):
# Make a PUT request to container controller to update an object # Make a PUT request to container controller to update an object
timestamp = timestamp or next(ts_iter) timestamp = timestamp or next(self.ts)
headers = headers or {} headers = headers or {}
headers.update({'X-Timestamp': timestamp.internal, headers.update({'X-Timestamp': timestamp.internal,
'X-Size': 17, 'X-Size': 17,
@@ -3752,7 +3737,7 @@ class TestContainerController(unittest.TestCase):
resp.headers['X-Backend-Redirect-Timestamp']) resp.headers['X-Backend-Redirect-Timestamp'])
# sanity check # sanity check
ts_bashful_orig = next(ts_iter) ts_bashful_orig = next(self.ts)
mocked_fn = 'swift.container.backend.ContainerBroker.get_shard_ranges' mocked_fn = 'swift.container.backend.ContainerBroker.get_shard_ranges'
with mock.patch(mocked_fn) as mock_get_shard_ranges: with mock.patch(mocked_fn) as mock_get_shard_ranges:
assert_not_redirected('bashful', ts_bashful_orig) assert_not_redirected('bashful', ts_bashful_orig)
@@ -3760,10 +3745,10 @@ class TestContainerController(unittest.TestCase):
shard_ranges = { shard_ranges = {
'dopey': ShardRange( 'dopey': ShardRange(
'.sharded_a/sr_dopey', next(ts_iter), '', 'dopey'), '.sharded_a/sr_dopey', next(self.ts), '', 'dopey'),
'happy': ShardRange( 'happy': ShardRange(
'.sharded_a/sr_happy', next(ts_iter), 'dopey', 'happy'), '.sharded_a/sr_happy', next(self.ts), 'dopey', 'happy'),
'': ShardRange('.sharded_a/sr_', next(ts_iter), 'happy', '') '': ShardRange('.sharded_a/sr_', next(self.ts), 'happy', '')
} }
# start with only the middle shard range # start with only the middle shard range
self._put_shard_range(shard_ranges['happy']) self._put_shard_range(shard_ranges['happy'])
@@ -3777,7 +3762,7 @@ class TestContainerController(unittest.TestCase):
for state in ShardRange.STATES: for state in ShardRange.STATES:
self.assertTrue( self.assertTrue(
sr_happy.update_state(state, sr_happy.update_state(state,
state_timestamp=next(ts_iter))) state_timestamp=next(self.ts)))
self._put_shard_range(sr_happy) self._put_shard_range(sr_happy)
with annotate_failure(state): with annotate_failure(state):
obj_name = 'grumpy%s' % state obj_name = 'grumpy%s' % state
@@ -3797,12 +3782,12 @@ class TestContainerController(unittest.TestCase):
[obj['name'] for obj in get_listing(-1)]) [obj['name'] for obj in get_listing(-1)])
# set broker to sharding state # set broker to sharding state
broker.enable_sharding(next(ts_iter)) broker.enable_sharding(next(self.ts))
self.assertTrue(broker.set_sharding_state()) self.assertTrue(broker.set_sharding_state())
for state in ShardRange.STATES: for state in ShardRange.STATES:
self.assertTrue( self.assertTrue(
sr_happy.update_state(state, sr_happy.update_state(state,
state_timestamp=next(ts_iter))) state_timestamp=next(self.ts)))
self._put_shard_range(sr_happy) self._put_shard_range(sr_happy)
with annotate_failure(state): with annotate_failure(state):
obj_name = 'grumpier%s' % state obj_name = 'grumpier%s' % state
@@ -3829,19 +3814,19 @@ class TestContainerController(unittest.TestCase):
# no shard for this object yet so it is accepted by root container # no shard for this object yet so it is accepted by root container
# and stored in misplaced objects... # and stored in misplaced objects...
assert_not_redirected('dopey', timestamp=next(ts_iter)) assert_not_redirected('dopey', timestamp=next(self.ts))
self.assertIn('dopey', [obj['name'] for obj in get_listing(-1)]) self.assertIn('dopey', [obj['name'] for obj in get_listing(-1)])
self.assertNotIn('dopey', [obj['name'] for obj in get_listing(0)]) self.assertNotIn('dopey', [obj['name'] for obj in get_listing(0)])
# now PUT the first shard range # now PUT the first shard range
sr_dopey = shard_ranges['dopey'] sr_dopey = shard_ranges['dopey']
sr_dopey.update_state(ShardRange.CLEAVED, sr_dopey.update_state(ShardRange.CLEAVED,
state_timestamp=next(ts_iter)) state_timestamp=next(self.ts))
self._put_shard_range(sr_dopey) self._put_shard_range(sr_dopey)
for state in ShardRange.STATES: for state in ShardRange.STATES:
self.assertTrue( self.assertTrue(
sr_happy.update_state(state, sr_happy.update_state(state,
state_timestamp=next(ts_iter))) state_timestamp=next(self.ts)))
self._put_shard_range(sr_happy) self._put_shard_range(sr_happy)
with annotate_failure(state): with annotate_failure(state):
obj_name = 'dopey%s' % state obj_name = 'dopey%s' % state
@@ -3879,7 +3864,7 @@ class TestContainerController(unittest.TestCase):
for state in ShardRange.STATES: for state in ShardRange.STATES:
self.assertTrue( self.assertTrue(
sr_happy.update_state(state, sr_happy.update_state(state,
state_timestamp=next(ts_iter))) state_timestamp=next(self.ts)))
self._put_shard_range(sr_happy) self._put_shard_range(sr_happy)
with annotate_failure(state): with annotate_failure(state):
obj_name = 'grumpiest%s' % state obj_name = 'grumpiest%s' % state