openstack/swift
Code Issues Proposed changes

Merge "py3: (mostly) port probe tests"

Browse Source
This commit is contained in:
Zuul 2019-09-12 00:49:29 +00:00 committed by Gerrit Code Review
commit 5cb53838a6
16 changed files with 166 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
test/probe/common.py
View File

@ -164,7 +164,7 @@ def add_ring_devs_to_ipport2server(ring, server_type, ipport2server,
# We'll number the servers by order of unique occurrence of:
# IP, if servers_per_port > 0 OR there > 1 IP in ring
# ipport, otherwise
unique_ip_count = len(set(dev['ip'] for dev in ring.devs if dev))
unique_ip_count = len({dev['ip'] for dev in ring.devs if dev})
things_to_number = {}
number = 0
for dev in filter(None, ring.devs):
@ -244,7 +244,7 @@ def get_ring(ring_name, required_replicas, required_devices,
if p.returncode:
raise unittest.SkipTest('unable to connect to rsync '
'export %s (%s)' % (rsync_export, cmd))
for line in stdout.splitlines():
for line in stdout.decode().splitlines():
if line.rsplit(None, 1)[-1] == dev['device']:
break
else:
@ -295,11 +295,11 @@ def kill_orphans():
class Body(object):
def __init__(self, total=3.5 * 2 ** 20):
self.length = total
self.length = int(total)
self.hasher = md5()
self.read_amount = 0
self.chunk = uuid4().hex * 2 ** 10
self.buff = ''
self.chunk = uuid4().hex.encode('ascii') * 2 ** 10
self.buff = b''
@property
def etag(self):
@ -320,9 +320,9 @@ class Body(object):
def __iter__(self):
return self
def next(self):
def __next__(self):
if self.buff:
rv, self.buff = self.buff, ''
rv, self.buff = self.buff, b''
return rv
if self.read_amount >= self.length:
raise StopIteration()
@ -331,8 +331,8 @@ class Body(object):
self.hasher.update(rv)
return rv
def __next__(self):
return next(self)
# for py2 compat:
next = __next__
class ProbeTest(unittest.TestCase):

6
test/probe/test_account_get_fake_responses_match.py
View File

@ -56,7 +56,7 @@ class TestAccountGetFakeResponsesMatch(ReplProbeTest):
raise Exception("Unexpected status %s\n%s" %
(resp.status, resp.read()))
response_headers = dict(resp.getheaders())
response_headers = {h.lower(): v for h, v in resp.getheaders()}
response_body = resp.read()
resp.close()
return response_headers, response_body
@ -98,8 +98,8 @@ class TestAccountGetFakeResponsesMatch(ReplProbeTest):
fake_acct, headers={'Accept': 'application/xml'})
# the account name is in the XML response
real_body = re.sub('AUTH_\w{4}', 'AUTH_someaccount', real_body)
fake_body = re.sub('AUTH_\w{4}', 'AUTH_someaccount', fake_body)
real_body = re.sub(br'AUTH_\w{4}', b'AUTH_someaccount', real_body)
fake_body = re.sub(br'AUTH_\w{4}', b'AUTH_someaccount', fake_body)
self.assertEqual(real_body, fake_body)
self.assertEqual(real_headers['content-type'],

2
test/probe/test_container_failures.py
View File

@ -130,7 +130,7 @@ class TestContainerFailures(ReplProbeTest):
onode = onodes[0]
db_files = []
for onode in onodes:
node_id = (onode['port'] - 6000) / 10
node_id = (onode['port'] - 6000) // 10
device = onode['device']
hash_str = hash_path(self.account, container)
server_conf = readconf(self.configs['container-server'][node_id])

59
test/probe/test_container_merge_policy_index.py
View File

@ -68,7 +68,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.brain.stop_handoff_half()
self.brain.put_container()
self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'},
contents='VERIFY')
contents=b'VERIFY')
self.brain.start_handoff_half()
# make sure we have some manner of split brain
container_part, container_nodes = self.container_ring.get_nodes(
@ -78,9 +78,9 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
found_policy_indexes = {
metadata['X-Backend-Storage-Policy-Index']
for node, metadata in head_responses}
self.assertGreater(
len(found_policy_indexes), 1,
'primary nodes did not disagree about policy index %r' %
@ -116,9 +116,9 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
found_policy_indexes = {
metadata['X-Backend-Storage-Policy-Index']
for node, metadata in head_responses}
self.assertEqual(len(found_policy_indexes), 1,
'primary nodes disagree about policy index %r' %
head_responses)
@ -144,7 +144,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
orig_policy_index))
# verify that the object data read by external client is correct
headers, data = self._get_object_patiently(expected_policy_index)
self.assertEqual('VERIFY', data)
self.assertEqual(b'VERIFY', data)
self.assertEqual('custom-meta', headers['x-object-meta-test'])
def test_reconcile_delete(self):
@ -165,9 +165,9 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for
node, metadata in head_responses)
found_policy_indexes = {
metadata['X-Backend-Storage-Policy-Index']
for node, metadata in head_responses}
self.assertGreater(
len(found_policy_indexes), 1,
'primary nodes did not disagree about policy index %r' %
@ -208,15 +208,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
metadata = direct_client.direct_head_container(
node, container_part, self.account, self.container_name)
head_responses.append((node, metadata))
new_found_policy_indexes = \
set(metadata['X-Backend-Storage-Policy-Index'] for node,
metadata in head_responses)
self.assertEqual(len(new_found_policy_indexes), 1,
node_to_policy = {
node['port']: metadata['X-Backend-Storage-Policy-Index']
for node, metadata in head_responses}
policies = set(node_to_policy.values())
self.assertEqual(len(policies), 1,
'primary nodes disagree about policy index %r' %
dict((node['port'],
metadata['X-Backend-Storage-Policy-Index'])
for node, metadata in head_responses))
expected_policy_index = new_found_policy_indexes.pop()
node_to_policy)
expected_policy_index = policies.pop()
self.assertEqual(orig_policy_index, expected_policy_index)
# validate object fully deleted
for policy_index in found_policy_indexes:
@ -257,7 +256,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
manifest_data = []
def write_part(i):
body = 'VERIFY%0.2d' % i + '\x00' * 1048576
body = b'VERIFY%0.2d' % i + b'\x00' * 1048576
part_name = 'manifest_part_%0.2d' % i
manifest_entry = {
"path": "/%s/%s" % (self.container_name, part_name),
@ -310,7 +309,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
'bytes': data['size_bytes'],
'name': data['path'],
}
direct_manifest_data = map(translate_direct, manifest_data)
direct_manifest_data = [translate_direct(item)
for item in manifest_data]
headers = {
'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n
in self.container_ring.devs),
@ -320,11 +320,12 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
'X-Backend-Storage-Policy-Index': wrong_policy.idx,
'X-Static-Large-Object': 'True',
}
body = utils.json.dumps(direct_manifest_data).encode('ascii')
for node in nodes:
direct_client.direct_put_object(
node, part, self.account, self.container_name,
direct_manifest_name,
contents=utils.json.dumps(direct_manifest_data),
contents=body,
headers=headers)
break # one should do it...
@ -347,8 +348,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.assertEqual(metadata['x-static-large-object'].lower(), 'true')
self.assertEqual(int(metadata['content-length']),
sum(part['size_bytes'] for part in manifest_data))
self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576
for i in range(20)))
self.assertEqual(body, b''.join(b'VERIFY%0.2d' % i + b'\x00' * 1048576
for i in range(20)))
# and regular upload should work now too
client.put_object(self.url, self.token, self.container_name,
@ -375,7 +376,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.brain.start_primary_half()
# write some target data
client.put_object(self.url, self.token, self.container_name, 'target',
contents='this is the target data')
contents=b'this is the target data')
# write the symlink
self.brain.stop_handoff_half()
@ -401,7 +402,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
self.assertEqual(metadata['x-symlink-target'],
'%s/target' % self.container_name)
self.assertEqual(metadata['content-type'], 'application/symlink')
self.assertEqual(body, '')
self.assertEqual(body, b'')
# ... although in the wrong policy
object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift')
part, nodes = object_ring.get_nodes(
@ -423,7 +424,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
# now the symlink works
metadata, body = client.get_object(self.url, self.token,
self.container_name, 'symlink')
self.assertEqual(body, 'this is the target data')
self.assertEqual(body, b'this is the target data')
# and it's in the correct policy
object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift')
part, nodes = object_ring.get_nodes(
@ -469,7 +470,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
# hopefully memcache still has the new policy cached
self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'},
contents='VERIFY')
contents=b'VERIFY')
# double-check object correctly written to new policy
conf_files = []
for server in Manager(['container-reconciler']).servers:
@ -546,7 +547,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest):
# verify that the object data read by external client is correct
headers, data = self._get_object_patiently(int(new_policy))
self.assertEqual('VERIFY', data)
self.assertEqual(b'VERIFY', data)
self.assertEqual('custom-meta', headers['x-object-meta-test'])

28
test/probe/test_container_sync.py
View File

@ -113,7 +113,7 @@ class TestContainerSync(BaseTestContainerSync):
resp_headers, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
self.assertEqual(body, b'test-body')
self.assertIn('x-object-meta-test', resp_headers)
self.assertEqual('put_value', resp_headers['x-object-meta-test'])
@ -136,7 +136,7 @@ class TestContainerSync(BaseTestContainerSync):
# verify that metadata changes were sync'd
resp_headers, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
self.assertEqual(body, b'test-body')
self.assertIn('x-object-meta-test', resp_headers)
self.assertEqual('post_value', resp_headers['x-object-meta-test'])
self.assertEqual('image/jpeg', resp_headers['content-type'])
@ -180,7 +180,7 @@ class TestContainerSync(BaseTestContainerSync):
# upload a segment to source
segment_name = 'segment-%s' % uuid.uuid4()
segment_data = 'segment body' # it's ok for first segment to be small
segment_data = b'segment body' # it's ok for first segment to be small
segment_etag = client.put_object(
self.url, self.token, segs_container, segment_name,
segment_data)
@ -270,7 +270,7 @@ class TestContainerSync(BaseTestContainerSync):
Manager(['container-sync']).once()
_junk, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
self.assertEqual(body, b'test-body')
def test_sync_lazy_dkey(self):
# Create synced containers, but with no key at dest
@ -297,7 +297,7 @@ class TestContainerSync(BaseTestContainerSync):
Manager(['container-sync']).once()
_junk, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
self.assertEqual(body, b'test-body')
def test_sync_with_stale_container_rows(self):
source_container, dest_container = self._setup_synced_containers()
@ -351,7 +351,7 @@ class TestContainerSync(BaseTestContainerSync):
# verify sync'd object has same content and headers
dest_headers, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'new-test-body')
self.assertEqual(body, b'new-test-body')
mismatched_headers = []
for k in ('etag', 'content-length', 'content-type', 'x-timestamp',
'last-modified'):
@ -381,7 +381,7 @@ class TestContainerSync(BaseTestContainerSync):
# verify that the remote object did not change
resp_headers, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'new-test-body')
self.assertEqual(body, b'new-test-body')
class TestContainerSyncAndSymlink(BaseTestContainerSync):
@ -413,7 +413,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
# upload a target to source
target_name = 'target-%s' % uuid.uuid4()
target_body = 'target body'
target_body = b'target body'
client.put_object(
self.url, self.token, tgt_container, target_name,
target_body)
@ -432,7 +432,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
resp_headers, symlink_body = client.get_object(
self.url, self.token, source_container, symlink_name,
query_string='symlink=get')
self.assertEqual('', symlink_body)
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
# verify symlink behavior
@ -453,7 +453,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
resp_headers, symlink_body = client.get_object(
dest_account['url'], dest_account['token'], dest_container,
symlink_name, query_string='symlink=get')
self.assertEqual('', symlink_body)
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
# attempt to GET the target object via symlink will fail because
@ -480,7 +480,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
resp_headers, symlink_body = client.get_object(
dest_account['url'], dest_account['token'], dest_container,
symlink_name, query_string='symlink=get')
self.assertEqual('', symlink_body)
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
# verify GET of target object via symlink now succeeds
@ -511,7 +511,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
# upload a target to source
target_name = 'target-%s' % uuid.uuid4()
target_body = 'target body'
target_body = b'target body'
client.put_object(tgt_account['url'], tgt_account['token'],
tgt_container, target_name, target_body)
@ -531,7 +531,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
resp_headers, symlink_body = client.get_object(
self.url, self.token, source_container, symlink_name,
query_string='symlink=get')
self.assertEqual('', symlink_body)
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
self.assertIn('x-symlink-target-account', resp_headers)
@ -553,7 +553,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync):
resp_headers, symlink_body = client.get_object(
self.url, self.token, dest_container,
symlink_name, query_string='symlink=get')
self.assertEqual('', symlink_body)
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
self.assertIn('x-symlink-target-account', resp_headers)

4
test/probe/test_empty_device_handoff.py
View File

@ -69,7 +69,7 @@ class TestEmptyDevice(ReplProbeTest):
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
@ -87,7 +87,7 @@ class TestEmptyDevice(ReplProbeTest):
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Restart those other two container/obj primary servers

6
test/probe/test_object_async_update.py
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from io import StringIO
from io import BytesIO
from unittest import main, SkipTest
from uuid import uuid4
@ -82,7 +82,7 @@ class TestObjectAsyncUpdate(ReplProbeTest):
self.assertEqual(err.http_status, 503)
# Assert handoff device has a container replica
another_cnode = self.container_ring.get_more_nodes(cpart).next()
another_cnode = next(self.container_ring.get_more_nodes(cpart))
direct_client.direct_get_container(
another_cnode, cpart, self.account, container)
@ -143,7 +143,7 @@ class TestUpdateOverrides(ReplProbeTest):
self.policy.name})
int_client.upload_object(
StringIO(u'stuff'), self.account, 'c1', 'o1', headers)
BytesIO(b'stuff'), self.account, 'c1', 'o1', headers)
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()

12
test/probe/test_object_conditional_requests.py
View File

@ -40,7 +40,7 @@ class TestPutIfNoneMatchRepl(ReplProbeTest):
self.brain.put_container()
self.brain.stop_primary_half()
# put object to only 1 of 3 primaries
self.brain.put_object(contents='VERIFY')
self.brain.put_object(contents=b'VERIFY')
self.brain.start_primary_half()
# Restart services and attempt to overwrite
@ -64,16 +64,16 @@ class TestPutIfNoneMatchRepl(ReplProbeTest):
# ...and verify the object was not overwritten
_headers, body = client.get_object(
self.url, self.token, self.container_name, self.object_name)
self.assertEqual(body, 'VERIFY')
self.assertEqual(body, b'VERIFY')
def test_content_length_nonzero(self):
self._do_test('OVERWRITE')
self._do_test(b'OVERWRITE')
def test_content_length_zero(self):
self._do_test('')
self._do_test(b'')
def test_chunked(self):
self._do_test(chunker('OVERWRITE'))
self._do_test(chunker(b'OVERWRITE'))
def test_chunked_empty(self):
self._do_test(chunker(''))
self._do_test(chunker(b''))

17
test/probe/test_object_failures.py
View File

@ -78,7 +78,7 @@ class TestObjectFailures(ReplProbeTest):
container = 'container-%s' % uuid4()
obj = 'object-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj,
'VERIFY')
b'VERIFY')
# Stash the on disk data for future comparison - this may not equal
# 'VERIFY' if for example the proxy has crypto enabled
backend_data = direct_client.direct_get_object(
@ -105,7 +105,7 @@ class TestObjectFailures(ReplProbeTest):
container = 'container-range-%s' % uuid4()
obj = 'object-range-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj,
'RANGE')
b'RANGE')
# Stash the on disk data for future comparison - this may not equal
# 'VERIFY' if for example the proxy has crypto enabled
backend_data = direct_client.direct_get_object(
@ -137,7 +137,8 @@ class TestObjectFailures(ReplProbeTest):
def run_quarantine_zero_byte_get(self):
container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
onode, opart, data_file = self._setup_data_file(
container, obj, b'DATA')
metadata = read_metadata(data_file)
unlink(data_file)
@ -155,7 +156,8 @@ class TestObjectFailures(ReplProbeTest):
def run_quarantine_zero_byte_head(self):
container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
onode, opart, data_file = self._setup_data_file(
container, obj, b'DATA')
metadata = read_metadata(data_file)
unlink(data_file)
@ -173,7 +175,8 @@ class TestObjectFailures(ReplProbeTest):
def run_quarantine_zero_byte_post(self):
container = 'container-zbyte-%s' % uuid4()
obj = 'object-zbyte-%s' % uuid4()
onode, opart, data_file = self._setup_data_file(container, obj, 'DATA')
onode, opart, data_file = self._setup_data_file(
container, obj, b'DATA')
metadata = read_metadata(data_file)
unlink(data_file)
@ -217,7 +220,7 @@ class TestECObjectFailures(ECProbeTest):
# PUT object, should go to primary nodes
client.put_object(self.url, self.token, container_name,
object_name, contents='object contents')
object_name, contents=b'object contents')
# get our node lists
opart, onodes = self.object_ring.get_nodes(
@ -226,7 +229,7 @@ class TestECObjectFailures(ECProbeTest):
# sanity test
odata = client.get_object(self.url, self.token, container_name,
object_name)[-1]
self.assertEqual('object contents', odata)
self.assertEqual(b'object contents', odata)
# make all fragments non-durable
for node in onodes:

22
test/probe/test_object_handoff.py
View File

@ -53,9 +53,9 @@ class TestObjectHandoff(ReplProbeTest):
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primary servers and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
@ -73,7 +73,7 @@ class TestObjectHandoff(ReplProbeTest):
# Indirectly through proxy assert we can get container/obj
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
@ -139,13 +139,13 @@ class TestObjectHandoff(ReplProbeTest):
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
node_id = (port_num - 6000) // 10
Manager(['object-replicator']).once(number=node_id)
try:
another_port_num = another_onode['replication_port']
except KeyError:
another_port_num = another_onode['port']
another_num = (another_port_num - 6000) / 10
another_num = (another_port_num - 6000) // 10
Manager(['object-replicator']).once(number=another_num)
# Assert the first container/obj primary server now has container/obj
@ -231,9 +231,9 @@ class TestObjectHandoff(ReplProbeTest):
port_num = node['replication_port']
except KeyError:
port_num = node['port']
node_id = (port_num - 6000) / 10
node_id = (port_num - 6000) // 10
Manager(['object-replicator']).once(number=node_id)
another_node_id = (another_port_num - 6000) / 10
another_node_id = (another_port_num - 6000) // 10
Manager(['object-replicator']).once(number=another_node_id)
# Assert primary node no longer has container/obj
@ -261,9 +261,9 @@ class TestObjectHandoff(ReplProbeTest):
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Create container/obj (goes to two primaries and one handoff)
client.put_object(self.url, self.token, container, obj, 'VERIFY')
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
@ -318,9 +318,9 @@ class TestObjectHandoff(ReplProbeTest):
# Create container/obj (goes to all three primaries)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, 'VERIFY')
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != 'VERIFY':
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))

10
test/probe/test_object_metadata_replication.py
View File

@ -13,7 +13,7 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from io import StringIO
from io import BytesIO
import unittest
import os
@ -134,9 +134,9 @@ class Test(ReplProbeTest):
modified = Timestamp(metadata['x-timestamp']).isoformat
self.assertEqual(listing['last_modified'], modified)
def _put_object(self, headers=None, body=u'stuff'):
def _put_object(self, headers=None, body=b'stuff'):
headers = headers or {}
self.int_client.upload_object(StringIO(body), self.account,
self.int_client.upload_object(BytesIO(body), self.account,
self.container_name,
self.object_name, headers)
@ -232,12 +232,12 @@ class Test(ReplProbeTest):
self.brain.put_container()
# put object
self._put_object(headers={'Content-Type': 'foo'}, body=u'older')
self._put_object(headers={'Content-Type': 'foo'}, body=b'older')
# put newer object to first server subset
self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar'}, body=u'newer')
self._put_object(headers={'Content-Type': 'bar'}, body=b'newer')
metadata = self._get_object_metadata()
etag = metadata['etag']
self.brain.start_primary_half()

4
test/probe/test_object_partpower_increase.py
View File

@ -51,8 +51,8 @@ class TestPartPowerIncrease(ProbeTest):
self.devices = [
self.device_dir('object', {'ip': ip, 'port': port, 'device': ''})
for ip, port in set((dev['ip'], dev['port'])
for dev in self.object_ring.devs)]
for ip, port in {(dev['ip'], dev['port'])
for dev in self.object_ring.devs}]
def tearDown(self):
# Keep a backup copy of the modified .builder file

32
test/probe/test_reconstructor_rebuild.py
View File

@ -24,6 +24,7 @@ import shutil
import random
import os
import time
import six
from swift.common.direct_client import DirectClientException
from test.probe.common import ECProbeTest
@ -38,10 +39,10 @@ from swiftclient import client, ClientException
class Body(object):
def __init__(self, total=3.5 * 2 ** 20):
self.total = total
self.total = int(total)
self.hasher = md5()
self.size = 0
self.chunk = 'test' * 16 * 2 ** 10
self.chunk = b'test' * 16 * 2 ** 10
@property
def etag(self):
@ -50,21 +51,21 @@ class Body(object):
def __iter__(self):
return self
def next(self):
def __next__(self):
if self.size > self.total:
raise StopIteration()
self.size += len(self.chunk)
self.hasher.update(self.chunk)
return self.chunk
def __next__(self):
return next(self)
# for py2 compat
next = __next__
class TestReconstructorRebuild(ECProbeTest):
def _make_name(self, prefix):
return '%s%s' % (prefix, uuid.uuid4())
return ('%s%s' % (prefix, uuid.uuid4())).encode()
def setUp(self):
super(TestReconstructorRebuild, self).setUp()
@ -131,10 +132,13 @@ class TestReconstructorRebuild(ECProbeTest):
{'X-Backend-Fragment-Preferences': json.dumps([])})
# node dict has unicode values so utf8 decode our path parts too in
# case they have non-ascii characters
if six.PY2:
acc, con, obj = (s.decode('utf8') for s in (
self.account, self.container_name, self.object_name))
else:
acc, con, obj = self.account, self.container_name, self.object_name
headers, data = direct_client.direct_get_object(
node, part, self.account.decode('utf8'),
self.container_name.decode('utf8'),
self.object_name.decode('utf8'), headers=req_headers,
node, part, acc, con, obj, headers=req_headers,
resp_chunk_size=64 * 2 ** 20)
hasher = md5()
for chunk in data:
@ -402,7 +406,7 @@ class TestReconstructorRebuild(ECProbeTest):
opart, onodes = self.object_ring.get_nodes(
self.account, self.container_name, self.object_name)
delete_at = int(time.time() + 3)
contents = 'body-%s' % uuid.uuid4()
contents = ('body-%s' % uuid.uuid4()).encode()
headers = {'x-delete-at': delete_at}
client.put_object(self.url, self.token, self.container_name,
self.object_name, headers=headers, contents=contents)
@ -418,15 +422,17 @@ class TestReconstructorRebuild(ECProbeTest):
# wait for the delete_at to pass, and check that it thinks the object
# is expired
timeout = time.time() + 5
err = None
while time.time() < timeout:
try:
direct_client.direct_head_object(
post_fail_node, opart, self.account, self.container_name,
self.object_name, headers={
'X-Backend-Storage-Policy-Index': int(self.policy)})
except direct_client.ClientException as err:
if err.http_status != 404:
except direct_client.ClientException as client_err:
if client_err.http_status != 404:
raise
err = client_err
break
else:
time.sleep(0.1)
@ -454,7 +460,7 @@ class TestReconstructorRebuild(ECProbeTest):
class TestReconstructorRebuildUTF8(TestReconstructorRebuild):
def _make_name(self, prefix):
return '%s\xc3\xa8-%s' % (prefix, uuid.uuid4())
return b'%s\xc3\xa8-%s' % (prefix.encode(), str(uuid.uuid4()).encode())
if __name__ == "__main__":

2
test/probe/test_reconstructor_revert.py
View File

@ -132,7 +132,7 @@ class TestReconstructorRevert(ECProbeTest):
# fire up reconstructor on handoff nodes only
for hnode in hnodes:
hnode_id = (hnode['port'] - 6000) / 10
hnode_id = (hnode['port'] - 6000) // 10
self.reconstructor.once(number=hnode_id)
# first three primaries have data again

73
test/probe/test_sharder.py
View File

@ -19,6 +19,7 @@ import shutil
import uuid
from nose import SkipTest
import six
from swift.common import direct_client, utils
from swift.common.manager import Manager
@ -65,8 +66,8 @@ class BaseTestContainerSharding(ReplProbeTest):
'container-server configs')
skip_reasons = []
auto_shard = all([config_true_value(c.get('auto_shard', False))
for c in cont_configs])
auto_shard = all(config_true_value(c.get('auto_shard', False))
for c in cont_configs)
if not auto_shard:
skip_reasons.append(
'auto_shard must be true in all container_sharder configs')
@ -83,7 +84,7 @@ class BaseTestContainerSharding(ReplProbeTest):
MAX_SHARD_CONTAINER_THRESHOLD))
def skip_check(reason_list, option, required):
values = set([int(c.get(option, required)) for c in cont_configs])
values = {int(c.get(option, required)) for c in cont_configs}
if values != {required}:
reason_list.append('%s must be %s' % (option, required))
@ -259,8 +260,8 @@ class BaseTestContainerSharding(ReplProbeTest):
def assert_dict_contains(self, expected_items, actual_dict):
ignored = set(expected_items) ^ set(actual_dict)
filtered_actual = dict((k, actual_dict[k])
for k in actual_dict if k not in ignored)
filtered_actual = {k: actual_dict[k]
for k in actual_dict if k not in ignored}
self.assertEqual(expected_items, filtered_actual)
def assert_shard_ranges_contiguous(self, expected_number, shard_ranges,
@ -268,8 +269,8 @@ class BaseTestContainerSharding(ReplProbeTest):
if shard_ranges and isinstance(shard_ranges[0], ShardRange):
actual_shard_ranges = sorted(shard_ranges)
else:
actual_shard_ranges = sorted([ShardRange.from_dict(d)
for d in shard_ranges])
actual_shard_ranges = sorted(ShardRange.from_dict(d)
for d in shard_ranges)
self.assertLengthEqual(actual_shard_ranges, expected_number)
if expected_number:
with annotate_failure('Ranges %s.' % actual_shard_ranges):
@ -300,7 +301,7 @@ class BaseTestContainerSharding(ReplProbeTest):
[sr.state for sr in shard_ranges])
def assert_total_object_count(self, expected_object_count, shard_ranges):
actual = sum([sr['object_count'] for sr in shard_ranges])
actual = sum(sr['object_count'] for sr in shard_ranges)
self.assertEqual(expected_object_count, actual)
def assert_container_listing(self, expected_listing):
@ -309,7 +310,8 @@ class BaseTestContainerSharding(ReplProbeTest):
self.assertIn('x-container-object-count', headers)
expected_obj_count = len(expected_listing)
self.assertEqual(expected_listing, [
x['name'].encode('utf-8') for x in actual_listing])
x['name'].encode('utf-8') if six.PY2 else x['name']
for x in actual_listing])
self.assertEqual(str(expected_obj_count),
headers['x-container-object-count'])
return headers, actual_listing
@ -407,15 +409,16 @@ class TestContainerShardingNonUTF8(BaseTestContainerSharding):
self.put_objects(obj_names)
# choose some names approx in middle of each expected shard range
markers = [
obj_names[i] for i in range(self.max_shard_size / 4,
obj_names[i] for i in range(self.max_shard_size // 4,
2 * self.max_shard_size,
self.max_shard_size / 2)]
self.max_shard_size // 2)]
def check_listing(objects, **params):
qs = '&'.join(['%s=%s' % param for param in params.items()])
headers, listing = client.get_container(
self.url, self.token, self.container_name, query_string=qs)
listing = [x['name'].encode('utf-8') for x in listing]
listing = [x['name'].encode('utf-8') if six.PY2 else x['name']
for x in listing]
if params.get('reverse'):
marker = params.get('marker', ShardRange.MAX)
end_marker = params.get('end_marker', ShardRange.MIN)
@ -443,16 +446,17 @@ class TestContainerShardingNonUTF8(BaseTestContainerSharding):
check_listing(objects, marker=markers[0], end_marker=markers[2])
check_listing(objects, marker=markers[1], end_marker=markers[3])
check_listing(objects, marker=markers[1], end_marker=markers[3],
limit=self.max_shard_size / 4)
limit=self.max_shard_size // 4)
check_listing(objects, marker=markers[1], end_marker=markers[3],
limit=self.max_shard_size / 4)
limit=self.max_shard_size // 4)
check_listing(objects, marker=markers[1], end_marker=markers[2],
limit=self.max_shard_size / 2)
limit=self.max_shard_size // 2)
check_listing(objects, marker=markers[1], end_marker=markers[1])
check_listing(objects, reverse=True)
check_listing(objects, reverse=True, end_marker=markers[1])
check_listing(objects, reverse=True, marker=markers[3],
end_marker=markers[1], limit=self.max_shard_size / 4)
end_marker=markers[1],
limit=self.max_shard_size // 4)
check_listing(objects, reverse=True, marker=markers[3],
end_marker=markers[1], limit=0)
check_listing([], marker=markers[0], end_marker=markers[0])
@ -468,9 +472,9 @@ class TestContainerShardingNonUTF8(BaseTestContainerSharding):
limit = self.cluster_info['swift']['container_listing_limit']
exc = check_listing_precondition_fails(limit=limit + 1)
self.assertIn('Maximum limit', exc.http_response_content)
self.assertIn(b'Maximum limit', exc.http_response_content)
exc = check_listing_precondition_fails(delimiter='ab')
self.assertIn('Bad delimiter', exc.http_response_content)
self.assertIn(b'Bad delimiter', exc.http_response_content)
# sanity checks
do_listing_checks(obj_names)
@ -544,7 +548,9 @@ class TestContainerShardingUTF8(TestContainerShardingNonUTF8):
obj_names = []
for x in range(number):
name = (u'obj-\u00e4\u00ea\u00ec\u00f2\u00fb-%04d' % x)
name = name.encode('utf8').ljust(name_length, 'o')
name = name.encode('utf8').ljust(name_length, b'o')
if not six.PY2:
name = name.decode('utf8')
obj_names.append(name)
return obj_names
@ -553,7 +559,9 @@ class TestContainerShardingUTF8(TestContainerShardingNonUTF8):
super(TestContainerShardingUTF8, self)._setup_container_name()
name_length = self.cluster_info['swift']['max_container_name_length']
cont_name = self.container_name + u'-\u00e4\u00ea\u00ec\u00f2\u00fb'
self.conainer_name = cont_name.encode('utf8').ljust(name_length, 'x')
self.conainer_name = cont_name.ljust(name_length, 'x')
if six.PY2:
self.conainer_name = self.container_name.encode('utf8')
class TestContainerSharding(BaseTestContainerSharding):
@ -573,8 +581,9 @@ class TestContainerSharding(BaseTestContainerSharding):
headers, pre_sharding_listing = client.get_container(
self.url, self.token, self.container_name)
self.assertEqual(obj_names, [x['name'].encode('utf-8')
for x in pre_sharding_listing]) # sanity
self.assertEqual(obj_names, [
x['name'].encode('utf-8') if six.PY2 else x['name']
for x in pre_sharding_listing]) # sanity
# Shard it
client.post_container(self.url, self.admin_token, self.container_name,
@ -901,7 +910,7 @@ class TestContainerSharding(BaseTestContainerSharding):
old_shard_range = by_name.pop(
orig_root_shard_ranges[0]['name'])
self.assertTrue(old_shard_range.deleted)
self.assert_shard_ranges_contiguous(4, by_name.values())
self.assert_shard_ranges_contiguous(4, list(by_name.values()))
else:
# Everyone's on the same page. Well, except for
# meta_timestamps, since the shards each reported
@ -1070,26 +1079,29 @@ class TestContainerSharding(BaseTestContainerSharding):
start_listing = [
o for o in obj_names if o <= expected_shard_ranges[1].upper]
self.assertEqual(
[x['name'].encode('utf-8') for x in listing[:len(start_listing)]],
[x['name'].encode('utf-8') if six.PY2 else x['name']
for x in listing[:len(start_listing)]],
start_listing)
# we can't assert much about the remaining listing, other than that
# there should be something
self.assertTrue(
[x['name'].encode('utf-8') for x in listing[len(start_listing):]])
[x['name'].encode('utf-8') if six.PY2 else x['name']
for x in listing[len(start_listing):]])
self.assertIn('x-container-object-count', headers)
self.assertEqual(str(len(listing)),
headers['x-container-object-count'])
headers, listing = client.get_container(self.url, self.token,
self.container_name,
query_string='reverse=on')
self.assertEqual([x['name'].encode('utf-8')
self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name']
for x in listing[-len(start_listing):]],
list(reversed(start_listing)))
self.assertIn('x-container-object-count', headers)
self.assertEqual(str(len(listing)),
headers['x-container-object-count'])
self.assertTrue(
[x['name'].encode('utf-8') for x in listing[:-len(start_listing)]])
[x['name'].encode('utf-8') if six.PY2 else x['name']
for x in listing[:-len(start_listing)]])
# Run the sharders again to get everything to settle
self.sharders.once()
@ -1099,7 +1111,8 @@ class TestContainerSharding(BaseTestContainerSharding):
# now all shards have been cleaved we should get the complete listing
headers, listing = client.get_container(self.url, self.token,
self.container_name)
self.assertEqual([x['name'].encode('utf-8') for x in listing],
self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name']
for x in listing],
obj_names)
def test_shrinking(self):
@ -1409,7 +1422,7 @@ class TestContainerSharding(BaseTestContainerSharding):
# put objects while all servers are up
obj_names = self._make_object_names(
num_shards * self.max_shard_size / 2)
num_shards * self.max_shard_size // 2)
self.put_objects(obj_names)
client.post_container(self.url, self.admin_token, self.container_name,
@ -2004,7 +2017,7 @@ class TestContainerSharding(BaseTestContainerSharding):
if n['id'] not in primary_ids)
num_shards = 3
obj_names = self._make_object_names(
num_shards * self.max_shard_size / 2)
num_shards * self.max_shard_size // 2)
self.put_objects(obj_names)
client.post_container(self.url, self.admin_token, self.container_name,
headers={'X-Container-Sharding': 'on'})

14
test/probe/test_signals.py
View File

@ -48,10 +48,10 @@ class TestWSGIServerProcessHandling(unittest.TestCase):
manager = Manager([server_name])
manager.start()
starting_pids = set(pid for server in manager.servers
for (_, pid) in server.iter_pid_files())
starting_pids = {pid for server in manager.servers
for (_, pid) in server.iter_pid_files()}
body = 'test' * 10
body = b'test' * 10
conn = httplib.HTTPConnection('%s:%s' % (ip, port))
# sanity request
@ -68,8 +68,8 @@ class TestWSGIServerProcessHandling(unittest.TestCase):
manager.reload()
post_reload_pids = set(pid for server in manager.servers
for (_, pid) in server.iter_pid_files())
post_reload_pids = {pid for server in manager.servers
for (_, pid) in server.iter_pid_files()}
# none of the pids we started with are being tracked after reload
msg = 'expected all pids from %r to have died, but found %r' % (
@ -92,8 +92,8 @@ class TestWSGIServerProcessHandling(unittest.TestCase):
conn.close()
# sanity
post_close_pids = set(pid for server in manager.servers
for (_, pid) in server.iter_pid_files())
post_close_pids = {pid for server in manager.servers
for (_, pid) in server.iter_pid_files()}
self.assertEqual(post_reload_pids, post_close_pids)
def test_proxy_reload(self):