From 1d7e1558b3b422073918b89df21f703215bd1e33 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 16 Jul 2019 17:01:19 -0700 Subject: [PATCH] py3: (mostly) port probe tests There's still one problem, though: since swiftclient on py3 doesn't support non-ASCII characters in metadata names, none of the tests in TestReconstructorRebuildUTF8 will pass. Change-Id: I4ec879ade534e09c3a625414d8aa1f16fd600fa4 --- test/probe/common.py | 18 ++--- .../test_account_get_fake_responses_match.py | 6 +- test/probe/test_container_failures.py | 2 +- .../test_container_merge_policy_index.py | 59 +++++++-------- test/probe/test_container_sync.py | 28 +++---- test/probe/test_empty_device_handoff.py | 4 +- test/probe/test_object_async_update.py | 6 +- .../probe/test_object_conditional_requests.py | 12 +-- test/probe/test_object_failures.py | 17 +++-- test/probe/test_object_handoff.py | 22 +++--- .../probe/test_object_metadata_replication.py | 10 +-- test/probe/test_object_partpower_increase.py | 4 +- test/probe/test_reconstructor_rebuild.py | 32 ++++---- test/probe/test_reconstructor_revert.py | 2 +- test/probe/test_sharder.py | 73 +++++++++++-------- test/probe/test_signals.py | 14 ++-- 16 files changed, 166 insertions(+), 143 deletions(-) diff --git a/test/probe/common.py b/test/probe/common.py index b205aa0516..6392b00163 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -164,7 +164,7 @@ def add_ring_devs_to_ipport2server(ring, server_type, ipport2server, # We'll number the servers by order of unique occurrence of: # IP, if servers_per_port > 0 OR there > 1 IP in ring # ipport, otherwise - unique_ip_count = len(set(dev['ip'] for dev in ring.devs if dev)) + unique_ip_count = len({dev['ip'] for dev in ring.devs if dev}) things_to_number = {} number = 0 for dev in filter(None, ring.devs): @@ -244,7 +244,7 @@ def get_ring(ring_name, required_replicas, required_devices, if p.returncode: raise unittest.SkipTest('unable to connect to rsync ' 'export %s (%s)' % (rsync_export, cmd)) - for line in stdout.splitlines(): + for line in stdout.decode().splitlines(): if line.rsplit(None, 1)[-1] == dev['device']: break else: @@ -295,11 +295,11 @@ def kill_orphans(): class Body(object): def __init__(self, total=3.5 * 2 ** 20): - self.length = total + self.length = int(total) self.hasher = md5() self.read_amount = 0 - self.chunk = uuid4().hex * 2 ** 10 - self.buff = '' + self.chunk = uuid4().hex.encode('ascii') * 2 ** 10 + self.buff = b'' @property def etag(self): @@ -320,9 +320,9 @@ class Body(object): def __iter__(self): return self - def next(self): + def __next__(self): if self.buff: - rv, self.buff = self.buff, '' + rv, self.buff = self.buff, b'' return rv if self.read_amount >= self.length: raise StopIteration() @@ -331,8 +331,8 @@ class Body(object): self.hasher.update(rv) return rv - def __next__(self): - return next(self) + # for py2 compat: + next = __next__ class ProbeTest(unittest.TestCase): diff --git a/test/probe/test_account_get_fake_responses_match.py b/test/probe/test_account_get_fake_responses_match.py index bd159300ea..ba664518b5 100644 --- a/test/probe/test_account_get_fake_responses_match.py +++ b/test/probe/test_account_get_fake_responses_match.py @@ -56,7 +56,7 @@ class TestAccountGetFakeResponsesMatch(ReplProbeTest): raise Exception("Unexpected status %s\n%s" % (resp.status, resp.read())) - response_headers = dict(resp.getheaders()) + response_headers = {h.lower(): v for h, v in resp.getheaders()} response_body = resp.read() resp.close() return response_headers, response_body @@ -98,8 +98,8 @@ class TestAccountGetFakeResponsesMatch(ReplProbeTest): fake_acct, headers={'Accept': 'application/xml'}) # the account name is in the XML response - real_body = re.sub('AUTH_\w{4}', 'AUTH_someaccount', real_body) - fake_body = re.sub('AUTH_\w{4}', 'AUTH_someaccount', fake_body) + real_body = re.sub(br'AUTH_\w{4}', b'AUTH_someaccount', real_body) + fake_body = re.sub(br'AUTH_\w{4}', b'AUTH_someaccount', fake_body) self.assertEqual(real_body, fake_body) self.assertEqual(real_headers['content-type'], diff --git a/test/probe/test_container_failures.py b/test/probe/test_container_failures.py index bc5c4e7298..6d8156088e 100644 --- a/test/probe/test_container_failures.py +++ b/test/probe/test_container_failures.py @@ -130,7 +130,7 @@ class TestContainerFailures(ReplProbeTest): onode = onodes[0] db_files = [] for onode in onodes: - node_id = (onode['port'] - 6000) / 10 + node_id = (onode['port'] - 6000) // 10 device = onode['device'] hash_str = hash_path(self.account, container) server_conf = readconf(self.configs['container-server'][node_id]) diff --git a/test/probe/test_container_merge_policy_index.py b/test/probe/test_container_merge_policy_index.py index 4f1e65d035..ccfea1bbf2 100644 --- a/test/probe/test_container_merge_policy_index.py +++ b/test/probe/test_container_merge_policy_index.py @@ -68,7 +68,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.brain.stop_handoff_half() self.brain.put_container() self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'}, - contents='VERIFY') + contents=b'VERIFY') self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( @@ -78,9 +78,9 @@ class TestContainerMergePolicyIndex(ReplProbeTest): metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) - found_policy_indexes = \ - set(metadata['X-Backend-Storage-Policy-Index'] for - node, metadata in head_responses) + found_policy_indexes = { + metadata['X-Backend-Storage-Policy-Index'] + for node, metadata in head_responses} self.assertGreater( len(found_policy_indexes), 1, 'primary nodes did not disagree about policy index %r' % @@ -116,9 +116,9 @@ class TestContainerMergePolicyIndex(ReplProbeTest): metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) - found_policy_indexes = \ - set(metadata['X-Backend-Storage-Policy-Index'] for - node, metadata in head_responses) + found_policy_indexes = { + metadata['X-Backend-Storage-Policy-Index'] + for node, metadata in head_responses} self.assertEqual(len(found_policy_indexes), 1, 'primary nodes disagree about policy index %r' % head_responses) @@ -144,7 +144,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): orig_policy_index)) # verify that the object data read by external client is correct headers, data = self._get_object_patiently(expected_policy_index) - self.assertEqual('VERIFY', data) + self.assertEqual(b'VERIFY', data) self.assertEqual('custom-meta', headers['x-object-meta-test']) def test_reconcile_delete(self): @@ -165,9 +165,9 @@ class TestContainerMergePolicyIndex(ReplProbeTest): metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) - found_policy_indexes = \ - set(metadata['X-Backend-Storage-Policy-Index'] for - node, metadata in head_responses) + found_policy_indexes = { + metadata['X-Backend-Storage-Policy-Index'] + for node, metadata in head_responses} self.assertGreater( len(found_policy_indexes), 1, 'primary nodes did not disagree about policy index %r' % @@ -208,15 +208,14 @@ class TestContainerMergePolicyIndex(ReplProbeTest): metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) - new_found_policy_indexes = \ - set(metadata['X-Backend-Storage-Policy-Index'] for node, - metadata in head_responses) - self.assertEqual(len(new_found_policy_indexes), 1, + node_to_policy = { + node['port']: metadata['X-Backend-Storage-Policy-Index'] + for node, metadata in head_responses} + policies = set(node_to_policy.values()) + self.assertEqual(len(policies), 1, 'primary nodes disagree about policy index %r' % - dict((node['port'], - metadata['X-Backend-Storage-Policy-Index']) - for node, metadata in head_responses)) - expected_policy_index = new_found_policy_indexes.pop() + node_to_policy) + expected_policy_index = policies.pop() self.assertEqual(orig_policy_index, expected_policy_index) # validate object fully deleted for policy_index in found_policy_indexes: @@ -257,7 +256,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): manifest_data = [] def write_part(i): - body = 'VERIFY%0.2d' % i + '\x00' * 1048576 + body = b'VERIFY%0.2d' % i + b'\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), @@ -310,7 +309,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest): 'bytes': data['size_bytes'], 'name': data['path'], } - direct_manifest_data = map(translate_direct, manifest_data) + direct_manifest_data = [translate_direct(item) + for item in manifest_data] headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), @@ -320,11 +320,12 @@ class TestContainerMergePolicyIndex(ReplProbeTest): 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } + body = utils.json.dumps(direct_manifest_data).encode('ascii') for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, - contents=utils.json.dumps(direct_manifest_data), + contents=body, headers=headers) break # one should do it... @@ -347,8 +348,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) - self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 - for i in range(20))) + self.assertEqual(body, b''.join(b'VERIFY%0.2d' % i + b'\x00' * 1048576 + for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, @@ -375,7 +376,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.brain.start_primary_half() # write some target data client.put_object(self.url, self.token, self.container_name, 'target', - contents='this is the target data') + contents=b'this is the target data') # write the symlink self.brain.stop_handoff_half() @@ -401,7 +402,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.assertEqual(metadata['x-symlink-target'], '%s/target' % self.container_name) self.assertEqual(metadata['content-type'], 'application/symlink') - self.assertEqual(body, '') + self.assertEqual(body, b'') # ... although in the wrong policy object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift') part, nodes = object_ring.get_nodes( @@ -423,7 +424,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): # now the symlink works metadata, body = client.get_object(self.url, self.token, self.container_name, 'symlink') - self.assertEqual(body, 'this is the target data') + self.assertEqual(body, b'this is the target data') # and it's in the correct policy object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift') part, nodes = object_ring.get_nodes( @@ -469,7 +470,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): # hopefully memcache still has the new policy cached self.brain.put_object(headers={'x-object-meta-test': 'custom-meta'}, - contents='VERIFY') + contents=b'VERIFY') # double-check object correctly written to new policy conf_files = [] for server in Manager(['container-reconciler']).servers: @@ -546,7 +547,7 @@ class TestContainerMergePolicyIndex(ReplProbeTest): # verify that the object data read by external client is correct headers, data = self._get_object_patiently(int(new_policy)) - self.assertEqual('VERIFY', data) + self.assertEqual(b'VERIFY', data) self.assertEqual('custom-meta', headers['x-object-meta-test']) diff --git a/test/probe/test_container_sync.py b/test/probe/test_container_sync.py index 138c6b0531..f04941c444 100644 --- a/test/probe/test_container_sync.py +++ b/test/probe/test_container_sync.py @@ -113,7 +113,7 @@ class TestContainerSync(BaseTestContainerSync): resp_headers, body = client.get_object(self.url, self.token, dest_container, object_name) - self.assertEqual(body, 'test-body') + self.assertEqual(body, b'test-body') self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('put_value', resp_headers['x-object-meta-test']) @@ -136,7 +136,7 @@ class TestContainerSync(BaseTestContainerSync): # verify that metadata changes were sync'd resp_headers, body = client.get_object(self.url, self.token, dest_container, object_name) - self.assertEqual(body, 'test-body') + self.assertEqual(body, b'test-body') self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('post_value', resp_headers['x-object-meta-test']) self.assertEqual('image/jpeg', resp_headers['content-type']) @@ -180,7 +180,7 @@ class TestContainerSync(BaseTestContainerSync): # upload a segment to source segment_name = 'segment-%s' % uuid.uuid4() - segment_data = 'segment body' # it's ok for first segment to be small + segment_data = b'segment body' # it's ok for first segment to be small segment_etag = client.put_object( self.url, self.token, segs_container, segment_name, segment_data) @@ -270,7 +270,7 @@ class TestContainerSync(BaseTestContainerSync): Manager(['container-sync']).once() _junk, body = client.get_object(self.url, self.token, dest_container, object_name) - self.assertEqual(body, 'test-body') + self.assertEqual(body, b'test-body') def test_sync_lazy_dkey(self): # Create synced containers, but with no key at dest @@ -297,7 +297,7 @@ class TestContainerSync(BaseTestContainerSync): Manager(['container-sync']).once() _junk, body = client.get_object(self.url, self.token, dest_container, object_name) - self.assertEqual(body, 'test-body') + self.assertEqual(body, b'test-body') def test_sync_with_stale_container_rows(self): source_container, dest_container = self._setup_synced_containers() @@ -351,7 +351,7 @@ class TestContainerSync(BaseTestContainerSync): # verify sync'd object has same content and headers dest_headers, body = client.get_object(self.url, self.token, dest_container, object_name) - self.assertEqual(body, 'new-test-body') + self.assertEqual(body, b'new-test-body') mismatched_headers = [] for k in ('etag', 'content-length', 'content-type', 'x-timestamp', 'last-modified'): @@ -381,7 +381,7 @@ class TestContainerSync(BaseTestContainerSync): # verify that the remote object did not change resp_headers, body = client.get_object(self.url, self.token, dest_container, object_name) - self.assertEqual(body, 'new-test-body') + self.assertEqual(body, b'new-test-body') class TestContainerSyncAndSymlink(BaseTestContainerSync): @@ -413,7 +413,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): # upload a target to source target_name = 'target-%s' % uuid.uuid4() - target_body = 'target body' + target_body = b'target body' client.put_object( self.url, self.token, tgt_container, target_name, target_body) @@ -432,7 +432,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): resp_headers, symlink_body = client.get_object( self.url, self.token, source_container, symlink_name, query_string='symlink=get') - self.assertEqual('', symlink_body) + self.assertEqual(b'', symlink_body) self.assertIn('x-symlink-target', resp_headers) # verify symlink behavior @@ -453,7 +453,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): resp_headers, symlink_body = client.get_object( dest_account['url'], dest_account['token'], dest_container, symlink_name, query_string='symlink=get') - self.assertEqual('', symlink_body) + self.assertEqual(b'', symlink_body) self.assertIn('x-symlink-target', resp_headers) # attempt to GET the target object via symlink will fail because @@ -480,7 +480,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): resp_headers, symlink_body = client.get_object( dest_account['url'], dest_account['token'], dest_container, symlink_name, query_string='symlink=get') - self.assertEqual('', symlink_body) + self.assertEqual(b'', symlink_body) self.assertIn('x-symlink-target', resp_headers) # verify GET of target object via symlink now succeeds @@ -511,7 +511,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): # upload a target to source target_name = 'target-%s' % uuid.uuid4() - target_body = 'target body' + target_body = b'target body' client.put_object(tgt_account['url'], tgt_account['token'], tgt_container, target_name, target_body) @@ -531,7 +531,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): resp_headers, symlink_body = client.get_object( self.url, self.token, source_container, symlink_name, query_string='symlink=get') - self.assertEqual('', symlink_body) + self.assertEqual(b'', symlink_body) self.assertIn('x-symlink-target', resp_headers) self.assertIn('x-symlink-target-account', resp_headers) @@ -553,7 +553,7 @@ class TestContainerSyncAndSymlink(BaseTestContainerSync): resp_headers, symlink_body = client.get_object( self.url, self.token, dest_container, symlink_name, query_string='symlink=get') - self.assertEqual('', symlink_body) + self.assertEqual(b'', symlink_body) self.assertIn('x-symlink-target', resp_headers) self.assertIn('x-symlink-target-account', resp_headers) diff --git a/test/probe/test_empty_device_handoff.py b/test/probe/test_empty_device_handoff.py index e1f8ade50c..36e2084eb9 100644 --- a/test/probe/test_empty_device_handoff.py +++ b/test/probe/test_empty_device_handoff.py @@ -69,7 +69,7 @@ class TestEmptyDevice(ReplProbeTest): # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] - if odata != 'VERIFY': + if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) @@ -87,7 +87,7 @@ class TestEmptyDevice(ReplProbeTest): # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] - if odata != 'VERIFY': + if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers diff --git a/test/probe/test_object_async_update.py b/test/probe/test_object_async_update.py index ee573adeaf..44a25eb148 100644 --- a/test/probe/test_object_async_update.py +++ b/test/probe/test_object_async_update.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from io import StringIO +from io import BytesIO from unittest import main, SkipTest from uuid import uuid4 @@ -82,7 +82,7 @@ class TestObjectAsyncUpdate(ReplProbeTest): self.assertEqual(err.http_status, 503) # Assert handoff device has a container replica - another_cnode = self.container_ring.get_more_nodes(cpart).next() + another_cnode = next(self.container_ring.get_more_nodes(cpart)) direct_client.direct_get_container( another_cnode, cpart, self.account, container) @@ -143,7 +143,7 @@ class TestUpdateOverrides(ReplProbeTest): self.policy.name}) int_client.upload_object( - StringIO(u'stuff'), self.account, 'c1', 'o1', headers) + BytesIO(b'stuff'), self.account, 'c1', 'o1', headers) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() diff --git a/test/probe/test_object_conditional_requests.py b/test/probe/test_object_conditional_requests.py index 29021ba681..e66cb0eaf8 100644 --- a/test/probe/test_object_conditional_requests.py +++ b/test/probe/test_object_conditional_requests.py @@ -40,7 +40,7 @@ class TestPutIfNoneMatchRepl(ReplProbeTest): self.brain.put_container() self.brain.stop_primary_half() # put object to only 1 of 3 primaries - self.brain.put_object(contents='VERIFY') + self.brain.put_object(contents=b'VERIFY') self.brain.start_primary_half() # Restart services and attempt to overwrite @@ -64,16 +64,16 @@ class TestPutIfNoneMatchRepl(ReplProbeTest): # ...and verify the object was not overwritten _headers, body = client.get_object( self.url, self.token, self.container_name, self.object_name) - self.assertEqual(body, 'VERIFY') + self.assertEqual(body, b'VERIFY') def test_content_length_nonzero(self): - self._do_test('OVERWRITE') + self._do_test(b'OVERWRITE') def test_content_length_zero(self): - self._do_test('') + self._do_test(b'') def test_chunked(self): - self._do_test(chunker('OVERWRITE')) + self._do_test(chunker(b'OVERWRITE')) def test_chunked_empty(self): - self._do_test(chunker('')) + self._do_test(chunker(b'')) diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py index 49502d5232..94cda07b8a 100644 --- a/test/probe/test_object_failures.py +++ b/test/probe/test_object_failures.py @@ -78,7 +78,7 @@ class TestObjectFailures(ReplProbeTest): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, - 'VERIFY') + b'VERIFY') # Stash the on disk data for future comparison - this may not equal # 'VERIFY' if for example the proxy has crypto enabled backend_data = direct_client.direct_get_object( @@ -105,7 +105,7 @@ class TestObjectFailures(ReplProbeTest): container = 'container-range-%s' % uuid4() obj = 'object-range-%s' % uuid4() onode, opart, data_file = self._setup_data_file(container, obj, - 'RANGE') + b'RANGE') # Stash the on disk data for future comparison - this may not equal # 'VERIFY' if for example the proxy has crypto enabled backend_data = direct_client.direct_get_object( @@ -137,7 +137,8 @@ class TestObjectFailures(ReplProbeTest): def run_quarantine_zero_byte_get(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() - onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') + onode, opart, data_file = self._setup_data_file( + container, obj, b'DATA') metadata = read_metadata(data_file) unlink(data_file) @@ -155,7 +156,8 @@ class TestObjectFailures(ReplProbeTest): def run_quarantine_zero_byte_head(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() - onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') + onode, opart, data_file = self._setup_data_file( + container, obj, b'DATA') metadata = read_metadata(data_file) unlink(data_file) @@ -173,7 +175,8 @@ class TestObjectFailures(ReplProbeTest): def run_quarantine_zero_byte_post(self): container = 'container-zbyte-%s' % uuid4() obj = 'object-zbyte-%s' % uuid4() - onode, opart, data_file = self._setup_data_file(container, obj, 'DATA') + onode, opart, data_file = self._setup_data_file( + container, obj, b'DATA') metadata = read_metadata(data_file) unlink(data_file) @@ -217,7 +220,7 @@ class TestECObjectFailures(ECProbeTest): # PUT object, should go to primary nodes client.put_object(self.url, self.token, container_name, - object_name, contents='object contents') + object_name, contents=b'object contents') # get our node lists opart, onodes = self.object_ring.get_nodes( @@ -226,7 +229,7 @@ class TestECObjectFailures(ECProbeTest): # sanity test odata = client.get_object(self.url, self.token, container_name, object_name)[-1] - self.assertEqual('object contents', odata) + self.assertEqual(b'object contents', odata) # make all fragments non-durable for node in onodes: diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py index 6c2ecb14bb..7338a88f97 100644 --- a/test/probe/test_object_handoff.py +++ b/test/probe/test_object_handoff.py @@ -53,9 +53,9 @@ class TestObjectHandoff(ReplProbeTest): kill_server((onode['ip'], onode['port']), self.ipport2server) # Create container/obj (goes to two primary servers and one handoff) - client.put_object(self.url, self.token, container, obj, 'VERIFY') + client.put_object(self.url, self.token, container, obj, b'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] - if odata != 'VERIFY': + if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) @@ -73,7 +73,7 @@ class TestObjectHandoff(ReplProbeTest): # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] - if odata != 'VERIFY': + if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) @@ -139,13 +139,13 @@ class TestObjectHandoff(ReplProbeTest): port_num = node['replication_port'] except KeyError: port_num = node['port'] - node_id = (port_num - 6000) / 10 + node_id = (port_num - 6000) // 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] - another_num = (another_port_num - 6000) / 10 + another_num = (another_port_num - 6000) // 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj @@ -231,9 +231,9 @@ class TestObjectHandoff(ReplProbeTest): port_num = node['replication_port'] except KeyError: port_num = node['port'] - node_id = (port_num - 6000) / 10 + node_id = (port_num - 6000) // 10 Manager(['object-replicator']).once(number=node_id) - another_node_id = (another_port_num - 6000) / 10 + another_node_id = (another_port_num - 6000) // 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj @@ -261,9 +261,9 @@ class TestObjectHandoff(ReplProbeTest): kill_server((onode['ip'], onode['port']), self.ipport2server) # Create container/obj (goes to two primaries and one handoff) - client.put_object(self.url, self.token, container, obj, 'VERIFY') + client.put_object(self.url, self.token, container, obj, b'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] - if odata != 'VERIFY': + if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) @@ -318,9 +318,9 @@ class TestObjectHandoff(ReplProbeTest): # Create container/obj (goes to all three primaries) obj = 'object-%s' % uuid4() - client.put_object(self.url, self.token, container, obj, 'VERIFY') + client.put_object(self.url, self.token, container, obj, b'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] - if odata != 'VERIFY': + if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) diff --git a/test/probe/test_object_metadata_replication.py b/test/probe/test_object_metadata_replication.py index ca2e8dde1d..c5cb93f98d 100644 --- a/test/probe/test_object_metadata_replication.py +++ b/test/probe/test_object_metadata_replication.py @@ -13,7 +13,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from io import StringIO +from io import BytesIO import unittest import os @@ -134,9 +134,9 @@ class Test(ReplProbeTest): modified = Timestamp(metadata['x-timestamp']).isoformat self.assertEqual(listing['last_modified'], modified) - def _put_object(self, headers=None, body=u'stuff'): + def _put_object(self, headers=None, body=b'stuff'): headers = headers or {} - self.int_client.upload_object(StringIO(body), self.account, + self.int_client.upload_object(BytesIO(body), self.account, self.container_name, self.object_name, headers) @@ -232,12 +232,12 @@ class Test(ReplProbeTest): self.brain.put_container() # put object - self._put_object(headers={'Content-Type': 'foo'}, body=u'older') + self._put_object(headers={'Content-Type': 'foo'}, body=b'older') # put newer object to first server subset self.brain.stop_primary_half() self.container_brain.stop_primary_half() - self._put_object(headers={'Content-Type': 'bar'}, body=u'newer') + self._put_object(headers={'Content-Type': 'bar'}, body=b'newer') metadata = self._get_object_metadata() etag = metadata['etag'] self.brain.start_primary_half() diff --git a/test/probe/test_object_partpower_increase.py b/test/probe/test_object_partpower_increase.py index e0eb5499ec..39b52cf7e2 100755 --- a/test/probe/test_object_partpower_increase.py +++ b/test/probe/test_object_partpower_increase.py @@ -51,8 +51,8 @@ class TestPartPowerIncrease(ProbeTest): self.devices = [ self.device_dir('object', {'ip': ip, 'port': port, 'device': ''}) - for ip, port in set((dev['ip'], dev['port']) - for dev in self.object_ring.devs)] + for ip, port in {(dev['ip'], dev['port']) + for dev in self.object_ring.devs}] def tearDown(self): # Keep a backup copy of the modified .builder file diff --git a/test/probe/test_reconstructor_rebuild.py b/test/probe/test_reconstructor_rebuild.py index 5ebfb7381f..7f580d74f7 100644 --- a/test/probe/test_reconstructor_rebuild.py +++ b/test/probe/test_reconstructor_rebuild.py @@ -24,6 +24,7 @@ import shutil import random import os import time +import six from swift.common.direct_client import DirectClientException from test.probe.common import ECProbeTest @@ -38,10 +39,10 @@ from swiftclient import client, ClientException class Body(object): def __init__(self, total=3.5 * 2 ** 20): - self.total = total + self.total = int(total) self.hasher = md5() self.size = 0 - self.chunk = 'test' * 16 * 2 ** 10 + self.chunk = b'test' * 16 * 2 ** 10 @property def etag(self): @@ -50,21 +51,21 @@ class Body(object): def __iter__(self): return self - def next(self): + def __next__(self): if self.size > self.total: raise StopIteration() self.size += len(self.chunk) self.hasher.update(self.chunk) return self.chunk - def __next__(self): - return next(self) + # for py2 compat + next = __next__ class TestReconstructorRebuild(ECProbeTest): def _make_name(self, prefix): - return '%s%s' % (prefix, uuid.uuid4()) + return ('%s%s' % (prefix, uuid.uuid4())).encode() def setUp(self): super(TestReconstructorRebuild, self).setUp() @@ -131,10 +132,13 @@ class TestReconstructorRebuild(ECProbeTest): {'X-Backend-Fragment-Preferences': json.dumps([])}) # node dict has unicode values so utf8 decode our path parts too in # case they have non-ascii characters + if six.PY2: + acc, con, obj = (s.decode('utf8') for s in ( + self.account, self.container_name, self.object_name)) + else: + acc, con, obj = self.account, self.container_name, self.object_name headers, data = direct_client.direct_get_object( - node, part, self.account.decode('utf8'), - self.container_name.decode('utf8'), - self.object_name.decode('utf8'), headers=req_headers, + node, part, acc, con, obj, headers=req_headers, resp_chunk_size=64 * 2 ** 20) hasher = md5() for chunk in data: @@ -402,7 +406,7 @@ class TestReconstructorRebuild(ECProbeTest): opart, onodes = self.object_ring.get_nodes( self.account, self.container_name, self.object_name) delete_at = int(time.time() + 3) - contents = 'body-%s' % uuid.uuid4() + contents = ('body-%s' % uuid.uuid4()).encode() headers = {'x-delete-at': delete_at} client.put_object(self.url, self.token, self.container_name, self.object_name, headers=headers, contents=contents) @@ -418,15 +422,17 @@ class TestReconstructorRebuild(ECProbeTest): # wait for the delete_at to pass, and check that it thinks the object # is expired timeout = time.time() + 5 + err = None while time.time() < timeout: try: direct_client.direct_head_object( post_fail_node, opart, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': int(self.policy)}) - except direct_client.ClientException as err: - if err.http_status != 404: + except direct_client.ClientException as client_err: + if client_err.http_status != 404: raise + err = client_err break else: time.sleep(0.1) @@ -454,7 +460,7 @@ class TestReconstructorRebuild(ECProbeTest): class TestReconstructorRebuildUTF8(TestReconstructorRebuild): def _make_name(self, prefix): - return '%s\xc3\xa8-%s' % (prefix, uuid.uuid4()) + return b'%s\xc3\xa8-%s' % (prefix.encode(), str(uuid.uuid4()).encode()) if __name__ == "__main__": diff --git a/test/probe/test_reconstructor_revert.py b/test/probe/test_reconstructor_revert.py index c518440a15..a96e92ddc8 100644 --- a/test/probe/test_reconstructor_revert.py +++ b/test/probe/test_reconstructor_revert.py @@ -132,7 +132,7 @@ class TestReconstructorRevert(ECProbeTest): # fire up reconstructor on handoff nodes only for hnode in hnodes: - hnode_id = (hnode['port'] - 6000) / 10 + hnode_id = (hnode['port'] - 6000) // 10 self.reconstructor.once(number=hnode_id) # first three primaries have data again diff --git a/test/probe/test_sharder.py b/test/probe/test_sharder.py index 6724e05e2e..b49eeee44c 100644 --- a/test/probe/test_sharder.py +++ b/test/probe/test_sharder.py @@ -19,6 +19,7 @@ import shutil import uuid from nose import SkipTest +import six from swift.common import direct_client, utils from swift.common.manager import Manager @@ -65,8 +66,8 @@ class BaseTestContainerSharding(ReplProbeTest): 'container-server configs') skip_reasons = [] - auto_shard = all([config_true_value(c.get('auto_shard', False)) - for c in cont_configs]) + auto_shard = all(config_true_value(c.get('auto_shard', False)) + for c in cont_configs) if not auto_shard: skip_reasons.append( 'auto_shard must be true in all container_sharder configs') @@ -83,7 +84,7 @@ class BaseTestContainerSharding(ReplProbeTest): MAX_SHARD_CONTAINER_THRESHOLD)) def skip_check(reason_list, option, required): - values = set([int(c.get(option, required)) for c in cont_configs]) + values = {int(c.get(option, required)) for c in cont_configs} if values != {required}: reason_list.append('%s must be %s' % (option, required)) @@ -259,8 +260,8 @@ class BaseTestContainerSharding(ReplProbeTest): def assert_dict_contains(self, expected_items, actual_dict): ignored = set(expected_items) ^ set(actual_dict) - filtered_actual = dict((k, actual_dict[k]) - for k in actual_dict if k not in ignored) + filtered_actual = {k: actual_dict[k] + for k in actual_dict if k not in ignored} self.assertEqual(expected_items, filtered_actual) def assert_shard_ranges_contiguous(self, expected_number, shard_ranges, @@ -268,8 +269,8 @@ class BaseTestContainerSharding(ReplProbeTest): if shard_ranges and isinstance(shard_ranges[0], ShardRange): actual_shard_ranges = sorted(shard_ranges) else: - actual_shard_ranges = sorted([ShardRange.from_dict(d) - for d in shard_ranges]) + actual_shard_ranges = sorted(ShardRange.from_dict(d) + for d in shard_ranges) self.assertLengthEqual(actual_shard_ranges, expected_number) if expected_number: with annotate_failure('Ranges %s.' % actual_shard_ranges): @@ -300,7 +301,7 @@ class BaseTestContainerSharding(ReplProbeTest): [sr.state for sr in shard_ranges]) def assert_total_object_count(self, expected_object_count, shard_ranges): - actual = sum([sr['object_count'] for sr in shard_ranges]) + actual = sum(sr['object_count'] for sr in shard_ranges) self.assertEqual(expected_object_count, actual) def assert_container_listing(self, expected_listing): @@ -309,7 +310,8 @@ class BaseTestContainerSharding(ReplProbeTest): self.assertIn('x-container-object-count', headers) expected_obj_count = len(expected_listing) self.assertEqual(expected_listing, [ - x['name'].encode('utf-8') for x in actual_listing]) + x['name'].encode('utf-8') if six.PY2 else x['name'] + for x in actual_listing]) self.assertEqual(str(expected_obj_count), headers['x-container-object-count']) return headers, actual_listing @@ -407,15 +409,16 @@ class TestContainerShardingNonUTF8(BaseTestContainerSharding): self.put_objects(obj_names) # choose some names approx in middle of each expected shard range markers = [ - obj_names[i] for i in range(self.max_shard_size / 4, + obj_names[i] for i in range(self.max_shard_size // 4, 2 * self.max_shard_size, - self.max_shard_size / 2)] + self.max_shard_size // 2)] def check_listing(objects, **params): qs = '&'.join(['%s=%s' % param for param in params.items()]) headers, listing = client.get_container( self.url, self.token, self.container_name, query_string=qs) - listing = [x['name'].encode('utf-8') for x in listing] + listing = [x['name'].encode('utf-8') if six.PY2 else x['name'] + for x in listing] if params.get('reverse'): marker = params.get('marker', ShardRange.MAX) end_marker = params.get('end_marker', ShardRange.MIN) @@ -443,16 +446,17 @@ class TestContainerShardingNonUTF8(BaseTestContainerSharding): check_listing(objects, marker=markers[0], end_marker=markers[2]) check_listing(objects, marker=markers[1], end_marker=markers[3]) check_listing(objects, marker=markers[1], end_marker=markers[3], - limit=self.max_shard_size / 4) + limit=self.max_shard_size // 4) check_listing(objects, marker=markers[1], end_marker=markers[3], - limit=self.max_shard_size / 4) + limit=self.max_shard_size // 4) check_listing(objects, marker=markers[1], end_marker=markers[2], - limit=self.max_shard_size / 2) + limit=self.max_shard_size // 2) check_listing(objects, marker=markers[1], end_marker=markers[1]) check_listing(objects, reverse=True) check_listing(objects, reverse=True, end_marker=markers[1]) check_listing(objects, reverse=True, marker=markers[3], - end_marker=markers[1], limit=self.max_shard_size / 4) + end_marker=markers[1], + limit=self.max_shard_size // 4) check_listing(objects, reverse=True, marker=markers[3], end_marker=markers[1], limit=0) check_listing([], marker=markers[0], end_marker=markers[0]) @@ -468,9 +472,9 @@ class TestContainerShardingNonUTF8(BaseTestContainerSharding): limit = self.cluster_info['swift']['container_listing_limit'] exc = check_listing_precondition_fails(limit=limit + 1) - self.assertIn('Maximum limit', exc.http_response_content) + self.assertIn(b'Maximum limit', exc.http_response_content) exc = check_listing_precondition_fails(delimiter='ab') - self.assertIn('Bad delimiter', exc.http_response_content) + self.assertIn(b'Bad delimiter', exc.http_response_content) # sanity checks do_listing_checks(obj_names) @@ -544,7 +548,9 @@ class TestContainerShardingUTF8(TestContainerShardingNonUTF8): obj_names = [] for x in range(number): name = (u'obj-\u00e4\u00ea\u00ec\u00f2\u00fb-%04d' % x) - name = name.encode('utf8').ljust(name_length, 'o') + name = name.encode('utf8').ljust(name_length, b'o') + if not six.PY2: + name = name.decode('utf8') obj_names.append(name) return obj_names @@ -553,7 +559,9 @@ class TestContainerShardingUTF8(TestContainerShardingNonUTF8): super(TestContainerShardingUTF8, self)._setup_container_name() name_length = self.cluster_info['swift']['max_container_name_length'] cont_name = self.container_name + u'-\u00e4\u00ea\u00ec\u00f2\u00fb' - self.conainer_name = cont_name.encode('utf8').ljust(name_length, 'x') + self.conainer_name = cont_name.ljust(name_length, 'x') + if six.PY2: + self.conainer_name = self.container_name.encode('utf8') class TestContainerSharding(BaseTestContainerSharding): @@ -573,8 +581,9 @@ class TestContainerSharding(BaseTestContainerSharding): headers, pre_sharding_listing = client.get_container( self.url, self.token, self.container_name) - self.assertEqual(obj_names, [x['name'].encode('utf-8') - for x in pre_sharding_listing]) # sanity + self.assertEqual(obj_names, [ + x['name'].encode('utf-8') if six.PY2 else x['name'] + for x in pre_sharding_listing]) # sanity # Shard it client.post_container(self.url, self.admin_token, self.container_name, @@ -901,7 +910,7 @@ class TestContainerSharding(BaseTestContainerSharding): old_shard_range = by_name.pop( orig_root_shard_ranges[0]['name']) self.assertTrue(old_shard_range.deleted) - self.assert_shard_ranges_contiguous(4, by_name.values()) + self.assert_shard_ranges_contiguous(4, list(by_name.values())) else: # Everyone's on the same page. Well, except for # meta_timestamps, since the shards each reported @@ -1070,26 +1079,29 @@ class TestContainerSharding(BaseTestContainerSharding): start_listing = [ o for o in obj_names if o <= expected_shard_ranges[1].upper] self.assertEqual( - [x['name'].encode('utf-8') for x in listing[:len(start_listing)]], + [x['name'].encode('utf-8') if six.PY2 else x['name'] + for x in listing[:len(start_listing)]], start_listing) # we can't assert much about the remaining listing, other than that # there should be something self.assertTrue( - [x['name'].encode('utf-8') for x in listing[len(start_listing):]]) + [x['name'].encode('utf-8') if six.PY2 else x['name'] + for x in listing[len(start_listing):]]) self.assertIn('x-container-object-count', headers) self.assertEqual(str(len(listing)), headers['x-container-object-count']) headers, listing = client.get_container(self.url, self.token, self.container_name, query_string='reverse=on') - self.assertEqual([x['name'].encode('utf-8') + self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name'] for x in listing[-len(start_listing):]], list(reversed(start_listing))) self.assertIn('x-container-object-count', headers) self.assertEqual(str(len(listing)), headers['x-container-object-count']) self.assertTrue( - [x['name'].encode('utf-8') for x in listing[:-len(start_listing)]]) + [x['name'].encode('utf-8') if six.PY2 else x['name'] + for x in listing[:-len(start_listing)]]) # Run the sharders again to get everything to settle self.sharders.once() @@ -1099,7 +1111,8 @@ class TestContainerSharding(BaseTestContainerSharding): # now all shards have been cleaved we should get the complete listing headers, listing = client.get_container(self.url, self.token, self.container_name) - self.assertEqual([x['name'].encode('utf-8') for x in listing], + self.assertEqual([x['name'].encode('utf-8') if six.PY2 else x['name'] + for x in listing], obj_names) def test_shrinking(self): @@ -1409,7 +1422,7 @@ class TestContainerSharding(BaseTestContainerSharding): # put objects while all servers are up obj_names = self._make_object_names( - num_shards * self.max_shard_size / 2) + num_shards * self.max_shard_size // 2) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, @@ -2004,7 +2017,7 @@ class TestContainerSharding(BaseTestContainerSharding): if n['id'] not in primary_ids) num_shards = 3 obj_names = self._make_object_names( - num_shards * self.max_shard_size / 2) + num_shards * self.max_shard_size // 2) self.put_objects(obj_names) client.post_container(self.url, self.admin_token, self.container_name, headers={'X-Container-Sharding': 'on'}) diff --git a/test/probe/test_signals.py b/test/probe/test_signals.py index f2867fbb5b..bfc6e299a3 100644 --- a/test/probe/test_signals.py +++ b/test/probe/test_signals.py @@ -48,10 +48,10 @@ class TestWSGIServerProcessHandling(unittest.TestCase): manager = Manager([server_name]) manager.start() - starting_pids = set(pid for server in manager.servers - for (_, pid) in server.iter_pid_files()) + starting_pids = {pid for server in manager.servers + for (_, pid) in server.iter_pid_files()} - body = 'test' * 10 + body = b'test' * 10 conn = httplib.HTTPConnection('%s:%s' % (ip, port)) # sanity request @@ -68,8 +68,8 @@ class TestWSGIServerProcessHandling(unittest.TestCase): manager.reload() - post_reload_pids = set(pid for server in manager.servers - for (_, pid) in server.iter_pid_files()) + post_reload_pids = {pid for server in manager.servers + for (_, pid) in server.iter_pid_files()} # none of the pids we started with are being tracked after reload msg = 'expected all pids from %r to have died, but found %r' % ( @@ -92,8 +92,8 @@ class TestWSGIServerProcessHandling(unittest.TestCase): conn.close() # sanity - post_close_pids = set(pid for server in manager.servers - for (_, pid) in server.iter_pid_files()) + post_close_pids = {pid for server in manager.servers + for (_, pid) in server.iter_pid_files()} self.assertEqual(post_reload_pids, post_close_pids) def test_proxy_reload(self):