From 8b519d1abcb4d7b34c8bbdfe535f472fe105231c Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 1 Mar 2019 12:13:27 -0800 Subject: [PATCH] Clean up func tests ahead of py3 - ConfigParser.set() requires that the value be a string - The stdlib HTTP client responses don't have a body property - We might raise a ResponseError with response=None - Bodies should be bytes - Headers should be strings - Make containers()/files() return native strings - file() isn't a thing on py3 - format should be a parm, not a header - Switch sorted() to use key instead of cmp - Use integer division explicitly Change-Id: I99d3eebc9d7ec4e8b295352294b831492135c568 --- test/functional/__init__.py | 8 +-- test/functional/swift_test_client.py | 89 ++++++++++++++++------------ test/functional/tests.py | 72 +++++++++++++--------- 3 files changed, 97 insertions(+), 72 deletions(-) diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 6435d175c6..472ce92837 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -150,7 +150,7 @@ def _in_process_setup_swift_conf(swift_conf_src, testdir): conf.set(section, 'swift_hash_path_prefix', 'inprocfunctests') section = 'swift-constraints' max_file_size = (8 * 1024 * 1024) + 2 # 8 MB + 2 - conf.set(section, 'max_file_size', max_file_size) + conf.set(section, 'max_file_size', str(max_file_size)) except NoSectionError: msg = 'Conf file %s is missing section %s' % (swift_conf_src, section) raise InProcessException(msg) @@ -232,8 +232,8 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir): sp_zero_section = sp_prefix + '0' conf.add_section(sp_zero_section) for (k, v) in policy_to_test.get_info(config=True).items(): - conf.set(sp_zero_section, k, v) - conf.set(sp_zero_section, 'default', True) + conf.set(sp_zero_section, k, str(v)) + conf.set(sp_zero_section, 'default', 'True') with open(swift_conf, 'w') as fp: conf.write(fp) @@ -714,7 +714,7 @@ def in_process_setup(the_object_server=object_server): '/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) resp = conn.getresponse() assert resp.status == 201, 'Unable to create account: %s\n%s' % ( - resp.status, resp.body) + resp.status, resp.read()) create_account('AUTH_test') create_account('AUTH_test2') diff --git a/test/functional/swift_test_client.py b/test/functional/swift_test_client.py index d742b4982c..c6af80d1a2 100644 --- a/test/functional/swift_test_client.py +++ b/test/functional/swift_test_client.py @@ -49,11 +49,11 @@ class RequestError(Exception): class ResponseError(Exception): def __init__(self, response, method=None, path=None, details=None): - self.status = response.status - self.reason = response.reason + self.status = getattr(response, 'status', 0) + self.reason = getattr(response, 'reason', '[unknown]') self.method = method self.path = path - self.headers = response.getheaders() + self.headers = getattr(response, 'getheaders', lambda: [])() self.details = details for name, value in self.headers: @@ -269,7 +269,7 @@ class Connection(object): headers.update(hdrs) return headers - def make_request(self, method, path=None, data='', hdrs=None, parms=None, + def make_request(self, method, path=None, data=b'', hdrs=None, parms=None, cfg=None): if path is None: path = [] @@ -294,9 +294,9 @@ class Connection(object): path = '%s?%s' % (path, '&'.join(query_args)) if not cfg.get('no_content_length'): if cfg.get('set_content_length'): - headers['Content-Length'] = cfg.get('set_content_length') + headers['Content-Length'] = str(cfg.get('set_content_length')) else: - headers['Content-Length'] = len(data) + headers['Content-Length'] = str(len(data)) def try_request(): self.http_connect() @@ -377,13 +377,13 @@ class Connection(object): def put_data(self, data, chunked=False): if chunked: - self.connection.send('%x\r\n%s\r\n' % (len(data), data)) + self.connection.send(b'%x\r\n%s\r\n' % (len(data), data)) else: self.connection.send(data) def put_end(self, chunked=False): if chunked: - self.connection.send('0\r\n\r\n') + self.connection.send(b'0\r\n\r\n') self.response = self.connection.getresponse() # Hope it isn't big! @@ -418,8 +418,8 @@ class Base(object): for return_key, header in required_fields: if header not in headers: - raise ValueError("%s was not found in response header" % - (header,)) + raise ValueError("%s was not found in response headers: %r" % + (header, headers)) if is_int_header(header): ret[return_key] = int(headers[header]) @@ -478,8 +478,9 @@ class Account(Base): if status == 200: if format_type == 'json': conts = json.loads(self.conn.response.read()) - for cont in conts: - cont['name'] = cont['name'].encode('utf-8') + if six.PY2: + for cont in conts: + cont['name'] = cont['name'].encode('utf-8') return conts elif format_type == 'xml': conts = [] @@ -491,13 +492,18 @@ class Account(Base): childNodes[0].nodeValue conts.append(cont) for cont in conts: - cont['name'] = cont['name'].encode('utf-8') + if six.PY2: + cont['name'] = cont['name'].encode('utf-8') + for key in ('count', 'bytes'): + cont[key] = int(cont[key]) return conts else: - lines = self.conn.response.read().split('\n') + lines = self.conn.response.read().split(b'\n') if lines and not lines[-1]: lines = lines[:-1] - return lines + if six.PY2: + return lines + return [line.decode('utf-8') for line in lines] elif status == 204: return [] @@ -617,10 +623,11 @@ class Container(Base): if format_type == 'json': files = json.loads(self.conn.response.read()) - for file_item in files: - for key in ('name', 'subdir', 'content_type'): - if key in file_item: - file_item[key] = file_item[key].encode('utf-8') + if six.PY2: + for file_item in files: + for key in ('name', 'subdir', 'content_type'): + if key in file_item: + file_item[key] = file_item[key].encode('utf-8') return files elif format_type == 'xml': files = [] @@ -643,28 +650,32 @@ class Container(Base): for file_item in files: if 'subdir' in file_item: - file_item['subdir'] = file_item['subdir'].\ - encode('utf-8') + if six.PY2: + file_item['subdir'] = \ + file_item['subdir'].encode('utf-8') else: - file_item['name'] = file_item['name'].encode('utf-8') - file_item['content_type'] = file_item['content_type'].\ - encode('utf-8') + if six.PY2: + file_item.update({ + k: file_item[k].encode('utf-8') + for k in ('name', 'content_type')}) file_item['bytes'] = int(file_item['bytes']) return files else: content = self.conn.response.read() if content: - lines = content.split('\n') + lines = content.split(b'\n') if lines and not lines[-1]: lines = lines[:-1] - return lines + if six.PY2: + return lines + return [line.decode('utf-8') for line in lines] else: return [] elif status == 204: return [] raise ResponseError(self.conn.response, 'GET', - self.conn.make_path(self.path)) + self.conn.make_path(self.path, cfg=cfg)) def info(self, hdrs=None, parms=None, cfg=None): if hdrs is None: @@ -719,11 +730,11 @@ class File(Base): headers = {} if not cfg.get('no_content_length'): if cfg.get('set_content_length'): - headers['Content-Length'] = cfg.get('set_content_length') + headers['Content-Length'] = str(cfg.get('set_content_length')) elif self.size: - headers['Content-Length'] = self.size + headers['Content-Length'] = str(self.size) else: - headers['Content-Length'] = 0 + headers['Content-Length'] = '0' if cfg.get('use_token'): headers['X-Auth-Token'] = cfg.get('use_token') @@ -744,8 +755,8 @@ class File(Base): def compute_md5sum(cls, data): block_size = 4096 - if isinstance(data, str): - data = six.StringIO(data) + if isinstance(data, bytes): + data = six.BytesIO(data) checksum = hashlib.md5() buff = data.read(block_size) @@ -894,7 +905,7 @@ class File(Base): def random_data(cls, size=None): if size is None: size = random.randint(1, 32768) - fd = open('/dev/urandom', 'r') + fd = open('/dev/urandom', 'rb') data = fd.read(size) fd.close() return data @@ -973,10 +984,10 @@ class File(Base): headers = self.make_headers(cfg=cfg) if not cfg.get('no_content_length'): if cfg.get('set_content_length'): - headers['Content-Length'] = \ - cfg.get('set_content_length') + headers['Content-Length'] = str( + cfg.get('set_content_length')) else: - headers['Content-Length'] = 0 + headers['Content-Length'] = '0' self.conn.make_request('POST', self.path, hdrs=headers, parms=parms, cfg=cfg) @@ -1024,7 +1035,7 @@ class File(Base): block_size = 2 ** 20 - if isinstance(data, file): + if all(hasattr(data, attr) for attr in ('flush', 'seek', 'fileno')): try: data.flush() data.seek(0) @@ -1086,7 +1097,7 @@ class File(Base): if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): raise ResponseError(self.conn.response, 'PUT', self.conn.make_path(self.path)) - self.md5 = self.compute_md5sum(six.StringIO(data)) + self.md5 = self.compute_md5sum(six.BytesIO(data)) return data def write_random_return_resp(self, size=None, hdrs=None, parms=None, @@ -1103,7 +1114,7 @@ class File(Base): return_resp=True) if not resp: raise ResponseError(self.conn.response) - self.md5 = self.compute_md5sum(six.StringIO(data)) + self.md5 = self.compute_md5sum(six.BytesIO(data)) return resp def post(self, hdrs=None, parms=None, cfg=None, return_resp=False): diff --git a/test/functional/tests.py b/test/functional/tests.py index 29fea5afb1..e3dbc5fc10 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -63,8 +63,11 @@ class Utils(object): u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\ u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ u'\u5608\u3706\u1804\u0903\u03A9\u2603' - return ''.join([random.choice(utf8_chars) - for x in range(length)]).encode('utf-8') + ustr = u''.join([random.choice(utf8_chars) + for x in range(length)]) + if six.PY2: + return ustr.encode('utf-8') + return ustr create_name = create_ascii_name @@ -101,6 +104,8 @@ class Base(unittest2.TestCase): tf.skip_if_no_xattrs() def assert_body(self, body): + if not isinstance(body, bytes): + body = body.encode('utf-8') response_body = self.env.conn.response.read() self.assertEqual(response_body, body, 'Body returned: %s' % (response_body)) @@ -165,7 +170,12 @@ class TestAccount(Base): self.assert_status([401, 412]) def testInvalidUTF8Path(self): - invalid_utf8 = Utils.create_utf8_name()[::-1] + valid_utf8 = Utils.create_utf8_name() + if six.PY2: + invalid_utf8 = valid_utf8[::-1] + else: + invalid_utf8 = (valid_utf8.encode('utf8')[::-1]).decode( + 'utf-8', 'surrogateescape') container = self.env.account.container(invalid_utf8) self.assertFalse(container.create(cfg={'no_path_quote': True})) self.assert_status(412) @@ -338,7 +348,8 @@ class TestAccount(Base): def testLastContainerMarker(self): for format_type in [None, 'json', 'xml']: - containers = self.env.account.containers({'format': format_type}) + containers = self.env.account.containers(parms={ + 'format': format_type}) self.assertEqual(len(containers), len(self.env.containers)) self.assert_status(200) @@ -373,7 +384,7 @@ class TestAccount(Base): parms={'format': format_type}) if isinstance(containers[0], dict): containers = [x['name'] for x in containers] - self.assertEqual(sorted(containers, cmp=locale.strcoll), + self.assertEqual(sorted(containers, key=locale.strxfrm), containers) def testQuotedWWWAuthenticateHeader(self): @@ -685,7 +696,11 @@ class TestContainer(Base): def testUtf8Container(self): valid_utf8 = Utils.create_utf8_name() - invalid_utf8 = valid_utf8[::-1] + if six.PY2: + invalid_utf8 = valid_utf8[::-1] + else: + invalid_utf8 = (valid_utf8.encode('utf8')[::-1]).decode( + 'utf-8', 'surrogateescape') container = self.env.account.container(valid_utf8) self.assertTrue(container.create(cfg={'no_path_quote': True})) self.assertIn(container.name, self.env.account.containers()) @@ -707,15 +722,13 @@ class TestContainer(Base): self.assert_status(202) def testSlashInName(self): - if Utils.create_name == Utils.create_utf8_name: - cont_name = list(six.text_type(Utils.create_name(), 'utf-8')) + if six.PY2: + cont_name = list(Utils.create_name().decode('utf-8')) else: cont_name = list(Utils.create_name()) - cont_name[random.randint(2, len(cont_name) - 2)] = '/' cont_name = ''.join(cont_name) - - if Utils.create_name == Utils.create_utf8_name: + if six.PY2: cont_name = cont_name.encode('utf-8') cont = self.env.account.container(cont_name) @@ -754,7 +767,7 @@ class TestContainer(Base): def testLastFileMarker(self): for format_type in [None, 'json', 'xml']: - files = self.env.container.files({'format': format_type}) + files = self.env.container.files(parms={'format': format_type}) self.assertEqual(len(files), len(self.env.files)) self.assert_status(200) @@ -830,7 +843,7 @@ class TestContainer(Base): files = self.env.container.files(parms={'format': format_type}) if isinstance(files[0], dict): files = [x['name'] for x in files] - self.assertEqual(sorted(files, cmp=locale.strcoll), files) + self.assertEqual(sorted(files, key=locale.strxfrm), files) def testContainerInfo(self): info = self.env.container.info() @@ -854,11 +867,12 @@ class TestContainer(Base): cont = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, cont.files) self.assertTrue(cont.create()) - cont.files() + self.assertEqual(cont.files(), []) cont = self.env.account.container(Utils.create_name()) self.assertRaises(ResponseError, cont.files) self.assertTrue(cont.create()) + # NB: no GET! Make sure the PUT cleared the cached 404 file_item = cont.file(Utils.create_name()) file_item.write_random() @@ -889,7 +903,7 @@ class TestContainer(Base): # PUT object doesn't change container last modified timestamp obj = container.file(Utils.create_name()) self.assertTrue( - obj.write("aaaaa", hdrs={'Content-Type': 'text/plain'})) + obj.write(b"aaaaa", hdrs={'Content-Type': 'text/plain'})) info = container.info() t3 = info['last_modified'] self.assertEqual(t2, t3) @@ -1149,7 +1163,7 @@ class TestContainerPaths(Base): def testStructure(self): def assert_listing(path, file_list): files = self.env.container.files(parms={'path': path}) - self.assertEqual(sorted(file_list, cmp=locale.strcoll), files) + self.assertEqual(sorted(file_list, key=locale.strxfrm), files) if not normalized_urls: assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) assert_listing('/dir1', @@ -1231,7 +1245,7 @@ class TestFile(Base): env = TestFileEnv def testGetResponseHeaders(self): - obj_data = 'test_body' + obj_data = b'test_body' def do_test(put_hdrs, get_hdrs, expected_hdrs, unexpected_hdrs): filename = Utils.create_name() @@ -1860,7 +1874,7 @@ class TestFile(Base): def testNameLimit(self): limit = load_constraint('max_object_name_length') - for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2): + for l in (1, 10, limit // 2, limit - 1, limit, limit + 1, limit * 2): file_item = self.env.container.file('a' * l) if l <= limit: @@ -1913,7 +1927,7 @@ class TestFile(Base): for i in (number_limit - 10, number_limit - 1, number_limit, number_limit + 1, number_limit + 10, number_limit + 100): - j = size_limit / (i * 2) + j = size_limit // (i * 2) metadata = {} while len(metadata.keys()) < i: @@ -1953,7 +1967,7 @@ class TestFile(Base): for i in file_types.keys(): file_item = container.file(Utils.create_name() + '.' + i) - file_item.write('', cfg={'no_content_type': True}) + file_item.write(b'', cfg={'no_content_type': True}) file_types_read = {} for i in container.files(parms={'format': 'json'}): @@ -1968,7 +1982,7 @@ class TestFile(Base): # that's a common EC segment size. The 1.33 multiple is to ensure we # aren't aligned on segment boundaries file_length = int(1048576 * 1.33) - range_size = file_length / 10 + range_size = file_length // 10 file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random(file_length) @@ -2027,8 +2041,8 @@ class TestFile(Base): def testMultiRangeGets(self): file_length = 10000 - range_size = file_length / 10 - subrange_size = range_size / 10 + range_size = file_length // 10 + subrange_size = range_size // 10 file_item = self.env.container.file(Utils.create_name()) data = file_item.write_random( file_length, hdrs={"Content-Type": @@ -2222,7 +2236,7 @@ class TestFile(Base): def testNoContentLengthForPut(self): file_item = self.env.container.file(Utils.create_name()) - self.assertRaises(ResponseError, file_item.write, 'testing', + self.assertRaises(ResponseError, file_item.write, b'testing', cfg={'no_content_length': True}) self.assert_status(411) @@ -2507,14 +2521,14 @@ class TestFile(Base): def testZeroByteFile(self): file_item = self.env.container.file(Utils.create_name()) - self.assertTrue(file_item.write('')) + self.assertTrue(file_item.write(b'')) self.assertIn(file_item.name, self.env.container.files()) - self.assertEqual(file_item.read(), '') + self.assertEqual(file_item.read(), b'') def testEtagResponse(self): file_item = self.env.container.file(Utils.create_name()) - data = six.StringIO(file_item.write_random(512)) + data = six.BytesIO(file_item.write_random(512)) etag = File.compute_md5sum(data) headers = dict(self.env.conn.response.getheaders()) @@ -2525,8 +2539,8 @@ class TestFile(Base): def testChunkedPut(self): if (tf.web_front_end == 'apache2'): - raise SkipTest("Chunked PUT can only be tested with apache2 web" - " front end") + raise SkipTest("Chunked PUT cannot be tested with apache2 web " + "front end") def chunks(s, length=3): i, j = 0, length