Merge "Clean up func tests ahead of py3"

This commit is contained in:
Zuul 2019-03-05 02:27:17 +00:00 committed by Gerrit Code Review
commit 703249d6cb
3 changed files with 97 additions and 72 deletions

View File

@ -150,7 +150,7 @@ def _in_process_setup_swift_conf(swift_conf_src, testdir):
conf.set(section, 'swift_hash_path_prefix', 'inprocfunctests') conf.set(section, 'swift_hash_path_prefix', 'inprocfunctests')
section = 'swift-constraints' section = 'swift-constraints'
max_file_size = (8 * 1024 * 1024) + 2 # 8 MB + 2 max_file_size = (8 * 1024 * 1024) + 2 # 8 MB + 2
conf.set(section, 'max_file_size', max_file_size) conf.set(section, 'max_file_size', str(max_file_size))
except NoSectionError: except NoSectionError:
msg = 'Conf file %s is missing section %s' % (swift_conf_src, section) msg = 'Conf file %s is missing section %s' % (swift_conf_src, section)
raise InProcessException(msg) raise InProcessException(msg)
@ -232,8 +232,8 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
sp_zero_section = sp_prefix + '0' sp_zero_section = sp_prefix + '0'
conf.add_section(sp_zero_section) conf.add_section(sp_zero_section)
for (k, v) in policy_to_test.get_info(config=True).items(): for (k, v) in policy_to_test.get_info(config=True).items():
conf.set(sp_zero_section, k, v) conf.set(sp_zero_section, k, str(v))
conf.set(sp_zero_section, 'default', True) conf.set(sp_zero_section, 'default', 'True')
with open(swift_conf, 'w') as fp: with open(swift_conf, 'w') as fp:
conf.write(fp) conf.write(fp)
@ -714,7 +714,7 @@ def in_process_setup(the_object_server=object_server):
'/' + act, {'X-Timestamp': ts, 'x-trans-id': act}) '/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
resp = conn.getresponse() resp = conn.getresponse()
assert resp.status == 201, 'Unable to create account: %s\n%s' % ( assert resp.status == 201, 'Unable to create account: %s\n%s' % (
resp.status, resp.body) resp.status, resp.read())
create_account('AUTH_test') create_account('AUTH_test')
create_account('AUTH_test2') create_account('AUTH_test2')

View File

@ -49,11 +49,11 @@ class RequestError(Exception):
class ResponseError(Exception): class ResponseError(Exception):
def __init__(self, response, method=None, path=None, details=None): def __init__(self, response, method=None, path=None, details=None):
self.status = response.status self.status = getattr(response, 'status', 0)
self.reason = response.reason self.reason = getattr(response, 'reason', '[unknown]')
self.method = method self.method = method
self.path = path self.path = path
self.headers = response.getheaders() self.headers = getattr(response, 'getheaders', lambda: [])()
self.details = details self.details = details
for name, value in self.headers: for name, value in self.headers:
@ -269,7 +269,7 @@ class Connection(object):
headers.update(hdrs) headers.update(hdrs)
return headers return headers
def make_request(self, method, path=None, data='', hdrs=None, parms=None, def make_request(self, method, path=None, data=b'', hdrs=None, parms=None,
cfg=None): cfg=None):
if path is None: if path is None:
path = [] path = []
@ -294,9 +294,9 @@ class Connection(object):
path = '%s?%s' % (path, '&'.join(query_args)) path = '%s?%s' % (path, '&'.join(query_args))
if not cfg.get('no_content_length'): if not cfg.get('no_content_length'):
if cfg.get('set_content_length'): if cfg.get('set_content_length'):
headers['Content-Length'] = cfg.get('set_content_length') headers['Content-Length'] = str(cfg.get('set_content_length'))
else: else:
headers['Content-Length'] = len(data) headers['Content-Length'] = str(len(data))
def try_request(): def try_request():
self.http_connect() self.http_connect()
@ -377,13 +377,13 @@ class Connection(object):
def put_data(self, data, chunked=False): def put_data(self, data, chunked=False):
if chunked: if chunked:
self.connection.send('%x\r\n%s\r\n' % (len(data), data)) self.connection.send(b'%x\r\n%s\r\n' % (len(data), data))
else: else:
self.connection.send(data) self.connection.send(data)
def put_end(self, chunked=False): def put_end(self, chunked=False):
if chunked: if chunked:
self.connection.send('0\r\n\r\n') self.connection.send(b'0\r\n\r\n')
self.response = self.connection.getresponse() self.response = self.connection.getresponse()
# Hope it isn't big! # Hope it isn't big!
@ -418,8 +418,8 @@ class Base(object):
for return_key, header in required_fields: for return_key, header in required_fields:
if header not in headers: if header not in headers:
raise ValueError("%s was not found in response header" % raise ValueError("%s was not found in response headers: %r" %
(header,)) (header, headers))
if is_int_header(header): if is_int_header(header):
ret[return_key] = int(headers[header]) ret[return_key] = int(headers[header])
@ -478,8 +478,9 @@ class Account(Base):
if status == 200: if status == 200:
if format_type == 'json': if format_type == 'json':
conts = json.loads(self.conn.response.read()) conts = json.loads(self.conn.response.read())
for cont in conts: if six.PY2:
cont['name'] = cont['name'].encode('utf-8') for cont in conts:
cont['name'] = cont['name'].encode('utf-8')
return conts return conts
elif format_type == 'xml': elif format_type == 'xml':
conts = [] conts = []
@ -491,13 +492,18 @@ class Account(Base):
childNodes[0].nodeValue childNodes[0].nodeValue
conts.append(cont) conts.append(cont)
for cont in conts: for cont in conts:
cont['name'] = cont['name'].encode('utf-8') if six.PY2:
cont['name'] = cont['name'].encode('utf-8')
for key in ('count', 'bytes'):
cont[key] = int(cont[key])
return conts return conts
else: else:
lines = self.conn.response.read().split('\n') lines = self.conn.response.read().split(b'\n')
if lines and not lines[-1]: if lines and not lines[-1]:
lines = lines[:-1] lines = lines[:-1]
return lines if six.PY2:
return lines
return [line.decode('utf-8') for line in lines]
elif status == 204: elif status == 204:
return [] return []
@ -617,10 +623,11 @@ class Container(Base):
if format_type == 'json': if format_type == 'json':
files = json.loads(self.conn.response.read()) files = json.loads(self.conn.response.read())
for file_item in files: if six.PY2:
for key in ('name', 'subdir', 'content_type'): for file_item in files:
if key in file_item: for key in ('name', 'subdir', 'content_type'):
file_item[key] = file_item[key].encode('utf-8') if key in file_item:
file_item[key] = file_item[key].encode('utf-8')
return files return files
elif format_type == 'xml': elif format_type == 'xml':
files = [] files = []
@ -643,28 +650,32 @@ class Container(Base):
for file_item in files: for file_item in files:
if 'subdir' in file_item: if 'subdir' in file_item:
file_item['subdir'] = file_item['subdir'].\ if six.PY2:
encode('utf-8') file_item['subdir'] = \
file_item['subdir'].encode('utf-8')
else: else:
file_item['name'] = file_item['name'].encode('utf-8') if six.PY2:
file_item['content_type'] = file_item['content_type'].\ file_item.update({
encode('utf-8') k: file_item[k].encode('utf-8')
for k in ('name', 'content_type')})
file_item['bytes'] = int(file_item['bytes']) file_item['bytes'] = int(file_item['bytes'])
return files return files
else: else:
content = self.conn.response.read() content = self.conn.response.read()
if content: if content:
lines = content.split('\n') lines = content.split(b'\n')
if lines and not lines[-1]: if lines and not lines[-1]:
lines = lines[:-1] lines = lines[:-1]
return lines if six.PY2:
return lines
return [line.decode('utf-8') for line in lines]
else: else:
return [] return []
elif status == 204: elif status == 204:
return [] return []
raise ResponseError(self.conn.response, 'GET', raise ResponseError(self.conn.response, 'GET',
self.conn.make_path(self.path)) self.conn.make_path(self.path, cfg=cfg))
def info(self, hdrs=None, parms=None, cfg=None): def info(self, hdrs=None, parms=None, cfg=None):
if hdrs is None: if hdrs is None:
@ -719,11 +730,11 @@ class File(Base):
headers = {} headers = {}
if not cfg.get('no_content_length'): if not cfg.get('no_content_length'):
if cfg.get('set_content_length'): if cfg.get('set_content_length'):
headers['Content-Length'] = cfg.get('set_content_length') headers['Content-Length'] = str(cfg.get('set_content_length'))
elif self.size: elif self.size:
headers['Content-Length'] = self.size headers['Content-Length'] = str(self.size)
else: else:
headers['Content-Length'] = 0 headers['Content-Length'] = '0'
if cfg.get('use_token'): if cfg.get('use_token'):
headers['X-Auth-Token'] = cfg.get('use_token') headers['X-Auth-Token'] = cfg.get('use_token')
@ -744,8 +755,8 @@ class File(Base):
def compute_md5sum(cls, data): def compute_md5sum(cls, data):
block_size = 4096 block_size = 4096
if isinstance(data, str): if isinstance(data, bytes):
data = six.StringIO(data) data = six.BytesIO(data)
checksum = hashlib.md5() checksum = hashlib.md5()
buff = data.read(block_size) buff = data.read(block_size)
@ -894,7 +905,7 @@ class File(Base):
def random_data(cls, size=None): def random_data(cls, size=None):
if size is None: if size is None:
size = random.randint(1, 32768) size = random.randint(1, 32768)
fd = open('/dev/urandom', 'r') fd = open('/dev/urandom', 'rb')
data = fd.read(size) data = fd.read(size)
fd.close() fd.close()
return data return data
@ -973,10 +984,10 @@ class File(Base):
headers = self.make_headers(cfg=cfg) headers = self.make_headers(cfg=cfg)
if not cfg.get('no_content_length'): if not cfg.get('no_content_length'):
if cfg.get('set_content_length'): if cfg.get('set_content_length'):
headers['Content-Length'] = \ headers['Content-Length'] = str(
cfg.get('set_content_length') cfg.get('set_content_length'))
else: else:
headers['Content-Length'] = 0 headers['Content-Length'] = '0'
self.conn.make_request('POST', self.path, hdrs=headers, self.conn.make_request('POST', self.path, hdrs=headers,
parms=parms, cfg=cfg) parms=parms, cfg=cfg)
@ -1024,7 +1035,7 @@ class File(Base):
block_size = 2 ** 20 block_size = 2 ** 20
if isinstance(data, file): if all(hasattr(data, attr) for attr in ('flush', 'seek', 'fileno')):
try: try:
data.flush() data.flush()
data.seek(0) data.seek(0)
@ -1086,7 +1097,7 @@ class File(Base):
if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg): if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg):
raise ResponseError(self.conn.response, 'PUT', raise ResponseError(self.conn.response, 'PUT',
self.conn.make_path(self.path)) self.conn.make_path(self.path))
self.md5 = self.compute_md5sum(six.StringIO(data)) self.md5 = self.compute_md5sum(six.BytesIO(data))
return data return data
def write_random_return_resp(self, size=None, hdrs=None, parms=None, def write_random_return_resp(self, size=None, hdrs=None, parms=None,
@ -1103,7 +1114,7 @@ class File(Base):
return_resp=True) return_resp=True)
if not resp: if not resp:
raise ResponseError(self.conn.response) raise ResponseError(self.conn.response)
self.md5 = self.compute_md5sum(six.StringIO(data)) self.md5 = self.compute_md5sum(six.BytesIO(data))
return resp return resp
def post(self, hdrs=None, parms=None, cfg=None, return_resp=False): def post(self, hdrs=None, parms=None, cfg=None, return_resp=False):

View File

@ -63,8 +63,11 @@ class Utils(object):
u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\ u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\
u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\ u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\
u'\u5608\u3706\u1804\u0903\u03A9\u2603' u'\u5608\u3706\u1804\u0903\u03A9\u2603'
return ''.join([random.choice(utf8_chars) ustr = u''.join([random.choice(utf8_chars)
for x in range(length)]).encode('utf-8') for x in range(length)])
if six.PY2:
return ustr.encode('utf-8')
return ustr
create_name = create_ascii_name create_name = create_ascii_name
@ -101,6 +104,8 @@ class Base(unittest2.TestCase):
tf.skip_if_no_xattrs() tf.skip_if_no_xattrs()
def assert_body(self, body): def assert_body(self, body):
if not isinstance(body, bytes):
body = body.encode('utf-8')
response_body = self.env.conn.response.read() response_body = self.env.conn.response.read()
self.assertEqual(response_body, body, self.assertEqual(response_body, body,
'Body returned: %s' % (response_body)) 'Body returned: %s' % (response_body))
@ -165,7 +170,12 @@ class TestAccount(Base):
self.assert_status([401, 412]) self.assert_status([401, 412])
def testInvalidUTF8Path(self): def testInvalidUTF8Path(self):
invalid_utf8 = Utils.create_utf8_name()[::-1] valid_utf8 = Utils.create_utf8_name()
if six.PY2:
invalid_utf8 = valid_utf8[::-1]
else:
invalid_utf8 = (valid_utf8.encode('utf8')[::-1]).decode(
'utf-8', 'surrogateescape')
container = self.env.account.container(invalid_utf8) container = self.env.account.container(invalid_utf8)
self.assertFalse(container.create(cfg={'no_path_quote': True})) self.assertFalse(container.create(cfg={'no_path_quote': True}))
self.assert_status(412) self.assert_status(412)
@ -338,7 +348,8 @@ class TestAccount(Base):
def testLastContainerMarker(self): def testLastContainerMarker(self):
for format_type in [None, 'json', 'xml']: for format_type in [None, 'json', 'xml']:
containers = self.env.account.containers({'format': format_type}) containers = self.env.account.containers(parms={
'format': format_type})
self.assertEqual(len(containers), len(self.env.containers)) self.assertEqual(len(containers), len(self.env.containers))
self.assert_status(200) self.assert_status(200)
@ -373,7 +384,7 @@ class TestAccount(Base):
parms={'format': format_type}) parms={'format': format_type})
if isinstance(containers[0], dict): if isinstance(containers[0], dict):
containers = [x['name'] for x in containers] containers = [x['name'] for x in containers]
self.assertEqual(sorted(containers, cmp=locale.strcoll), self.assertEqual(sorted(containers, key=locale.strxfrm),
containers) containers)
def testQuotedWWWAuthenticateHeader(self): def testQuotedWWWAuthenticateHeader(self):
@ -685,7 +696,11 @@ class TestContainer(Base):
def testUtf8Container(self): def testUtf8Container(self):
valid_utf8 = Utils.create_utf8_name() valid_utf8 = Utils.create_utf8_name()
invalid_utf8 = valid_utf8[::-1] if six.PY2:
invalid_utf8 = valid_utf8[::-1]
else:
invalid_utf8 = (valid_utf8.encode('utf8')[::-1]).decode(
'utf-8', 'surrogateescape')
container = self.env.account.container(valid_utf8) container = self.env.account.container(valid_utf8)
self.assertTrue(container.create(cfg={'no_path_quote': True})) self.assertTrue(container.create(cfg={'no_path_quote': True}))
self.assertIn(container.name, self.env.account.containers()) self.assertIn(container.name, self.env.account.containers())
@ -707,15 +722,13 @@ class TestContainer(Base):
self.assert_status(202) self.assert_status(202)
def testSlashInName(self): def testSlashInName(self):
if Utils.create_name == Utils.create_utf8_name: if six.PY2:
cont_name = list(six.text_type(Utils.create_name(), 'utf-8')) cont_name = list(Utils.create_name().decode('utf-8'))
else: else:
cont_name = list(Utils.create_name()) cont_name = list(Utils.create_name())
cont_name[random.randint(2, len(cont_name) - 2)] = '/' cont_name[random.randint(2, len(cont_name) - 2)] = '/'
cont_name = ''.join(cont_name) cont_name = ''.join(cont_name)
if six.PY2:
if Utils.create_name == Utils.create_utf8_name:
cont_name = cont_name.encode('utf-8') cont_name = cont_name.encode('utf-8')
cont = self.env.account.container(cont_name) cont = self.env.account.container(cont_name)
@ -754,7 +767,7 @@ class TestContainer(Base):
def testLastFileMarker(self): def testLastFileMarker(self):
for format_type in [None, 'json', 'xml']: for format_type in [None, 'json', 'xml']:
files = self.env.container.files({'format': format_type}) files = self.env.container.files(parms={'format': format_type})
self.assertEqual(len(files), len(self.env.files)) self.assertEqual(len(files), len(self.env.files))
self.assert_status(200) self.assert_status(200)
@ -830,7 +843,7 @@ class TestContainer(Base):
files = self.env.container.files(parms={'format': format_type}) files = self.env.container.files(parms={'format': format_type})
if isinstance(files[0], dict): if isinstance(files[0], dict):
files = [x['name'] for x in files] files = [x['name'] for x in files]
self.assertEqual(sorted(files, cmp=locale.strcoll), files) self.assertEqual(sorted(files, key=locale.strxfrm), files)
def testContainerInfo(self): def testContainerInfo(self):
info = self.env.container.info() info = self.env.container.info()
@ -854,11 +867,12 @@ class TestContainer(Base):
cont = self.env.account.container(Utils.create_name()) cont = self.env.account.container(Utils.create_name())
self.assertRaises(ResponseError, cont.files) self.assertRaises(ResponseError, cont.files)
self.assertTrue(cont.create()) self.assertTrue(cont.create())
cont.files() self.assertEqual(cont.files(), [])
cont = self.env.account.container(Utils.create_name()) cont = self.env.account.container(Utils.create_name())
self.assertRaises(ResponseError, cont.files) self.assertRaises(ResponseError, cont.files)
self.assertTrue(cont.create()) self.assertTrue(cont.create())
# NB: no GET! Make sure the PUT cleared the cached 404
file_item = cont.file(Utils.create_name()) file_item = cont.file(Utils.create_name())
file_item.write_random() file_item.write_random()
@ -889,7 +903,7 @@ class TestContainer(Base):
# PUT object doesn't change container last modified timestamp # PUT object doesn't change container last modified timestamp
obj = container.file(Utils.create_name()) obj = container.file(Utils.create_name())
self.assertTrue( self.assertTrue(
obj.write("aaaaa", hdrs={'Content-Type': 'text/plain'})) obj.write(b"aaaaa", hdrs={'Content-Type': 'text/plain'}))
info = container.info() info = container.info()
t3 = info['last_modified'] t3 = info['last_modified']
self.assertEqual(t2, t3) self.assertEqual(t2, t3)
@ -1149,7 +1163,7 @@ class TestContainerPaths(Base):
def testStructure(self): def testStructure(self):
def assert_listing(path, file_list): def assert_listing(path, file_list):
files = self.env.container.files(parms={'path': path}) files = self.env.container.files(parms={'path': path})
self.assertEqual(sorted(file_list, cmp=locale.strcoll), files) self.assertEqual(sorted(file_list, key=locale.strxfrm), files)
if not normalized_urls: if not normalized_urls:
assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A']) assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A'])
assert_listing('/dir1', assert_listing('/dir1',
@ -1231,7 +1245,7 @@ class TestFile(Base):
env = TestFileEnv env = TestFileEnv
def testGetResponseHeaders(self): def testGetResponseHeaders(self):
obj_data = 'test_body' obj_data = b'test_body'
def do_test(put_hdrs, get_hdrs, expected_hdrs, unexpected_hdrs): def do_test(put_hdrs, get_hdrs, expected_hdrs, unexpected_hdrs):
filename = Utils.create_name() filename = Utils.create_name()
@ -1860,7 +1874,7 @@ class TestFile(Base):
def testNameLimit(self): def testNameLimit(self):
limit = load_constraint('max_object_name_length') limit = load_constraint('max_object_name_length')
for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2): for l in (1, 10, limit // 2, limit - 1, limit, limit + 1, limit * 2):
file_item = self.env.container.file('a' * l) file_item = self.env.container.file('a' * l)
if l <= limit: if l <= limit:
@ -1913,7 +1927,7 @@ class TestFile(Base):
for i in (number_limit - 10, number_limit - 1, number_limit, for i in (number_limit - 10, number_limit - 1, number_limit,
number_limit + 1, number_limit + 10, number_limit + 100): number_limit + 1, number_limit + 10, number_limit + 100):
j = size_limit / (i * 2) j = size_limit // (i * 2)
metadata = {} metadata = {}
while len(metadata.keys()) < i: while len(metadata.keys()) < i:
@ -1953,7 +1967,7 @@ class TestFile(Base):
for i in file_types.keys(): for i in file_types.keys():
file_item = container.file(Utils.create_name() + '.' + i) file_item = container.file(Utils.create_name() + '.' + i)
file_item.write('', cfg={'no_content_type': True}) file_item.write(b'', cfg={'no_content_type': True})
file_types_read = {} file_types_read = {}
for i in container.files(parms={'format': 'json'}): for i in container.files(parms={'format': 'json'}):
@ -1968,7 +1982,7 @@ class TestFile(Base):
# that's a common EC segment size. The 1.33 multiple is to ensure we # that's a common EC segment size. The 1.33 multiple is to ensure we
# aren't aligned on segment boundaries # aren't aligned on segment boundaries
file_length = int(1048576 * 1.33) file_length = int(1048576 * 1.33)
range_size = file_length / 10 range_size = file_length // 10
file_item = self.env.container.file(Utils.create_name()) file_item = self.env.container.file(Utils.create_name())
data = file_item.write_random(file_length) data = file_item.write_random(file_length)
@ -2027,8 +2041,8 @@ class TestFile(Base):
def testMultiRangeGets(self): def testMultiRangeGets(self):
file_length = 10000 file_length = 10000
range_size = file_length / 10 range_size = file_length // 10
subrange_size = range_size / 10 subrange_size = range_size // 10
file_item = self.env.container.file(Utils.create_name()) file_item = self.env.container.file(Utils.create_name())
data = file_item.write_random( data = file_item.write_random(
file_length, hdrs={"Content-Type": file_length, hdrs={"Content-Type":
@ -2222,7 +2236,7 @@ class TestFile(Base):
def testNoContentLengthForPut(self): def testNoContentLengthForPut(self):
file_item = self.env.container.file(Utils.create_name()) file_item = self.env.container.file(Utils.create_name())
self.assertRaises(ResponseError, file_item.write, 'testing', self.assertRaises(ResponseError, file_item.write, b'testing',
cfg={'no_content_length': True}) cfg={'no_content_length': True})
self.assert_status(411) self.assert_status(411)
@ -2507,14 +2521,14 @@ class TestFile(Base):
def testZeroByteFile(self): def testZeroByteFile(self):
file_item = self.env.container.file(Utils.create_name()) file_item = self.env.container.file(Utils.create_name())
self.assertTrue(file_item.write('')) self.assertTrue(file_item.write(b''))
self.assertIn(file_item.name, self.env.container.files()) self.assertIn(file_item.name, self.env.container.files())
self.assertEqual(file_item.read(), '') self.assertEqual(file_item.read(), b'')
def testEtagResponse(self): def testEtagResponse(self):
file_item = self.env.container.file(Utils.create_name()) file_item = self.env.container.file(Utils.create_name())
data = six.StringIO(file_item.write_random(512)) data = six.BytesIO(file_item.write_random(512))
etag = File.compute_md5sum(data) etag = File.compute_md5sum(data)
headers = dict(self.env.conn.response.getheaders()) headers = dict(self.env.conn.response.getheaders())
@ -2525,8 +2539,8 @@ class TestFile(Base):
def testChunkedPut(self): def testChunkedPut(self):
if (tf.web_front_end == 'apache2'): if (tf.web_front_end == 'apache2'):
raise SkipTest("Chunked PUT can only be tested with apache2 web" raise SkipTest("Chunked PUT cannot be tested with apache2 web "
" front end") "front end")
def chunks(s, length=3): def chunks(s, length=3):
i, j = 0, length i, j = 0, length