openstack/swift
Code Issues Proposed changes

Merge "Clean up func tests ahead of py3"

Browse Source
This commit is contained in:
Zuul 2019-03-05 02:27:17 +00:00 committed by Gerrit Code Review
commit 703249d6cb
3 changed files with 97 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
test/functional/__init__.py
View File

@ -150,7 +150,7 @@ def _in_process_setup_swift_conf(swift_conf_src, testdir):
conf.set(section, 'swift_hash_path_prefix', 'inprocfunctests')
section = 'swift-constraints'
max_file_size = (8 * 1024 * 1024) + 2 # 8 MB + 2
conf.set(section, 'max_file_size', max_file_size)
conf.set(section, 'max_file_size', str(max_file_size))
except NoSectionError:
msg = 'Conf file %s is missing section %s' % (swift_conf_src, section)
raise InProcessException(msg)
@ -232,8 +232,8 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
sp_zero_section = sp_prefix + '0'
conf.add_section(sp_zero_section)
for (k, v) in policy_to_test.get_info(config=True).items():
conf.set(sp_zero_section, k, v)
conf.set(sp_zero_section, 'default', True)
conf.set(sp_zero_section, k, str(v))
conf.set(sp_zero_section, 'default', 'True')
with open(swift_conf, 'w') as fp:
conf.write(fp)
@ -714,7 +714,7 @@ def in_process_setup(the_object_server=object_server):
'/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
resp = conn.getresponse()
assert resp.status == 201, 'Unable to create account: %s\n%s' % (
resp.status, resp.body)
resp.status, resp.read())
create_account('AUTH_test')
create_account('AUTH_test2')

89
test/functional/swift_test_client.py
View File

@ -49,11 +49,11 @@ class RequestError(Exception):
class ResponseError(Exception):
def __init__(self, response, method=None, path=None, details=None):
self.status = response.status
self.reason = response.reason
self.status = getattr(response, 'status', 0)
self.reason = getattr(response, 'reason', '[unknown]')
self.method = method
self.path = path
self.headers = response.getheaders()
self.headers = getattr(response, 'getheaders', lambda: [])()
self.details = details
for name, value in self.headers:
@ -269,7 +269,7 @@ class Connection(object):
headers.update(hdrs)
return headers
def make_request(self, method, path=None, data='', hdrs=None, parms=None,
def make_request(self, method, path=None, data=b'', hdrs=None, parms=None,
cfg=None):
if path is None:
path = []
@ -294,9 +294,9 @@ class Connection(object):
path = '%s?%s' % (path, '&'.join(query_args))
if not cfg.get('no_content_length'):
if cfg.get('set_content_length'):
headers['Content-Length'] = cfg.get('set_content_length')
headers['Content-Length'] = str(cfg.get('set_content_length'))
else:
headers['Content-Length'] = len(data)
headers['Content-Length'] = str(len(data))
def try_request():
self.http_connect()
@ -377,13 +377,13 @@ class Connection(object):
def put_data(self, data, chunked=False):
if chunked:
self.connection.send('%x\r\n%s\r\n' % (len(data), data))
self.connection.send(b'%x\r\n%s\r\n' % (len(data), data))
else:
self.connection.send(data)
def put_end(self, chunked=False):
if chunked:
self.connection.send('0\r\n\r\n')
self.connection.send(b'0\r\n\r\n')
self.response = self.connection.getresponse()
# Hope it isn't big!
@ -418,8 +418,8 @@ class Base(object):
for return_key, header in required_fields:
if header not in headers:
raise ValueError("%s was not found in response header" %
(header,))
raise ValueError("%s was not found in response headers: %r" %
(header, headers))
if is_int_header(header):
ret[return_key] = int(headers[header])
@ -478,8 +478,9 @@ class Account(Base):
if status == 200:
if format_type == 'json':
conts = json.loads(self.conn.response.read())
for cont in conts:
cont['name'] = cont['name'].encode('utf-8')
if six.PY2:
for cont in conts:
cont['name'] = cont['name'].encode('utf-8')
return conts
elif format_type == 'xml':
conts = []
@ -491,13 +492,18 @@ class Account(Base):
childNodes[0].nodeValue
conts.append(cont)
for cont in conts:
cont['name'] = cont['name'].encode('utf-8')
if six.PY2:
cont['name'] = cont['name'].encode('utf-8')
for key in ('count', 'bytes'):
cont[key] = int(cont[key])
return conts
else:
lines = self.conn.response.read().split('\n')
lines = self.conn.response.read().split(b'\n')
if lines and not lines[-1]:
lines = lines[:-1]
return lines
if six.PY2:
return lines
return [line.decode('utf-8') for line in lines]
elif status == 204:
return []
@ -617,10 +623,11 @@ class Container(Base):
if format_type == 'json':
files = json.loads(self.conn.response.read())
for file_item in files:
for key in ('name', 'subdir', 'content_type'):
if key in file_item:
file_item[key] = file_item[key].encode('utf-8')
if six.PY2:
for file_item in files:
for key in ('name', 'subdir', 'content_type'):
if key in file_item:
file_item[key] = file_item[key].encode('utf-8')
return files
elif format_type == 'xml':
files = []
@ -643,28 +650,32 @@ class Container(Base):
for file_item in files:
if 'subdir' in file_item:
file_item['subdir'] = file_item['subdir'].\
encode('utf-8')
if six.PY2:
file_item['subdir'] = \
file_item['subdir'].encode('utf-8')
else:
file_item['name'] = file_item['name'].encode('utf-8')
file_item['content_type'] = file_item['content_type'].\
encode('utf-8')
if six.PY2:
file_item.update({
k: file_item[k].encode('utf-8')
for k in ('name', 'content_type')})
file_item['bytes'] = int(file_item['bytes'])
return files
else:
content = self.conn.response.read()
if content:
lines = content.split('\n')
lines = content.split(b'\n')
if lines and not lines[-1]:
lines = lines[:-1]
return lines
if six.PY2:
return lines
return [line.decode('utf-8') for line in lines]
else:
return []
elif status == 204:
return []
raise ResponseError(self.conn.response, 'GET',
self.conn.make_path(self.path))
self.conn.make_path(self.path, cfg=cfg))
def info(self, hdrs=None, parms=None, cfg=None):
if hdrs is None:
@ -719,11 +730,11 @@ class File(Base):
headers = {}
if not cfg.get('no_content_length'):
if cfg.get('set_content_length'):
headers['Content-Length'] = cfg.get('set_content_length')
headers['Content-Length'] = str(cfg.get('set_content_length'))
elif self.size:
headers['Content-Length'] = self.size
headers['Content-Length'] = str(self.size)
else:
headers['Content-Length'] = 0
headers['Content-Length'] = '0'
if cfg.get('use_token'):
headers['X-Auth-Token'] = cfg.get('use_token')
@ -744,8 +755,8 @@ class File(Base):
def compute_md5sum(cls, data):
block_size = 4096
if isinstance(data, str):
data = six.StringIO(data)
if isinstance(data, bytes):
data = six.BytesIO(data)
checksum = hashlib.md5()
buff = data.read(block_size)
@ -894,7 +905,7 @@ class File(Base):
def random_data(cls, size=None):
if size is None:
size = random.randint(1, 32768)
fd = open('/dev/urandom', 'r')
fd = open('/dev/urandom', 'rb')
data = fd.read(size)
fd.close()
return data
@ -973,10 +984,10 @@ class File(Base):
headers = self.make_headers(cfg=cfg)
if not cfg.get('no_content_length'):
if cfg.get('set_content_length'):
headers['Content-Length'] = \
cfg.get('set_content_length')
headers['Content-Length'] = str(
cfg.get('set_content_length'))
else:
headers['Content-Length'] = 0
headers['Content-Length'] = '0'
self.conn.make_request('POST', self.path, hdrs=headers,
parms=parms, cfg=cfg)
@ -1024,7 +1035,7 @@ class File(Base):
block_size = 2 ** 20
if isinstance(data, file):
if all(hasattr(data, attr) for attr in ('flush', 'seek', 'fileno')):
try:
data.flush()
data.seek(0)
@ -1086,7 +1097,7 @@ class File(Base):
if not self.write(data, hdrs=hdrs, parms=parms, cfg=cfg):
raise ResponseError(self.conn.response, 'PUT',
self.conn.make_path(self.path))
self.md5 = self.compute_md5sum(six.StringIO(data))
self.md5 = self.compute_md5sum(six.BytesIO(data))
return data
def write_random_return_resp(self, size=None, hdrs=None, parms=None,
@ -1103,7 +1114,7 @@ class File(Base):
return_resp=True)
if not resp:
raise ResponseError(self.conn.response)
self.md5 = self.compute_md5sum(six.StringIO(data))
self.md5 = self.compute_md5sum(six.BytesIO(data))
return resp
def post(self, hdrs=None, parms=None, cfg=None, return_resp=False):

72
test/functional/tests.py
View File

@ -63,8 +63,11 @@ class Utils(object):
u'\u1802\u0901\uF111\uD20F\uB30D\u940B\u850A\u5607'\
u'\u3705\u1803\u0902\uF112\uD210\uB30E\u940C\u850B'\
u'\u5608\u3706\u1804\u0903\u03A9\u2603'
return ''.join([random.choice(utf8_chars)
for x in range(length)]).encode('utf-8')
ustr = u''.join([random.choice(utf8_chars)
for x in range(length)])
if six.PY2:
return ustr.encode('utf-8')
return ustr
create_name = create_ascii_name
@ -101,6 +104,8 @@ class Base(unittest2.TestCase):
tf.skip_if_no_xattrs()
def assert_body(self, body):
if not isinstance(body, bytes):
body = body.encode('utf-8')
response_body = self.env.conn.response.read()
self.assertEqual(response_body, body,
'Body returned: %s' % (response_body))
@ -165,7 +170,12 @@ class TestAccount(Base):
self.assert_status([401, 412])
def testInvalidUTF8Path(self):
invalid_utf8 = Utils.create_utf8_name()[::-1]
valid_utf8 = Utils.create_utf8_name()
if six.PY2:
invalid_utf8 = valid_utf8[::-1]
else:
invalid_utf8 = (valid_utf8.encode('utf8')[::-1]).decode(
'utf-8', 'surrogateescape')
container = self.env.account.container(invalid_utf8)
self.assertFalse(container.create(cfg={'no_path_quote': True}))
self.assert_status(412)
@ -338,7 +348,8 @@ class TestAccount(Base):
def testLastContainerMarker(self):
for format_type in [None, 'json', 'xml']:
containers = self.env.account.containers({'format': format_type})
containers = self.env.account.containers(parms={
'format': format_type})
self.assertEqual(len(containers), len(self.env.containers))
self.assert_status(200)
@ -373,7 +384,7 @@ class TestAccount(Base):
parms={'format': format_type})
if isinstance(containers[0], dict):
containers = [x['name'] for x in containers]
self.assertEqual(sorted(containers, cmp=locale.strcoll),
self.assertEqual(sorted(containers, key=locale.strxfrm),
containers)
def testQuotedWWWAuthenticateHeader(self):
@ -685,7 +696,11 @@ class TestContainer(Base):
def testUtf8Container(self):
valid_utf8 = Utils.create_utf8_name()
invalid_utf8 = valid_utf8[::-1]
if six.PY2:
invalid_utf8 = valid_utf8[::-1]
else:
invalid_utf8 = (valid_utf8.encode('utf8')[::-1]).decode(
'utf-8', 'surrogateescape')
container = self.env.account.container(valid_utf8)
self.assertTrue(container.create(cfg={'no_path_quote': True}))
self.assertIn(container.name, self.env.account.containers())
@ -707,15 +722,13 @@ class TestContainer(Base):
self.assert_status(202)
def testSlashInName(self):
if Utils.create_name == Utils.create_utf8_name:
cont_name = list(six.text_type(Utils.create_name(), 'utf-8'))
if six.PY2:
cont_name = list(Utils.create_name().decode('utf-8'))
else:
cont_name = list(Utils.create_name())
cont_name[random.randint(2, len(cont_name) - 2)] = '/'
cont_name = ''.join(cont_name)
if Utils.create_name == Utils.create_utf8_name:
if six.PY2:
cont_name = cont_name.encode('utf-8')
cont = self.env.account.container(cont_name)
@ -754,7 +767,7 @@ class TestContainer(Base):
def testLastFileMarker(self):
for format_type in [None, 'json', 'xml']:
files = self.env.container.files({'format': format_type})
files = self.env.container.files(parms={'format': format_type})
self.assertEqual(len(files), len(self.env.files))
self.assert_status(200)
@ -830,7 +843,7 @@ class TestContainer(Base):
files = self.env.container.files(parms={'format': format_type})
if isinstance(files[0], dict):
files = [x['name'] for x in files]
self.assertEqual(sorted(files, cmp=locale.strcoll), files)
self.assertEqual(sorted(files, key=locale.strxfrm), files)
def testContainerInfo(self):
info = self.env.container.info()
@ -854,11 +867,12 @@ class TestContainer(Base):
cont = self.env.account.container(Utils.create_name())
self.assertRaises(ResponseError, cont.files)
self.assertTrue(cont.create())
cont.files()
self.assertEqual(cont.files(), [])
cont = self.env.account.container(Utils.create_name())
self.assertRaises(ResponseError, cont.files)
self.assertTrue(cont.create())
# NB: no GET! Make sure the PUT cleared the cached 404
file_item = cont.file(Utils.create_name())
file_item.write_random()
@ -889,7 +903,7 @@ class TestContainer(Base):
# PUT object doesn't change container last modified timestamp
obj = container.file(Utils.create_name())
self.assertTrue(
obj.write("aaaaa", hdrs={'Content-Type': 'text/plain'}))
obj.write(b"aaaaa", hdrs={'Content-Type': 'text/plain'}))
info = container.info()
t3 = info['last_modified']
self.assertEqual(t2, t3)
@ -1149,7 +1163,7 @@ class TestContainerPaths(Base):
def testStructure(self):
def assert_listing(path, file_list):
files = self.env.container.files(parms={'path': path})
self.assertEqual(sorted(file_list, cmp=locale.strcoll), files)
self.assertEqual(sorted(file_list, key=locale.strxfrm), files)
if not normalized_urls:
assert_listing('/', ['/dir1/', '/dir2/', '/file1', '/file A'])
assert_listing('/dir1',
@ -1231,7 +1245,7 @@ class TestFile(Base):
env = TestFileEnv
def testGetResponseHeaders(self):
obj_data = 'test_body'
obj_data = b'test_body'
def do_test(put_hdrs, get_hdrs, expected_hdrs, unexpected_hdrs):
filename = Utils.create_name()
@ -1860,7 +1874,7 @@ class TestFile(Base):
def testNameLimit(self):
limit = load_constraint('max_object_name_length')
for l in (1, 10, limit / 2, limit - 1, limit, limit + 1, limit * 2):
for l in (1, 10, limit // 2, limit - 1, limit, limit + 1, limit * 2):
file_item = self.env.container.file('a' * l)
if l <= limit:
@ -1913,7 +1927,7 @@ class TestFile(Base):
for i in (number_limit - 10, number_limit - 1, number_limit,
number_limit + 1, number_limit + 10, number_limit + 100):
j = size_limit / (i * 2)
j = size_limit // (i * 2)
metadata = {}
while len(metadata.keys()) < i:
@ -1953,7 +1967,7 @@ class TestFile(Base):
for i in file_types.keys():
file_item = container.file(Utils.create_name() + '.' + i)
file_item.write('', cfg={'no_content_type': True})
file_item.write(b'', cfg={'no_content_type': True})
file_types_read = {}
for i in container.files(parms={'format': 'json'}):
@ -1968,7 +1982,7 @@ class TestFile(Base):
# that's a common EC segment size. The 1.33 multiple is to ensure we
# aren't aligned on segment boundaries
file_length = int(1048576 * 1.33)
range_size = file_length / 10
range_size = file_length // 10
file_item = self.env.container.file(Utils.create_name())
data = file_item.write_random(file_length)
@ -2027,8 +2041,8 @@ class TestFile(Base):
def testMultiRangeGets(self):
file_length = 10000
range_size = file_length / 10
subrange_size = range_size / 10
range_size = file_length // 10
subrange_size = range_size // 10
file_item = self.env.container.file(Utils.create_name())
data = file_item.write_random(
file_length, hdrs={"Content-Type":
@ -2222,7 +2236,7 @@ class TestFile(Base):
def testNoContentLengthForPut(self):
file_item = self.env.container.file(Utils.create_name())
self.assertRaises(ResponseError, file_item.write, 'testing',
self.assertRaises(ResponseError, file_item.write, b'testing',
cfg={'no_content_length': True})
self.assert_status(411)
@ -2507,14 +2521,14 @@ class TestFile(Base):
def testZeroByteFile(self):
file_item = self.env.container.file(Utils.create_name())
self.assertTrue(file_item.write(''))
self.assertTrue(file_item.write(b''))
self.assertIn(file_item.name, self.env.container.files())
self.assertEqual(file_item.read(), '')
self.assertEqual(file_item.read(), b'')
def testEtagResponse(self):
file_item = self.env.container.file(Utils.create_name())
data = six.StringIO(file_item.write_random(512))
data = six.BytesIO(file_item.write_random(512))
etag = File.compute_md5sum(data)
headers = dict(self.env.conn.response.getheaders())
@ -2525,8 +2539,8 @@ class TestFile(Base):
def testChunkedPut(self):
if (tf.web_front_end == 'apache2'):
raise SkipTest("Chunked PUT can only be tested with apache2 web"
" front end")
raise SkipTest("Chunked PUT cannot be tested with apache2 web "
"front end")
def chunks(s, length=3):
i, j = 0, length