openstack
/
swift
Code Issues Proposed changes

py3: port s3api 

Drive-by: When passing a list or tuple to swob.Response as an app_iter,
check that it's full of byte strings.

Change-Id: Ifc35aacb2e45004f74c871f08ff3c52bc57c1463
This commit is contained in:
Tim Burke 2019-04-09 16:47:20 -07:00
parent f55167a735
commit 3a9f3f8419
23 changed files with 308 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
swift/common/middleware/s3api/controllers/bucket.py
View File

@ -18,6 +18,7 @@ from base64 import standard_b64decode as b64decode
from six.moves.urllib.parse import quote
from swift.common import swob
from swift.common.http import HTTP_OK
from swift.common.utils import json, public, config_true_value
@ -66,8 +67,9 @@ class BucketController(Controller):
segments = json.loads(resp.body)
for seg in segments:
try:
req.get_response(self.app, 'DELETE', container,
seg['name'].encode('utf8'))
req.get_response(
self.app, 'DELETE', container,
swob.bytes_to_wsgi(seg['name'].encode('utf8')))
except NoSuchKey:
pass
except InternalError:

29
swift/common/middleware/s3api/controllers/multi_upload.py
View File

@ -59,11 +59,14 @@ Static Large Object when the multipart upload is completed.
"""
import binascii
from hashlib import md5
import os
import re
import time
import six
from swift.common.swob import Range
from swift.common.utils import json, public, reiterate
from swift.common.db import utf8encode
@ -222,8 +225,9 @@ class UploadsController(Controller):
:return (non_delimited_uploads, common_prefixes)
"""
(prefix, delimiter) = \
utf8encode(prefix, delimiter)
if six.PY2:
(prefix, delimiter) = \
utf8encode(prefix, delimiter)
non_delimited_uploads = []
common_prefixes = set()
for upload in uploads:
@ -440,7 +444,7 @@ class UploadController(Controller):
# If the caller requested a list starting at a specific part number,
# construct a sub-set of the object list.
objList = filter(filter_part_num_marker, objects)
objList = [obj for obj in objects if filter_part_num_marker(obj)]
# pylint: disable-msg=E1103
objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
@ -603,7 +607,7 @@ class UploadController(Controller):
'path': '/%s/%s/%s/%d' % (
container, req.object_name, upload_id, part_number),
'etag': etag})
s3_etag_hasher.update(etag.decode('hex'))
s3_etag_hasher.update(binascii.a2b_hex(etag))
except (XMLSyntaxError, DocumentInvalid):
# NB: our schema definitions catch uploads with no parts here
raise MalformedXML()
@ -661,8 +665,8 @@ class UploadController(Controller):
# ceph-s3tests happy
continue
if not yielded_anything:
yield ('<?xml version="1.0" '
'encoding="UTF-8"?>\n')
yield (b'<?xml version="1.0" '
b'encoding="UTF-8"?>\n')
yielded_anything = True
yield chunk
continue
@ -708,8 +712,13 @@ class UploadController(Controller):
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
# Why are we doing our own port parsing? Because py3 decided
# to start raising ValueErrors on access after parsing such
# an invalid port
netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
if ':' in netloc:
port = netloc.split(':', 2)[1]
host_url += ':%s' % port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
@ -717,13 +726,13 @@ class UploadController(Controller):
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
resp.headers.pop('ETag', None)
if yielded_anything:
yield '\n'
yield b'\n'
yield tostring(result_elem,
xml_declaration=not yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
yield '\n'
yield b'\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status

2
swift/common/middleware/s3api/controllers/obj.py
View File

@ -156,7 +156,7 @@ class ObjectController(Controller):
for chunk in resp.app_iter:
pass # drain the bulk-deleter response
resp.status = HTTP_NO_CONTENT
resp.body = ''
resp.body = b''
except NoSuchKey:
# expect to raise NoSuchBucket when the bucket doesn't exist
req.get_container_info(self.app)

4
swift/common/middleware/s3api/etree.py
View File

@ -120,7 +120,9 @@ class _Element(lxml.etree.ElementBase):
"""
utf-8 wrapper property of lxml.etree.Element.text
"""
return utf8encode(lxml.etree.ElementBase.text.__get__(self))
if six.PY2:
return utf8encode(lxml.etree.ElementBase.text.__get__(self))
return lxml.etree.ElementBase.text.__get__(self)
@text.setter
def text(self, value):

4
swift/common/middleware/s3api/s3api.py
View File

@ -167,7 +167,7 @@ class ListingEtagMiddleware(object):
ctx._response_exc_info)
return [body]
body = json.dumps(listing)
body = json.dumps(listing).encode('ascii')
ctx._response_headers[cl_index] = (
ctx._response_headers[cl_index][0],
str(len(body)),
@ -237,7 +237,7 @@ class S3ApiMiddleware(object):
resp = err_resp
except Exception as e:
self.logger.exception(e)
resp = InternalError(reason=e)
resp = InternalError(reason=str(e))
if isinstance(resp, S3ResponseBase) and 'swift.trans_id' in env:
resp.headers['x-amz-id-2'] = env['swift.trans_id']

84
swift/common/middleware/s3api/s3request.py
View File

@ -14,6 +14,7 @@
# limitations under the License.
import base64
import binascii
from collections import defaultdict, OrderedDict
from email.header import Header
from hashlib import sha1, sha256, md5
@ -127,7 +128,8 @@ class HashingInput(object):
chunk = self._input.read(size)
self._hasher.update(chunk)
self._to_read -= len(chunk)
if self._to_read < 0 or (size > len(chunk) and self._to_read) or (
short_read = bool(chunk) if size is None else (len(chunk) < size)
if self._to_read < 0 or (short_read and self._to_read) or (
self._to_read == 0 and
self._hasher.hexdigest() != self._expected):
self.close()
@ -149,10 +151,10 @@ class SigV4Mixin(object):
def check_signature(self, secret):
secret = utf8encode(secret)
user_signature = self.signature
derived_secret = 'AWS4' + secret
derived_secret = b'AWS4' + secret
for scope_piece in self.scope.values():
derived_secret = hmac.new(
derived_secret, scope_piece, sha256).digest()
derived_secret, scope_piece.encode('utf8'), sha256).digest()
valid_signature = hmac.new(
derived_secret, self.string_to_sign, sha256).hexdigest()
return user_signature == valid_signature
@ -331,10 +333,10 @@ class SigV4Mixin(object):
def _canonical_query_string(self):
return '&'.join(
'%s=%s' % (quote(key, safe='-_.~'),
quote(value, safe='-_.~'))
'%s=%s' % (swob.wsgi_quote(key, safe='-_.~'),
swob.wsgi_quote(value, safe='-_.~'))
for key, value in sorted(self.params.items())
if key not in ('Signature', 'X-Amz-Signature'))
if key not in ('Signature', 'X-Amz-Signature')).encode('ascii')
def _headers_to_sign(self):
"""
@ -383,7 +385,7 @@ class SigV4Mixin(object):
"""
It won't require bucket name in canonical_uri for v4.
"""
return self.environ.get('RAW_PATH_INFO', self.path)
return swob.wsgi_to_bytes(self.environ.get('RAW_PATH_INFO', self.path))
def _canonical_request(self):
# prepare 'canonical_request'
@ -401,7 +403,7 @@ class SigV4Mixin(object):
#
# 1. Add verb like: GET
cr = [self.method.upper()]
cr = [swob.wsgi_to_bytes(self.method.upper())]
# 2. Add path like: /
path = self._canonical_uri()
@ -415,12 +417,12 @@ class SigV4Mixin(object):
# host:iam.amazonaws.com
# x-amz-date:20150830T123600Z
headers_to_sign = self._headers_to_sign()
cr.append(''.join('%s:%s\n' % (key, value)
for key, value in headers_to_sign))
cr.append(b''.join(swob.wsgi_to_bytes('%s:%s\n' % (key, value))
for key, value in headers_to_sign))
# 5. Add signed headers into canonical request like
# content-type;host;x-amz-date
cr.append(';'.join(k for k, v in headers_to_sign))
cr.append(b';'.join(swob.wsgi_to_bytes(k) for k, v in headers_to_sign))
# 6. Add payload string at the tail
if 'X-Amz-Credential' in self.params:
@ -446,8 +448,8 @@ class SigV4Mixin(object):
# else, not provided -- Swift will kick out a 411 Length Required
# which will get translated back to a S3-style response in
# S3Request._swift_error_codes
cr.append(hashed_payload)
return '\n'.join(cr).encode('utf-8')
cr.append(swob.wsgi_to_bytes(hashed_payload))
return b'\n'.join(cr)
@property
def scope(self):
@ -462,10 +464,11 @@ class SigV4Mixin(object):
"""
Create 'StringToSign' value in Amazon terminology for v4.
"""
return '\n'.join(['AWS4-HMAC-SHA256',
self.timestamp.amz_date_format,
'/'.join(self.scope.values()),
sha256(self._canonical_request()).hexdigest()])
return b'\n'.join([
b'AWS4-HMAC-SHA256',
self.timestamp.amz_date_format.encode('ascii'),
'/'.join(self.scope.values()).encode('utf8'),
sha256(self._canonical_request()).hexdigest().encode('ascii')])
def signature_does_not_match_kwargs(self):
kwargs = super(SigV4Mixin, self).signature_does_not_match_kwargs()
@ -473,7 +476,7 @@ class SigV4Mixin(object):
kwargs.update({
'canonical_request': cr,
'canonical_request_bytes': ' '.join(
format(ord(c), '02x') for c in cr),
format(ord(c), '02x') for c in cr.decode('latin1')),
})
return kwargs
@ -545,6 +548,8 @@ class S3Request(swob.Request):
user_signature = self.signature
valid_signature = base64.b64encode(hmac.new(
secret, self.string_to_sign, sha1).digest()).strip()
if not six.PY2:
valid_signature = valid_signature.decode('ascii')
return user_signature == valid_signature
@property
@ -613,7 +618,7 @@ class S3Request(swob.Request):
return None
def _parse_uri(self):
if not check_utf8(self.environ['PATH_INFO']):
if not check_utf8(swob.wsgi_to_str(self.environ['PATH_INFO'])):
raise InvalidURI(self.path)
if self.bucket_in_host:
@ -739,8 +744,10 @@ class S3Request(swob.Request):
# Non-base64-alphabet characters in value.
raise InvalidDigest(content_md5=value)
try:
self.headers['ETag'] = value.decode('base64').encode('hex')
except Exception:
self.headers['ETag'] = binascii.b2a_hex(
binascii.a2b_base64(value))
except binascii.error:
# incorrect padding, most likely
raise InvalidDigest(content_md5=value)
if len(self.headers['ETag']) != 32:
@ -825,10 +832,11 @@ class S3Request(swob.Request):
'functionality that is not implemented',
header='Transfer-Encoding')
if self.message_length() > max_length:
ml = self.message_length()
if ml and ml > max_length:
raise MalformedXML()
if te or self.message_length():
if te or ml:
# Limit the read similar to how SLO handles manifests
body = self.body_file.read(max_length)
else:
@ -843,7 +851,7 @@ class S3Request(swob.Request):
raise InvalidRequest('Missing required header for this request: '
'Content-MD5')
digest = md5(body).digest().encode('base64').strip()
digest = base64.b64encode(md5(body).digest()).strip().decode('ascii')
if self.environ['HTTP_CONTENT_MD5'] != digest:
raise BadDigest(content_md5=self.environ['HTTP_CONTENT_MD5'])
@ -927,9 +935,10 @@ class S3Request(swob.Request):
"""
amz_headers = {}
buf = [self.method,
_header_strip(self.headers.get('Content-MD5')) or '',
_header_strip(self.headers.get('Content-Type')) or '']
buf = [swob.wsgi_to_bytes(wsgi_str) for wsgi_str in [
self.method,
_header_strip(self.headers.get('Content-MD5')) or '',
_header_strip(self.headers.get('Content-Type')) or '']]
if 'headers_raw' in self.environ: # eventlet >= 0.19.0
# See https://github.com/eventlet/eventlet/commit/67ec999
@ -948,18 +957,18 @@ class S3Request(swob.Request):
if self._is_header_auth:
if 'x-amz-date' in amz_headers:
buf.append('')
buf.append(b'')
elif 'Date' in self.headers:
buf.append(self.headers['Date'])
buf.append(swob.wsgi_to_bytes(self.headers['Date']))
elif self._is_query_auth:
buf.append(self.params['Expires'])
buf.append(swob.wsgi_to_bytes(self.params['Expires']))
else:
# Should have already raised NotS3Request in _parse_auth_info,
# but as a sanity check...
raise AccessDenied()
for key, value in sorted(amz_headers.items()):
buf.append("%s:%s" % (key, value))
buf.append(swob.wsgi_to_bytes("%s:%s" % (key, value)))
path = self._canonical_uri()
if self.query_string:
@ -971,10 +980,10 @@ class S3Request(swob.Request):
if key in ALLOWED_SUB_RESOURCES:
params.append('%s=%s' % (key, value) if value else key)
if params:
buf.append('%s?%s' % (path, '&'.join(params)))
buf.append(swob.wsgi_to_bytes('%s?%s' % (path, '&'.join(params))))
else:
buf.append(path)
return '\n'.join(buf)
buf.append(swob.wsgi_to_bytes(path))
return b'\n'.join(buf)
def signature_does_not_match_kwargs(self):
return {
@ -982,7 +991,8 @@ class S3Request(swob.Request):
'string_to_sign': self.string_to_sign,
'signature_provided': self.signature,
'string_to_sign_bytes': ' '.join(
format(ord(c), '02x') for c in self.string_to_sign),
format(ord(c), '02x')
for c in self.string_to_sign.decode('latin1')),
}
@property
@ -1320,7 +1330,6 @@ class S3Request(swob.Request):
# reuse account and tokens
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True)
self.account = utf8encode(self.account)
resp = S3Response.from_swift_resp(sw_resp)
status = resp.status_int # pylint: disable-msg=E1101
@ -1354,7 +1363,7 @@ class S3Request(swob.Request):
raise err_resp()
if status == HTTP_BAD_REQUEST:
raise BadSwiftRequest(err_msg)
raise BadSwiftRequest(err_msg.decode('utf8'))
if status == HTTP_UNAUTHORIZED:
raise SignatureDoesNotMatch(
**self.signature_does_not_match_kwargs())
@ -1487,7 +1496,6 @@ class S3AclRequest(S3Request):
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
2, 3, True)
self.account = utf8encode(self.account)
if 'HTTP_X_USER_NAME' in sw_resp.environ:
# keystone

4
swift/common/middleware/s3api/s3response.py
View File

@ -243,8 +243,10 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException):
if isinstance(value, (dict, MutableMapping)):
self._dict_to_etree(elem, value)
else:
if isinstance(value, (int, float, bool)):
value = str(value)
try:
elem.text = str(value)
elem.text = value
except ValueError:
# We set an invalid string for XML.
elem.text = '(invalid string)'

14
swift/common/middleware/s3api/subresource.py
View File

@ -43,6 +43,8 @@ http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html
"""
from functools import partial
import six
from swift.common.utils import json
from swift.common.middleware.s3api.s3response import InvalidArgument, \
@ -218,6 +220,11 @@ class User(Grantee):
def __str__(self):
return self.display_name
def __lt__(self, other):
if not isinstance(other, User):
return NotImplemented
return self.id < other.id
class Owner(object):
"""
@ -415,9 +422,14 @@ class ACL(object):
self.s3_acl = s3_acl
self.allow_no_owner = allow_no_owner
def __repr__(self):
def __bytes__(self):
return tostring(self.elem())
def __repr__(self):
if six.PY2:
return self.__bytes__()
return self.__bytes__().decode('utf8')
@classmethod
def from_elem(cls, elem, s3_acl=False, allow_no_owner=False):
"""

10
swift/common/swob.py
View File

@ -309,15 +309,15 @@ def str_to_wsgi(native_str):
return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
def wsgi_quote(wsgi_str):
def wsgi_quote(wsgi_str, safe='/'):
if six.PY2:
if not isinstance(wsgi_str, bytes):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote(wsgi_str)
return urllib.parse.quote(wsgi_str, safe=safe)
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
return urllib.parse.quote(wsgi_str, encoding='latin-1')
return urllib.parse.quote(wsgi_str, safe=safe, encoding='latin-1')
def wsgi_unquote(wsgi_str):
@ -478,6 +478,10 @@ def _resp_app_iter_property():
def setter(self, value):
if isinstance(value, (list, tuple)):
for i, item in enumerate(value):
if not isinstance(item, bytes):
raise TypeError('WSGI responses must be bytes; '
'got %s for item %d' % (type(item), i))
self.content_length = sum(map(len, value))
elif value is not None:
self.content_length = None

4
test/unit/common/middleware/s3api/helpers.py
View File

@ -165,12 +165,16 @@ class FakeSwift(object):
# keep old sysmeta for s3acl
headers.update({key: value})
if body is not None and not isinstance(body, (bytes, list)):
body = body.encode('utf8')
self._responses[(method, path)] = (response_class, headers, body)
def register_unconditionally(self, method, path, response_class, headers,
body):
# register() keeps old sysmeta around, but
# register_unconditionally() keeps nothing.
if body is not None and not isinstance(body, bytes):
body = body.encode('utf8')
self._responses[(method, path)] = (response_class, headers, body)
def clear_calls(self):

14
test/unit/common/middleware/s3api/test_acl.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import unittest
import mock
@ -131,8 +132,8 @@ class TestS3ApiAcl(S3ApiTestCase):
'UnexpectedContent')
def _test_put_no_body(self, use_content_length=False,
use_transfer_encoding=False, string_to_md5=''):
content_md5 = md5(string_to_md5).digest().encode('base64').strip()
use_transfer_encoding=False, string_to_md5=b''):
content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip()
with UnreadableInput(self) as fake_input:
req = Request.blank(
'/bucket?acl',
@ -153,16 +154,17 @@ class TestS3ApiAcl(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader')
self.assertEqual(self._get_error_message(body),
'Your request was missing a required header.')
self.assertIn('<MissingHeaderName>x-amz-acl</MissingHeaderName>', body)
self.assertIn(b'<MissingHeaderName>x-amz-acl</MissingHeaderName>',
body)
@s3acl
def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self):
self._test_put_no_body()
self._test_put_no_body(string_to_md5='test')
self._test_put_no_body(string_to_md5=b'test')
self._test_put_no_body(use_content_length=True)
self._test_put_no_body(use_content_length=True, string_to_md5='test')
self._test_put_no_body(use_content_length=True, string_to_md5=b'test')
self._test_put_no_body(use_transfer_encoding=True)
self._test_put_no_body(use_transfer_encoding=True, string_to_md5='zz')
self._test_put_no_body(use_transfer_encoding=True, string_to_md5=b'zz')
def test_object_acl_GET(self):
req = Request.blank('/bucket/object?acl',

23
test/unit/common/middleware/s3api/test_bucket.py
View File

@ -17,6 +17,7 @@ import unittest
import cgi
import mock
import six
from six.moves.urllib.parse import quote
from swift.common import swob
@ -62,7 +63,8 @@ class TestS3ApiBucket(S3ApiTestCase):
for name, _, _, _ in self.objects:
self.swift.register(
'DELETE',
'/v1/AUTH_test/bucket+segments/' + name.encode('utf-8'),
'/v1/AUTH_test/bucket+segments/' +
swob.bytes_to_wsgi(name.encode('utf-8')),
swob.HTTPNoContent, {}, json.dumps([]))
self.swift.register(
'GET',
@ -118,7 +120,7 @@ class TestS3ApiBucket(S3ApiTestCase):
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404')
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
def test_bucket_HEAD_slash(self):
req = Request.blank('/junk/',
@ -168,7 +170,8 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual('2011-01-05T02:19:14.275Z',
o.find('./LastModified').text)
self.assertEqual(items, [
(i[0].encode('utf-8'), '"0-N"' if i[0] == 'slo' else '"0"')
(i[0].encode('utf-8') if six.PY2 else i[0],
'"0-N"' if i[0] == 'slo' else '"0"')
for i in self.objects])
def test_bucket_GET_url_encoded(self):
@ -519,8 +522,12 @@ class TestS3ApiBucket(S3ApiTestCase):
self.assertEqual(elem.findall('./DeleteMarker'), [])
versions = elem.findall('./Version')
objects = list(self.objects)
self.assertEqual([v.find('./Key').text for v in versions],
[v[0].encode('utf-8') for v in objects])
if six.PY2:
self.assertEqual([v.find('./Key').text for v in versions],
[v[0].encode('utf-8') for v in objects])
else:
self.assertEqual([v.find('./Key').text for v in versions],
[v[0] for v in objects])
self.assertEqual([v.find('./IsLatest').text for v in versions],
['true' for v in objects])
self.assertEqual([v.find('./VersionId').text for v in versions],
@ -598,7 +605,7 @@ class TestS3ApiBucket(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(body, '')
self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket')
@ -610,7 +617,7 @@ class TestS3ApiBucket(S3ApiTestCase):
'Date': self.get_date_header(),
'Transfer-Encoding': 'chunked'})
status, headers, body = self.call_s3api(req)
self.assertEqual(body, '')
self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket')
@ -622,7 +629,7 @@ class TestS3ApiBucket(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac',
'Date': self.get_date_header()})
status, headers, body = self.call_s3api(req)
self.assertEqual(body, '')
self.assertEqual(body, b'')
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Location'], '/bucket')

15
test/unit/common/middleware/s3api/test_etree.py
View File

@ -15,6 +15,8 @@
import unittest
import six
from swift.common.middleware.s3api import etree
@ -58,15 +60,18 @@ class TestS3ApiEtree(unittest.TestCase):
sub.text = '\xef\xbc\xa1'
self.assertTrue(isinstance(sub.text, str))
xml_string = etree.tostring(elem)
self.assertTrue(isinstance(xml_string, str))
self.assertIsInstance(xml_string, bytes)
def test_fromstring_with_nonascii_text(self):
input_str = '<?xml version="1.0" encoding="UTF-8"?>\n' \
'<Test><FOO>\xef\xbc\xa1</FOO></Test>'
input_str = b'<?xml version="1.0" encoding="UTF-8"?>\n' \
b'<Test><FOO>\xef\xbc\xa1</FOO></Test>'
elem = etree.fromstring(input_str)
text = elem.find('FOO').text
self.assertEqual(text, '\xef\xbc\xa1')
self.assertTrue(isinstance(text, str))
if six.PY2:
self.assertEqual(text, b'\xef\xbc\xa1')
else:
self.assertEqual(text, b'\xef\xbc\xa1'.decode('utf8'))
self.assertIsInstance(text, str)
if __name__ == '__main__':

29
test/unit/common/middleware/s3api/test_multi_delete.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import json
import unittest
from datetime import datetime
@ -43,7 +44,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'object'
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket/object?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -80,7 +81,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -133,7 +134,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -180,7 +181,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -207,7 +208,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key')
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -232,7 +233,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
SubElement(obj, 'Key').text = key
SubElement(obj, 'VersionId').text = 'not-supported'
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -286,7 +287,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = name
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -308,7 +309,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = 'x' * 1000 + str(i)
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -333,7 +334,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
obj = SubElement(elem, 'Object')
SubElement(obj, 'Key').text = key
body = tostring(elem, use_s3ns=False)
content_md5 = md5(body).digest().encode('base64').strip()
content_md5 = base64.b64encode(md5(body).digest()).strip()
req = Request.blank('/bucket?delete',
environ={'REQUEST_METHOD': 'POST'},
@ -374,8 +375,8 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
self.assertEqual(len(elem.findall('Deleted')), len(self.keys))
def _test_no_body(self, use_content_length=False,
use_transfer_encoding=False, string_to_md5=''):
content_md5 = md5(string_to_md5).digest().encode('base64').strip()
use_transfer_encoding=False, string_to_md5=b''):
content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip()
with UnreadableInput(self) as fake_input:
req = Request.blank(
'/bucket?delete',
@ -398,11 +399,11 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
@s3acl
def test_object_multi_DELETE_empty_body(self):
self._test_no_body()
self._test_no_body(string_to_md5='test')
self._test_no_body(string_to_md5=b'test')
self._test_no_body(use_content_length=True)
self._test_no_body(use_content_length=True, string_to_md5='test')
self._test_no_body(use_content_length=True, string_to_md5=b'test')
self._test_no_body(use_transfer_encoding=True)
self._test_no_body(use_transfer_encoding=True, string_to_md5='test')
self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test')
if __name__ == '__main__':
unittest.main()

73
test/unit/common/middleware/s3api/test_multi_upload.py
View File

@ -14,12 +14,13 @@
# limitations under the License.
import base64
import binascii
import hashlib
from mock import patch
import os
import time
import unittest
from urllib import quote
from six.moves.urllib.parse import quote
from swift.common import swob
from swift.common.swob import Request
@ -66,9 +67,9 @@ MULTIPARTS_TEMPLATE = \
('subdir/object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210',
41))
S3_ETAG = '"%s-2"' % hashlib.md5((
S3_ETAG = '"%s-2"' % hashlib.md5(binascii.a2b_hex(
'0123456789abcdef0123456789abcdef'
'fedcba9876543210fedcba9876543210').decode('hex')).hexdigest()
'fedcba9876543210fedcba9876543210')).hexdigest()
class TestS3ApiMultiUpload(S3ApiTestCase):
@ -83,9 +84,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.s3api.conf.min_segment_size = 1
objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]},
OBJECTS_TEMPLATE)
objects = [{'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]}
for item in OBJECTS_TEMPLATE]
object_list = json.dumps(objects)
self.swift.register('PUT', segment_bucket,
@ -172,10 +173,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
multiparts=None):
segment_bucket = '/v1/AUTH_test/bucket+segments'
objects = multiparts or MULTIPARTS_TEMPLATE
objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]},
objects)
object_list = json.dumps(objects)
objects = [{'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]}
for item in objects]
object_list = json.dumps(objects).encode('ascii')
self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
object_list)
@ -568,7 +569,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self._test_object_multipart_upload_initiate({})
self._test_object_multipart_upload_initiate({'Etag': 'blahblahblah'})
self._test_object_multipart_upload_initiate({
'Content-MD5': base64.b64encode('blahblahblahblah').strip()})
'Content-MD5': base64.b64encode(b'blahblahblahblah').strip()})
@s3acl(s3acl_only=True)
@patch('swift.common.middleware.s3api.controllers.multi_upload.'
@ -667,7 +668,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'NoSuchBucket')
def test_object_multipart_upload_complete(self):
content_md5 = base64.b64encode(hashlib.md5(XML).digest())
content_md5 = base64.b64encode(hashlib.md5(
XML.encode('ascii')).digest())
req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
@ -701,7 +703,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(headers.get(h), override_etag)
def test_object_multipart_upload_invalid_md5(self):
bad_md5 = base64.b64encode(hashlib.md5(XML + 'some junk').digest())
bad_md5 = base64.b64encode(hashlib.md5(
XML.encode('ascii') + b'some junk').digest())
req = Request.blank('/bucket/object?uploadId=X',
environ={'REQUEST_METHOD': 'POST'},
headers={'Authorization': 'AWS test:tester:hmac',
@ -726,11 +729,11 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
]))
self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-ok',
swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({
swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Etag': '"slo-etag"',
'Response Status': '201 Created',
'Errors': [],
})])
}).encode('ascii')])
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
@ -748,14 +751,14 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), },
body=XML)
status, headers, body = self.call_s3api(req)
lines = body.split('\n')
self.assertTrue(lines[0].startswith('<?xml '))
lines = body.split(b'\n')
self.assertTrue(lines[0].startswith(b'<?xml '))
self.assertTrue(lines[1])
self.assertFalse(lines[1].strip())
fromstring(body, 'CompleteMultipartUploadResult')
self.assertEqual(status.split()[0], '200')
# NB: S3_ETAG includes quotes
self.assertIn('<ETag>%s</ETag>' % S3_ETAG, body)
self.assertIn(('<ETag>%s</ETag>' % S3_ETAG).encode('ascii'), body)
self.assertEqual(self.swift.calls, [
('HEAD', '/v1/AUTH_test/bucket'),
('HEAD', '/v1/AUTH_test/bucket+segments/heartbeat-ok/X'),
@ -779,10 +782,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
]))
self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-fail',
swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({
swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Response Status': '400 Bad Request',
'Errors': [['some/object', '403 Forbidden']],
})])
}).encode('ascii')])
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
@ -797,8 +800,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), },
body=XML)
status, headers, body = self.call_s3api(req)
lines = body.split('\n')
self.assertTrue(lines[0].startswith('<?xml '), (status, lines))
lines = body.split(b'\n')
self.assertTrue(lines[0].startswith(b'<?xml '), (status, lines))
self.assertTrue(lines[1])
self.assertFalse(lines[1].strip())
fromstring(body, 'Error')
@ -828,10 +831,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
]))
self.swift.register(
'PUT', '/v1/AUTH_test/bucket/heartbeat-fail',
swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({
swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({
'Response Status': '400 Bad Request',
'Errors': [['some/object', '404 Not Found']],
})])
}).encode('ascii')])
mock_time.time.side_effect = (
1, # start_time
12, # first whitespace
@ -846,8 +849,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
'Date': self.get_date_header(), },
body=XML)
status, headers, body = self.call_s3api(req)
lines = body.split('\n')
self.assertTrue(lines[0].startswith('<?xml '))
lines = body.split(b'\n')
self.assertTrue(lines[0].startswith(b'<?xml '))
self.assertTrue(lines[1])
self.assertFalse(lines[1].strip())
fromstring(body, 'Error')
@ -1116,8 +1119,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'CompleteMultipartUploadResult')
self.assertNotIn('Etag', headers)
expected_etag = '"%s-3"' % hashlib.md5(''.join(
x['hash'] for x in object_list).decode('hex')).hexdigest()
expected_etag = '"%s-3"' % hashlib.md5(binascii.unhexlify(''.join(
x['hash'] for x in object_list))).hexdigest()
self.assertEqual(elem.find('ETag').text, expected_etag)
self.assertEqual(self.swift.calls, [
@ -1843,8 +1846,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
account, src_headers={'Content-Length': '10'}, put_header=header)
self.assertEqual(status.split()[0], '400')
self.assertIn('Range specified is not valid for '
'source object of size: 10', body)
self.assertIn(b'Range specified is not valid for '
b'source object of size: 10', body)
self.assertEqual([
('HEAD', '/v1/AUTH_test/bucket'),
@ -1887,9 +1890,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
self.assertEqual('/src_bucket/src_obj', put_headers['X-Copy-From'])
def _test_no_body(self, use_content_length=False,
use_transfer_encoding=False, string_to_md5=''):
use_transfer_encoding=False, string_to_md5=b''):
raw_md5 = hashlib.md5(string_to_md5).digest()
content_md5 = raw_md5.encode('base64').strip()
content_md5 = base64.b64encode(raw_md5).strip()
with UnreadableInput(self) as fake_input:
req = Request.blank(
'/bucket/object?uploadId=X',
@ -1914,11 +1917,11 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
@s3acl
def test_object_multi_upload_empty_body(self):
self._test_no_body()
self._test_no_body(string_to_md5='test')
self._test_no_body(string_to_md5=b'test')
self._test_no_body(use_content_length=True)
self._test_no_body(use_content_length=True, string_to_md5='test')
self._test_no_body(use_content_length=True, string_to_md5=b'test')
self._test_no_body(use_transfer_encoding=True)
self._test_no_body(use_transfer_encoding=True, string_to_md5='test')
self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test')
class TestS3ApiMultiUploadNonUTC(TestS3ApiMultiUpload):

30
test/unit/common/middleware/s3api/test_obj.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import binascii
import unittest
from datetime import datetime
import hashlib
@ -20,6 +21,7 @@ import os
from os.path import join
import time
from mock import patch
import six
from swift.common import swob
from swift.common.swob import Request
@ -37,7 +39,7 @@ class TestS3ApiObj(S3ApiTestCase):
def setUp(self):
super(TestS3ApiObj, self).setUp()
self.object_body = 'hello'
self.object_body = b'hello'
self.etag = hashlib.md5(self.object_body).hexdigest()
self.last_modified = 'Fri, 01 Apr 2014 12:00:00 GMT'
@ -110,7 +112,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPUnauthorized, {}, None)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
@ -120,7 +122,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPForbidden, {}, None)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
@ -130,7 +132,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPNotFound, {}, None)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404')
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
@ -140,7 +142,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPPreconditionFailed, {}, None)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '412')
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
@ -150,7 +152,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPServerError, {}, None)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '500')
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
req = Request.blank('/bucket/object',
environ={'REQUEST_METHOD': 'HEAD'},
@ -160,7 +162,7 @@ class TestS3ApiObj(S3ApiTestCase):
swob.HTTPServiceUnavailable, {}, None)
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '500')
self.assertEqual(body, '') # sanity
self.assertEqual(body, b'') # sanity
def test_object_HEAD(self):
self._test_object_GETorHEAD('HEAD')
@ -448,7 +450,9 @@ class TestS3ApiObj(S3ApiTestCase):
@s3acl
def test_object_PUT(self):
etag = self.response_headers['etag']
content_md5 = etag.decode('hex').encode('base64').strip()
content_md5 = binascii.b2a_base64(binascii.a2b_hex(etag)).strip()
if not six.PY2:
content_md5 = content_md5.decode('ascii')
req = Request.blank(
'/bucket/object',
@ -524,7 +528,9 @@ class TestS3ApiObj(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'BadDigest')
def test_object_PUT_headers(self):
content_md5 = self.etag.decode('hex').encode('base64').strip()
content_md5 = binascii.b2a_base64(binascii.a2b_hex(self.etag)).strip()
if not six.PY2:
content_md5 = content_md5.decode('ascii')
self.swift.register('HEAD', '/v1/AUTH_test/some/source',
swob.HTTPOk, {'last-modified': self.last_modified},
@ -540,10 +546,12 @@ class TestS3ApiObj(S3ApiTestCase):
'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04',
'X-Amz-Copy-Source': '/some/source',
'Content-MD5': content_md5,
'Date': self.get_date_header()})
'Date': self.get_date_header()},
body=self.object_body)
req.date = datetime.now()
req.content_type = 'text/plain'
status, headers, body = self.call_s3api(req)
self.assertEqual('200 ', status[:4], body)
# Check that s3api does not return an etag header,
# specified copy source.
self.assertTrue(headers.get('etag') is None)
@ -1002,7 +1010,7 @@ class TestS3ApiObj(S3ApiTestCase):
'Content-Type': 'foo/bar'})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '204')
self.assertEqual(body, '')
self.assertEqual(body, b'')
self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object'),
self.swift.calls)

9
test/unit/common/middleware/s3api/test_s3_acl.py
View File

@ -66,9 +66,10 @@ def s3acl(func=None, s3acl_only=False):
exc_type, exc_instance, exc_traceback = sys.exc_info()
formatted_traceback = ''.join(traceback.format_tb(
exc_traceback))
message = '\n%s\n%s:\n%s' % (formatted_traceback,
exc_type.__name__,
exc_instance.message)
message = '\n%s\n%s' % (formatted_traceback,
exc_type.__name__)
if exc_instance.args:
message += ':\n%s' % (exc_instance.args[0],)
message += failing_point
raise exc_type(message)
@ -114,7 +115,7 @@ def generate_s3acl_environ(account, swift, owner):
account_name = '%s:%s' % (account, permission.lower())
return Grant(User(account_name), permission)
grants = map(gen_grant, PERMISSIONS)
grants = [gen_grant(perm) for perm in PERMISSIONS]
container_headers = _gen_test_headers(owner, grants)
object_headers = _gen_test_headers(owner, grants, 'object')
object_body = 'hello'

58
test/unit/common/middleware/s3api/test_s3api.py
View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base64
import unittest
from mock import patch, MagicMock
import calendar
@ -22,7 +23,8 @@ import mock
import requests
import json
import copy
from urllib import unquote, quote
import six
from six.moves.urllib.parse import unquote, quote
import swift.common.middleware.s3api
from swift.common.middleware.keystoneauth import KeystoneAuth
@ -100,7 +102,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
def test_non_s3_request_passthrough(self):
req = Request.blank('/something')
status, headers, body = self.call_s3api(req)
self.assertEqual(body, 'FAKE APP')
self.assertEqual(body, b'FAKE APP')
def test_bad_format_authorization(self):
req = Request.blank('/something',
@ -336,7 +338,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertIsNone(headers['X-Auth-Token'])
def test_signed_urls_v4_bad_credential(self):
def test(credential, message, extra=''):
def test(credential, message, extra=b''):
req = Request.blank(
'/bucket/object'
'?X-Amz-Algorithm=AWS4-HMAC-SHA256'
@ -364,7 +366,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
test('test:tester/%s/us-west-1/s3/aws4_request' % dt,
"Error parsing the X-Amz-Credential parameter; the region "
"'us-west-1' is wrong; expecting 'us-east-1'",
'<Region>us-east-1</Region>')
b'<Region>us-east-1</Region>')
test('test:tester/%s/us-east-1/not-s3/aws4_request' % dt,
'Error parsing the X-Amz-Credential parameter; incorrect service '
'"not-s3". This endpoint belongs to "s3".')
@ -483,9 +485,9 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(req.environ['s3api.auth_details'], {
'access_key': 'test:tester',
'signature': 'hmac',
'string_to_sign': '\n'.join([
'PUT', '', '', date_header,
'/bucket/object?partNumber=1&uploadId=123456789abcdef']),
'string_to_sign': b'\n'.join([
b'PUT', b'', b'', date_header.encode('ascii'),
b'/bucket/object?partNumber=1&uploadId=123456789abcdef']),
'check_signature': mock_cs})
def test_invalid_uri(self):
@ -506,8 +508,10 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'InvalidDigest')
def test_object_create_bad_md5_too_short(self):
too_short_digest = hashlib.md5('hey').hexdigest()[:-1]
md5_str = too_short_digest.encode('base64').strip()
too_short_digest = hashlib.md5(b'hey').digest()[:-1]
md5_str = base64.b64encode(too_short_digest).strip()
if not six.PY2:
md5_str = md5_str.decode('ascii')
req = Request.blank(
'/bucket/object',
environ={'REQUEST_METHOD': 'PUT',
@ -518,8 +522,10 @@ class TestS3ApiMiddleware(S3ApiTestCase):
self.assertEqual(self._get_error_code(body), 'InvalidDigest')
def test_object_create_bad_md5_too_long(self):
too_long_digest = hashlib.md5('hey').hexdigest() + 'suffix'
md5_str = too_long_digest.encode('base64').strip()
too_long_digest = hashlib.md5(b'hey').digest() + b'suffix'
md5_str = base64.b64encode(too_long_digest).strip()
if not six.PY2:
md5_str = md5_str.decode('ascii')
req = Request.blank(
'/bucket/object',
environ={'REQUEST_METHOD': 'PUT',
@ -705,13 +711,13 @@ class TestS3ApiMiddleware(S3ApiTestCase):
with self.assertRaises(ValueError) as cm:
self.s3api.check_pipeline(self.conf)
self.assertIn('expected auth between s3api and proxy-server',
cm.exception.message)
cm.exception.args[0])
pipeline.return_value = 'proxy-server'
with self.assertRaises(ValueError) as cm:
self.s3api.check_pipeline(self.conf)
self.assertIn("missing filters ['s3api']",
cm.exception.message)
cm.exception.args[0])
def test_s3api_initialization_with_disabled_pipeline_check(self):
with patch("swift.common.middleware.s3api.s3api.loadcontext"), \
@ -799,7 +805,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'Missing required header for this request: x-amz-content-sha256')
def test_signature_v4_bad_authorization_string(self):
def test(auth_str, error, msg, extra=''):
def test(auth_str, error, msg, extra=b''):
environ = {
'REQUEST_METHOD': 'GET'}
headers = {
@ -835,7 +841,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
test(auth_str, 'AuthorizationHeaderMalformed',
"The authorization header is malformed; "
"the region 'us-west-2' is wrong; expecting 'us-east-1'",
'<Region>us-east-1</Region>')
b'<Region>us-east-1</Region>')
auth_str = ('AWS4-HMAC-SHA256 '
'Credential=test:tester/%s/us-east-1/not-s3/aws4_request, '
@ -901,8 +907,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
patch.object(swift.common.middleware.s3api.s3request,
'SERVICE', 'host'):
req = _get_req(path, environ)
hash_in_sts = req._string_to_sign().split('\n')[3]
self.assertEqual(hash_val, hash_in_sts)
hash_in_sts = req._string_to_sign().split(b'\n')[3]
self.assertEqual(hash_val, hash_in_sts.decode('ascii'))
self.assertTrue(req.check_signature(
'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY'))
@ -963,7 +969,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
'validate_bucket_name'):
verify('27ba31df5dbc6e063d8f87d62eb07143'
'f7f271c5330a917840586ac1c85b6f6b',
unquote('/%E1%88%B4'), env)
swob.wsgi_unquote('/%E1%88%B4'), env)
# get-vanilla-query-order-key
env = {
@ -1101,12 +1107,12 @@ class TestS3ApiMiddleware(S3ApiTestCase):
swob.HTTPOk, {}, None)
with patch.object(self.s3_token, '_json_request') as mock_req:
mock_resp = requests.Response()
mock_resp._content = json.dumps(GOOD_RESPONSE_V2)
mock_resp._content = json.dumps(GOOD_RESPONSE_V2).encode('ascii')
mock_resp.status_code = 201
mock_req.return_value = mock_resp
status, headers, body = self.call_s3api(req)
self.assertEqual(body, '')
self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count)
def test_s3api_with_only_s3_token_v3(self):
@ -1127,12 +1133,12 @@ class TestS3ApiMiddleware(S3ApiTestCase):
swob.HTTPOk, {}, None)
with patch.object(self.s3_token, '_json_request') as mock_req:
mock_resp = requests.Response()
mock_resp._content = json.dumps(GOOD_RESPONSE_V3)
mock_resp._content = json.dumps(GOOD_RESPONSE_V3).encode('ascii')
mock_resp.status_code = 200
mock_req.return_value = mock_resp
status, headers, body = self.call_s3api(req)
self.assertEqual(body, '')
self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count)
def test_s3api_with_s3_token_and_auth_token(self):
@ -1157,7 +1163,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
with patch.object(self.auth_token,
'_do_fetch_token') as mock_fetch:
mock_resp = requests.Response()
mock_resp._content = json.dumps(GOOD_RESPONSE_V2)
mock_resp._content = json.dumps(
GOOD_RESPONSE_V2).encode('ascii')
mock_resp.status_code = 201
mock_req.return_value = mock_resp
@ -1167,7 +1174,7 @@ class TestS3ApiMiddleware(S3ApiTestCase):
mock_fetch.return_value = (MagicMock(), mock_access_info)
status, headers, body = self.call_s3api(req)
self.assertEqual(body, '')
self.assertEqual(body, b'')
self.assertEqual(1, mock_req.call_count)
# With X-Auth-Token, auth_token will call _do_fetch_token to
# connect to keystone in auth_token, again
@ -1198,7 +1205,8 @@ class TestS3ApiMiddleware(S3ApiTestCase):
no_token_id_good_resp = copy.deepcopy(GOOD_RESPONSE_V2)
# delete token id
del no_token_id_good_resp['access']['token']['id']
mock_resp._content = json.dumps(no_token_id_good_resp)
mock_resp._content = json.dumps(
no_token_id_good_resp).encode('ascii')
mock_resp.status_code = 201
mock_req.return_value = mock_resp

106
test/unit/common/middleware/s3api/test_s3request.py
View File

@ -143,7 +143,7 @@ class TestRequest(S3ApiTestCase):
def test_get_response_without_match_ACL_MAP(self):
with self.assertRaises(Exception) as e:
self._test_get_response('POST', req_klass=S3AclRequest)
self.assertEqual(e.exception.message,
self.assertEqual(e.exception.args[0],
'No permission to be checked exists')
def test_get_response_without_duplication_HEAD_request(self):
@ -215,8 +215,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', '1' * 30)
with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1)
self.assertTrue(
'not an integer or within integer range' in result.exception.body)
self.assertIn(
b'not an integer or within integer range', result.exception.body)
self.assertEqual(
result.exception.headers['content-type'], 'application/xml')
@ -224,8 +224,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', '-1')
with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1)
self.assertTrue(
'must be an integer between 0 and' in result.exception.body)
self.assertIn(
b'must be an integer between 0 and', result.exception.body)
self.assertEqual(
result.exception.headers['content-type'], 'application/xml')
@ -233,8 +233,8 @@ class TestRequest(S3ApiTestCase):
s3req = create_s3request_with_param('max-keys', 'invalid')
with self.assertRaises(InvalidArgument) as result:
s3req.get_validated_param('max-keys', 1)
self.assertTrue(
'not an integer or within integer range' in result.exception.body)
self.assertIn(
b'not an integer or within integer range', result.exception.body)
self.assertEqual(
result.exception.headers['content-type'], 'application/xml')
@ -351,7 +351,7 @@ class TestRequest(S3ApiTestCase):
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '')
self.assertEqual(body, b'')
def test_date_header_expired(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@ -363,7 +363,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '')
self.assertEqual(body, b'')
def test_date_header_with_x_amz_date_valid(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@ -376,7 +376,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '404')
self.assertEqual(body, '')
self.assertEqual(body, b'')
def test_date_header_with_x_amz_date_expired(self):
self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
@ -390,7 +390,7 @@ class TestRequest(S3ApiTestCase):
status, headers, body = self.call_s3api(req)
self.assertEqual(status.split()[0], '403')
self.assertEqual(body, '')
self.assertEqual(body, b'')
def _test_request_timestamp_sigv4(self, date_header):
# signature v4 here
@ -428,7 +428,7 @@ class TestRequest(S3ApiTestCase):
def test_request_timestamp_sigv4(self):
access_denied_message = \
'AWS authentication requires a valid Date or x-amz-date header'
b'AWS authentication requires a valid Date or x-amz-date header'
# normal X-Amz-Date header
date_header = {'X-Amz-Date': self.get_v4_amz_date_header()}
@ -443,7 +443,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# mangled Date header
@ -451,7 +451,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# Negative timestamp
@ -459,7 +459,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# far-past Date header
@ -467,7 +467,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# near-future X-Amz-Date header
@ -481,9 +481,9 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(RequestTimeTooSkewed) as cm:
self._test_request_timestamp_sigv4(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertIn('The difference between the request time and the '
'current time is too large.', cm.exception.body)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(b'The difference between the request time and the '
b'current time is too large.', cm.exception.body)
def _test_request_timestamp_sigv2(self, date_header):
# signature v4 here
@ -505,7 +505,7 @@ class TestRequest(S3ApiTestCase):
def test_request_timestamp_sigv2(self):
access_denied_message = \
'AWS authentication requires a valid Date or x-amz-date header'
b'AWS authentication requires a valid Date or x-amz-date header'
# In v2 format, normal X-Amz-Date header is same
date_header = {'X-Amz-Date': self.get_date_header()}
@ -520,7 +520,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# mangled Date header
@ -528,7 +528,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# Negative timestamp
@ -536,7 +536,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# far-past Date header
@ -544,7 +544,7 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(AccessDenied) as cm:
self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(access_denied_message, cm.exception.body)
# far-future Date header
@ -552,9 +552,9 @@ class TestRequest(S3ApiTestCase):
with self.assertRaises(RequestTimeTooSkewed) as cm:
self._test_request_timestamp_sigv2(date_header)
self.assertEqual('403 Forbidden', cm.exception.message)
self.assertIn('The difference between the request time and the '
'current time is too large.', cm.exception.body)
self.assertEqual('403 Forbidden', cm.exception.args[0])
self.assertIn(b'The difference between the request time and the '
b'current time is too large.', cm.exception.body)
def test_headers_to_sign_sigv4(self):
environ = {
@ -681,14 +681,14 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/')
self.assertEqual(uri, b'/')
self.assertEqual(req.environ['PATH_INFO'], '/')
req = Request.blank('/obj1', environ=environ, headers=headers)
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/obj1')
self.assertEqual(uri, b'/obj1')
self.assertEqual(req.environ['PATH_INFO'], '/obj1')
environ = {
@ -701,7 +701,7 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/')
self.assertEqual(uri, b'/')
self.assertEqual(req.environ['PATH_INFO'], '/')
req = Request.blank('/bucket/obj1',
@ -710,7 +710,7 @@ class TestRequest(S3ApiTestCase):
sigv4_req = SigV4Request(req.environ)
uri = sigv4_req._canonical_uri()
self.assertEqual(uri, '/bucket/obj1')
self.assertEqual(uri, b'/bucket/obj1')
self.assertEqual(req.environ['PATH_INFO'], '/bucket/obj1')
@patch.object(S3Request, '_validate_dates', lambda *a: None)
@ -724,12 +724,12 @@ class TestRequest(S3ApiTestCase):
'bWq2s1WEIj+Ydj0vQ697zp+IXMU='),
})
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
expected_sts = '\n'.join([
'GET',
'',
'',
'Tue, 27 Mar 2007 19:36:42 +0000',
'/johnsmith/photos/puppy.jpg',
expected_sts = b'\n'.join([
b'GET',
b'',
b'',
b'Tue, 27 Mar 2007 19:36:42 +0000',
b'/johnsmith/photos/puppy.jpg',
])
self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret))
@ -743,12 +743,12 @@ class TestRequest(S3ApiTestCase):
'MyyxeRY7whkBe+bq8fHCL/2kKUg='),
})
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
expected_sts = '\n'.join([
'PUT',
'',
'image/jpeg',
'Tue, 27 Mar 2007 21:15:45 +0000',
'/johnsmith/photos/puppy.jpg',
expected_sts = b'\n'.join([
b'PUT',
b'',
b'image/jpeg',
b'Tue, 27 Mar 2007 21:15:45 +0000',
b'/johnsmith/photos/puppy.jpg',
])
self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret))
@ -763,12 +763,12 @@ class TestRequest(S3ApiTestCase):
'htDYFYduRNen8P9ZfE/s9SuKy0U='),
})
sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com')
expected_sts = '\n'.join([
'GET',
'',
'',
'Tue, 27 Mar 2007 19:42:41 +0000',
'/johnsmith/',
expected_sts = b'\n'.join([
b'GET',
b'',
b'',
b'Tue, 27 Mar 2007 19:42:41 +0000',
b'/johnsmith/',
])
self.assertEqual(expected_sts, sigv2_req._string_to_sign())
self.assertTrue(sigv2_req.check_signature(secret))
@ -846,7 +846,7 @@ class TestHashingInput(S3ApiTestCase):
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected
with self.assertRaises(swob.Response) as raised:
with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input
@ -859,7 +859,7 @@ class TestHashingInput(S3ApiTestCase):
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'56', wrapped.read(2))
# even though the hash matches, there was more data than we expected
with self.assertRaises(swob.Response) as raised:
with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed)
@ -870,14 +870,14 @@ class TestHashingInput(S3ApiTestCase):
hashlib.md5(raw).hexdigest())
self.assertEqual(b'1234', wrapped.read(4))
self.assertEqual(b'5678', wrapped.read(4))
with self.assertRaises(swob.Response) as raised:
with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(4)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
self.assertTrue(wrapped._input.closed)
def test_empty_bad_hash(self):
wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope')
with self.assertRaises(swob.Response) as raised:
with self.assertRaises(swob.HTTPException) as raised:
wrapped.read(3)
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
# the error causes us to close the input

2
test/unit/common/middleware/s3api/test_service.py
View File

@ -135,7 +135,7 @@ class TestS3ApiService(S3ApiTestCase):
self.assertEqual(len(names), len(expected))
for i in expected:
self.assertTrue(i[0] in names)
self.assertIn(i[0], names)
def _test_service_GET_for_check_bucket_owner(self, buckets):
self.s3api.conf.check_bucket_owner = True

2
test/unit/common/test_internal_client.py
View File

@ -1289,7 +1289,7 @@ class TestInternalClient(unittest.TestCase):
def fake_app(self, env, start_response):
start_response('404 Not Found', [])
return ['one\ntwo\nthree']
return [b'one\ntwo\nthree']
client = InternalClient()
lines = []

14
test/unit/common/test_swob.py
View File

@ -670,7 +670,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
return ['hi']
return [b'hi']
# Request environment contains valid account in path
req = swift.common.swob.Request.blank('/v1/account-name')
@ -692,7 +692,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
return ['hi']
return [b'hi']
# Request environment contains bad path
req = swift.common.swob.Request.blank('/random')
@ -706,7 +706,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
return ['no creds in request']
return [b'no creds in request']
# Request to get token
req = swift.common.swob.Request.blank('/v1.0/auth')
@ -729,7 +729,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', {
'Www-Authenticate': 'Me realm="whatever"'})
return ['no creds in request']
return [b'no creds in request']
# Auth middleware sets own Www-Authenticate
req = swift.common.swob.Request.blank('/auth/v1.0')
@ -743,7 +743,7 @@ class TestRequest(unittest.TestCase):
def test_app(environ, start_response):
start_response('401 Unauthorized', [])
return ['hi']
return [b'hi']
hacker = 'account-name\n\n<b>foo<br>' # url injection test
quoted_hacker = quote(hacker)
@ -766,7 +766,7 @@ class TestRequest(unittest.TestCase):
# Other status codes should not have WWW-Authenticate in response
def test_app(environ, start_response):
start_response('200 OK', [])
return ['hi']
return [b'hi']
req = swift.common.swob.Request.blank('/')
resp = req.get_response(test_app)
@ -1763,7 +1763,7 @@ class TestConditionalIfMatch(unittest.TestCase):
def fake_app_404(environ, start_response):
start_response('404 Not Found', [])
return ['hi']
return [b'hi']
req = swift.common.swob.Request.blank(
'/', headers={'If-Match': '*'})

2
tox.ini
View File

@ -40,7 +40,7 @@ commands =
test/unit/account \
test/unit/cli \
test/unit/common/middleware/crypto \
test/unit/common/middleware/s3api/test_s3token.py \
test/unit/common/middleware/s3api/ \
test/unit/common/middleware/test_account_quotas.py \
test/unit/common/middleware/test_acl.py \
test/unit/common/middleware/test_catch_errors.py \