From 3a9f3f841957f54801c32832356c540dbef69817 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 9 Apr 2019 16:47:20 -0700 Subject: [PATCH] py3: port s3api Drive-by: When passing a list or tuple to swob.Response as an app_iter, check that it's full of byte strings. Change-Id: Ifc35aacb2e45004f74c871f08ff3c52bc57c1463 --- .../middleware/s3api/controllers/bucket.py | 6 +- .../s3api/controllers/multi_upload.py | 29 +++-- .../middleware/s3api/controllers/obj.py | 2 +- swift/common/middleware/s3api/etree.py | 4 +- swift/common/middleware/s3api/s3api.py | 4 +- swift/common/middleware/s3api/s3request.py | 84 +++++++------- swift/common/middleware/s3api/s3response.py | 4 +- swift/common/middleware/s3api/subresource.py | 14 ++- swift/common/swob.py | 10 +- test/unit/common/middleware/s3api/helpers.py | 4 + test/unit/common/middleware/s3api/test_acl.py | 14 ++- .../common/middleware/s3api/test_bucket.py | 23 ++-- .../common/middleware/s3api/test_etree.py | 15 ++- .../middleware/s3api/test_multi_delete.py | 29 ++--- .../middleware/s3api/test_multi_upload.py | 73 ++++++------ test/unit/common/middleware/s3api/test_obj.py | 30 +++-- .../common/middleware/s3api/test_s3_acl.py | 9 +- .../common/middleware/s3api/test_s3api.py | 58 +++++----- .../common/middleware/s3api/test_s3request.py | 106 +++++++++--------- .../common/middleware/s3api/test_service.py | 2 +- test/unit/common/test_internal_client.py | 2 +- test/unit/common/test_swob.py | 14 +-- tox.ini | 2 +- 23 files changed, 308 insertions(+), 230 deletions(-) diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index b4f662acb5..350e7ba9c4 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -18,6 +18,7 @@ from base64 import standard_b64decode as b64decode from six.moves.urllib.parse import quote +from swift.common import swob from swift.common.http import HTTP_OK from swift.common.utils import json, public, config_true_value @@ -66,8 +67,9 @@ class BucketController(Controller): segments = json.loads(resp.body) for seg in segments: try: - req.get_response(self.app, 'DELETE', container, - seg['name'].encode('utf8')) + req.get_response( + self.app, 'DELETE', container, + swob.bytes_to_wsgi(seg['name'].encode('utf8'))) except NoSuchKey: pass except InternalError: diff --git a/swift/common/middleware/s3api/controllers/multi_upload.py b/swift/common/middleware/s3api/controllers/multi_upload.py index 91240a9159..10fd8ed68d 100644 --- a/swift/common/middleware/s3api/controllers/multi_upload.py +++ b/swift/common/middleware/s3api/controllers/multi_upload.py @@ -59,11 +59,14 @@ Static Large Object when the multipart upload is completed. """ +import binascii from hashlib import md5 import os import re import time +import six + from swift.common.swob import Range from swift.common.utils import json, public, reiterate from swift.common.db import utf8encode @@ -222,8 +225,9 @@ class UploadsController(Controller): :return (non_delimited_uploads, common_prefixes) """ - (prefix, delimiter) = \ - utf8encode(prefix, delimiter) + if six.PY2: + (prefix, delimiter) = \ + utf8encode(prefix, delimiter) non_delimited_uploads = [] common_prefixes = set() for upload in uploads: @@ -440,7 +444,7 @@ class UploadController(Controller): # If the caller requested a list starting at a specific part number, # construct a sub-set of the object list. - objList = filter(filter_part_num_marker, objects) + objList = [obj for obj in objects if filter_part_num_marker(obj)] # pylint: disable-msg=E1103 objList.sort(key=lambda o: int(o['name'].split('/')[-1])) @@ -603,7 +607,7 @@ class UploadController(Controller): 'path': '/%s/%s/%s/%d' % ( container, req.object_name, upload_id, part_number), 'etag': etag}) - s3_etag_hasher.update(etag.decode('hex')) + s3_etag_hasher.update(binascii.a2b_hex(etag)) except (XMLSyntaxError, DocumentInvalid): # NB: our schema definitions catch uploads with no parts here raise MalformedXML() @@ -661,8 +665,8 @@ class UploadController(Controller): # ceph-s3tests happy continue if not yielded_anything: - yield ('\n') + yield (b'\n') yielded_anything = True yield chunk continue @@ -708,8 +712,13 @@ class UploadController(Controller): # in detail, https://github.com/boto/boto/pull/3513 parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) - if parsed_url.port: - host_url += ':%s' % parsed_url.port + # Why are we doing our own port parsing? Because py3 decided + # to start raising ValueErrors on access after parsing such + # an invalid port + netloc = parsed_url.netloc.split('@')[-1].split(']')[-1] + if ':' in netloc: + port = netloc.split(':', 2)[1] + host_url += ':%s' % port SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = req.container_name @@ -717,13 +726,13 @@ class UploadController(Controller): SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag resp.headers.pop('ETag', None) if yielded_anything: - yield '\n' + yield b'\n' yield tostring(result_elem, xml_declaration=not yielded_anything) except ErrorResponse as err_resp: if yielded_anything: err_resp.xml_declaration = False - yield '\n' + yield b'\n' else: # Oh good, we can still change HTTP status code, too! resp.status = err_resp.status diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py index 0a5a341d50..5a30c44dec 100644 --- a/swift/common/middleware/s3api/controllers/obj.py +++ b/swift/common/middleware/s3api/controllers/obj.py @@ -156,7 +156,7 @@ class ObjectController(Controller): for chunk in resp.app_iter: pass # drain the bulk-deleter response resp.status = HTTP_NO_CONTENT - resp.body = '' + resp.body = b'' except NoSuchKey: # expect to raise NoSuchBucket when the bucket doesn't exist req.get_container_info(self.app) diff --git a/swift/common/middleware/s3api/etree.py b/swift/common/middleware/s3api/etree.py index dcdd7f616d..29adbc38ed 100644 --- a/swift/common/middleware/s3api/etree.py +++ b/swift/common/middleware/s3api/etree.py @@ -120,7 +120,9 @@ class _Element(lxml.etree.ElementBase): """ utf-8 wrapper property of lxml.etree.Element.text """ - return utf8encode(lxml.etree.ElementBase.text.__get__(self)) + if six.PY2: + return utf8encode(lxml.etree.ElementBase.text.__get__(self)) + return lxml.etree.ElementBase.text.__get__(self) @text.setter def text(self, value): diff --git a/swift/common/middleware/s3api/s3api.py b/swift/common/middleware/s3api/s3api.py index c6bb27b04c..9682714f64 100644 --- a/swift/common/middleware/s3api/s3api.py +++ b/swift/common/middleware/s3api/s3api.py @@ -167,7 +167,7 @@ class ListingEtagMiddleware(object): ctx._response_exc_info) return [body] - body = json.dumps(listing) + body = json.dumps(listing).encode('ascii') ctx._response_headers[cl_index] = ( ctx._response_headers[cl_index][0], str(len(body)), @@ -237,7 +237,7 @@ class S3ApiMiddleware(object): resp = err_resp except Exception as e: self.logger.exception(e) - resp = InternalError(reason=e) + resp = InternalError(reason=str(e)) if isinstance(resp, S3ResponseBase) and 'swift.trans_id' in env: resp.headers['x-amz-id-2'] = env['swift.trans_id'] diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index 65051ebcdf..2ae7ac4f6a 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -14,6 +14,7 @@ # limitations under the License. import base64 +import binascii from collections import defaultdict, OrderedDict from email.header import Header from hashlib import sha1, sha256, md5 @@ -127,7 +128,8 @@ class HashingInput(object): chunk = self._input.read(size) self._hasher.update(chunk) self._to_read -= len(chunk) - if self._to_read < 0 or (size > len(chunk) and self._to_read) or ( + short_read = bool(chunk) if size is None else (len(chunk) < size) + if self._to_read < 0 or (short_read and self._to_read) or ( self._to_read == 0 and self._hasher.hexdigest() != self._expected): self.close() @@ -149,10 +151,10 @@ class SigV4Mixin(object): def check_signature(self, secret): secret = utf8encode(secret) user_signature = self.signature - derived_secret = 'AWS4' + secret + derived_secret = b'AWS4' + secret for scope_piece in self.scope.values(): derived_secret = hmac.new( - derived_secret, scope_piece, sha256).digest() + derived_secret, scope_piece.encode('utf8'), sha256).digest() valid_signature = hmac.new( derived_secret, self.string_to_sign, sha256).hexdigest() return user_signature == valid_signature @@ -331,10 +333,10 @@ class SigV4Mixin(object): def _canonical_query_string(self): return '&'.join( - '%s=%s' % (quote(key, safe='-_.~'), - quote(value, safe='-_.~')) + '%s=%s' % (swob.wsgi_quote(key, safe='-_.~'), + swob.wsgi_quote(value, safe='-_.~')) for key, value in sorted(self.params.items()) - if key not in ('Signature', 'X-Amz-Signature')) + if key not in ('Signature', 'X-Amz-Signature')).encode('ascii') def _headers_to_sign(self): """ @@ -383,7 +385,7 @@ class SigV4Mixin(object): """ It won't require bucket name in canonical_uri for v4. """ - return self.environ.get('RAW_PATH_INFO', self.path) + return swob.wsgi_to_bytes(self.environ.get('RAW_PATH_INFO', self.path)) def _canonical_request(self): # prepare 'canonical_request' @@ -401,7 +403,7 @@ class SigV4Mixin(object): # # 1. Add verb like: GET - cr = [self.method.upper()] + cr = [swob.wsgi_to_bytes(self.method.upper())] # 2. Add path like: / path = self._canonical_uri() @@ -415,12 +417,12 @@ class SigV4Mixin(object): # host:iam.amazonaws.com # x-amz-date:20150830T123600Z headers_to_sign = self._headers_to_sign() - cr.append(''.join('%s:%s\n' % (key, value) - for key, value in headers_to_sign)) + cr.append(b''.join(swob.wsgi_to_bytes('%s:%s\n' % (key, value)) + for key, value in headers_to_sign)) # 5. Add signed headers into canonical request like # content-type;host;x-amz-date - cr.append(';'.join(k for k, v in headers_to_sign)) + cr.append(b';'.join(swob.wsgi_to_bytes(k) for k, v in headers_to_sign)) # 6. Add payload string at the tail if 'X-Amz-Credential' in self.params: @@ -446,8 +448,8 @@ class SigV4Mixin(object): # else, not provided -- Swift will kick out a 411 Length Required # which will get translated back to a S3-style response in # S3Request._swift_error_codes - cr.append(hashed_payload) - return '\n'.join(cr).encode('utf-8') + cr.append(swob.wsgi_to_bytes(hashed_payload)) + return b'\n'.join(cr) @property def scope(self): @@ -462,10 +464,11 @@ class SigV4Mixin(object): """ Create 'StringToSign' value in Amazon terminology for v4. """ - return '\n'.join(['AWS4-HMAC-SHA256', - self.timestamp.amz_date_format, - '/'.join(self.scope.values()), - sha256(self._canonical_request()).hexdigest()]) + return b'\n'.join([ + b'AWS4-HMAC-SHA256', + self.timestamp.amz_date_format.encode('ascii'), + '/'.join(self.scope.values()).encode('utf8'), + sha256(self._canonical_request()).hexdigest().encode('ascii')]) def signature_does_not_match_kwargs(self): kwargs = super(SigV4Mixin, self).signature_does_not_match_kwargs() @@ -473,7 +476,7 @@ class SigV4Mixin(object): kwargs.update({ 'canonical_request': cr, 'canonical_request_bytes': ' '.join( - format(ord(c), '02x') for c in cr), + format(ord(c), '02x') for c in cr.decode('latin1')), }) return kwargs @@ -545,6 +548,8 @@ class S3Request(swob.Request): user_signature = self.signature valid_signature = base64.b64encode(hmac.new( secret, self.string_to_sign, sha1).digest()).strip() + if not six.PY2: + valid_signature = valid_signature.decode('ascii') return user_signature == valid_signature @property @@ -613,7 +618,7 @@ class S3Request(swob.Request): return None def _parse_uri(self): - if not check_utf8(self.environ['PATH_INFO']): + if not check_utf8(swob.wsgi_to_str(self.environ['PATH_INFO'])): raise InvalidURI(self.path) if self.bucket_in_host: @@ -739,8 +744,10 @@ class S3Request(swob.Request): # Non-base64-alphabet characters in value. raise InvalidDigest(content_md5=value) try: - self.headers['ETag'] = value.decode('base64').encode('hex') - except Exception: + self.headers['ETag'] = binascii.b2a_hex( + binascii.a2b_base64(value)) + except binascii.error: + # incorrect padding, most likely raise InvalidDigest(content_md5=value) if len(self.headers['ETag']) != 32: @@ -825,10 +832,11 @@ class S3Request(swob.Request): 'functionality that is not implemented', header='Transfer-Encoding') - if self.message_length() > max_length: + ml = self.message_length() + if ml and ml > max_length: raise MalformedXML() - if te or self.message_length(): + if te or ml: # Limit the read similar to how SLO handles manifests body = self.body_file.read(max_length) else: @@ -843,7 +851,7 @@ class S3Request(swob.Request): raise InvalidRequest('Missing required header for this request: ' 'Content-MD5') - digest = md5(body).digest().encode('base64').strip() + digest = base64.b64encode(md5(body).digest()).strip().decode('ascii') if self.environ['HTTP_CONTENT_MD5'] != digest: raise BadDigest(content_md5=self.environ['HTTP_CONTENT_MD5']) @@ -927,9 +935,10 @@ class S3Request(swob.Request): """ amz_headers = {} - buf = [self.method, - _header_strip(self.headers.get('Content-MD5')) or '', - _header_strip(self.headers.get('Content-Type')) or ''] + buf = [swob.wsgi_to_bytes(wsgi_str) for wsgi_str in [ + self.method, + _header_strip(self.headers.get('Content-MD5')) or '', + _header_strip(self.headers.get('Content-Type')) or '']] if 'headers_raw' in self.environ: # eventlet >= 0.19.0 # See https://github.com/eventlet/eventlet/commit/67ec999 @@ -948,18 +957,18 @@ class S3Request(swob.Request): if self._is_header_auth: if 'x-amz-date' in amz_headers: - buf.append('') + buf.append(b'') elif 'Date' in self.headers: - buf.append(self.headers['Date']) + buf.append(swob.wsgi_to_bytes(self.headers['Date'])) elif self._is_query_auth: - buf.append(self.params['Expires']) + buf.append(swob.wsgi_to_bytes(self.params['Expires'])) else: # Should have already raised NotS3Request in _parse_auth_info, # but as a sanity check... raise AccessDenied() for key, value in sorted(amz_headers.items()): - buf.append("%s:%s" % (key, value)) + buf.append(swob.wsgi_to_bytes("%s:%s" % (key, value))) path = self._canonical_uri() if self.query_string: @@ -971,10 +980,10 @@ class S3Request(swob.Request): if key in ALLOWED_SUB_RESOURCES: params.append('%s=%s' % (key, value) if value else key) if params: - buf.append('%s?%s' % (path, '&'.join(params))) + buf.append(swob.wsgi_to_bytes('%s?%s' % (path, '&'.join(params)))) else: - buf.append(path) - return '\n'.join(buf) + buf.append(swob.wsgi_to_bytes(path)) + return b'\n'.join(buf) def signature_does_not_match_kwargs(self): return { @@ -982,7 +991,8 @@ class S3Request(swob.Request): 'string_to_sign': self.string_to_sign, 'signature_provided': self.signature, 'string_to_sign_bytes': ' '.join( - format(ord(c), '02x') for c in self.string_to_sign), + format(ord(c), '02x') + for c in self.string_to_sign.decode('latin1')), } @property @@ -1320,7 +1330,6 @@ class S3Request(swob.Request): # reuse account and tokens _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'], 2, 3, True) - self.account = utf8encode(self.account) resp = S3Response.from_swift_resp(sw_resp) status = resp.status_int # pylint: disable-msg=E1101 @@ -1354,7 +1363,7 @@ class S3Request(swob.Request): raise err_resp() if status == HTTP_BAD_REQUEST: - raise BadSwiftRequest(err_msg) + raise BadSwiftRequest(err_msg.decode('utf8')) if status == HTTP_UNAUTHORIZED: raise SignatureDoesNotMatch( **self.signature_does_not_match_kwargs()) @@ -1487,7 +1496,6 @@ class S3AclRequest(S3Request): _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'], 2, 3, True) - self.account = utf8encode(self.account) if 'HTTP_X_USER_NAME' in sw_resp.environ: # keystone diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py index 9e6a759bd7..b71efa7f18 100644 --- a/swift/common/middleware/s3api/s3response.py +++ b/swift/common/middleware/s3api/s3response.py @@ -243,8 +243,10 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException): if isinstance(value, (dict, MutableMapping)): self._dict_to_etree(elem, value) else: + if isinstance(value, (int, float, bool)): + value = str(value) try: - elem.text = str(value) + elem.text = value except ValueError: # We set an invalid string for XML. elem.text = '(invalid string)' diff --git a/swift/common/middleware/s3api/subresource.py b/swift/common/middleware/s3api/subresource.py index 42bd67f003..c85d0b7b33 100644 --- a/swift/common/middleware/s3api/subresource.py +++ b/swift/common/middleware/s3api/subresource.py @@ -43,6 +43,8 @@ http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html """ from functools import partial +import six + from swift.common.utils import json from swift.common.middleware.s3api.s3response import InvalidArgument, \ @@ -218,6 +220,11 @@ class User(Grantee): def __str__(self): return self.display_name + def __lt__(self, other): + if not isinstance(other, User): + return NotImplemented + return self.id < other.id + class Owner(object): """ @@ -415,9 +422,14 @@ class ACL(object): self.s3_acl = s3_acl self.allow_no_owner = allow_no_owner - def __repr__(self): + def __bytes__(self): return tostring(self.elem()) + def __repr__(self): + if six.PY2: + return self.__bytes__() + return self.__bytes__().decode('utf8') + @classmethod def from_elem(cls, elem, s3_acl=False, allow_no_owner=False): """ diff --git a/swift/common/swob.py b/swift/common/swob.py index 71b97ff019..8149c451ca 100644 --- a/swift/common/swob.py +++ b/swift/common/swob.py @@ -309,15 +309,15 @@ def str_to_wsgi(native_str): return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape')) -def wsgi_quote(wsgi_str): +def wsgi_quote(wsgi_str, safe='/'): if six.PY2: if not isinstance(wsgi_str, bytes): raise TypeError('Expected a WSGI string; got %r' % wsgi_str) - return urllib.parse.quote(wsgi_str) + return urllib.parse.quote(wsgi_str, safe=safe) if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str): raise TypeError('Expected a WSGI string; got %r' % wsgi_str) - return urllib.parse.quote(wsgi_str, encoding='latin-1') + return urllib.parse.quote(wsgi_str, safe=safe, encoding='latin-1') def wsgi_unquote(wsgi_str): @@ -478,6 +478,10 @@ def _resp_app_iter_property(): def setter(self, value): if isinstance(value, (list, tuple)): + for i, item in enumerate(value): + if not isinstance(item, bytes): + raise TypeError('WSGI responses must be bytes; ' + 'got %s for item %d' % (type(item), i)) self.content_length = sum(map(len, value)) elif value is not None: self.content_length = None diff --git a/test/unit/common/middleware/s3api/helpers.py b/test/unit/common/middleware/s3api/helpers.py index 54051889a0..525dc35111 100644 --- a/test/unit/common/middleware/s3api/helpers.py +++ b/test/unit/common/middleware/s3api/helpers.py @@ -165,12 +165,16 @@ class FakeSwift(object): # keep old sysmeta for s3acl headers.update({key: value}) + if body is not None and not isinstance(body, (bytes, list)): + body = body.encode('utf8') self._responses[(method, path)] = (response_class, headers, body) def register_unconditionally(self, method, path, response_class, headers, body): # register() keeps old sysmeta around, but # register_unconditionally() keeps nothing. + if body is not None and not isinstance(body, bytes): + body = body.encode('utf8') self._responses[(method, path)] = (response_class, headers, body) def clear_calls(self): diff --git a/test/unit/common/middleware/s3api/test_acl.py b/test/unit/common/middleware/s3api/test_acl.py index f7e800e5da..9037b32c88 100644 --- a/test/unit/common/middleware/s3api/test_acl.py +++ b/test/unit/common/middleware/s3api/test_acl.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 import unittest import mock @@ -131,8 +132,8 @@ class TestS3ApiAcl(S3ApiTestCase): 'UnexpectedContent') def _test_put_no_body(self, use_content_length=False, - use_transfer_encoding=False, string_to_md5=''): - content_md5 = md5(string_to_md5).digest().encode('base64').strip() + use_transfer_encoding=False, string_to_md5=b''): + content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip() with UnreadableInput(self) as fake_input: req = Request.blank( '/bucket?acl', @@ -153,16 +154,17 @@ class TestS3ApiAcl(S3ApiTestCase): self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader') self.assertEqual(self._get_error_message(body), 'Your request was missing a required header.') - self.assertIn('x-amz-acl', body) + self.assertIn(b'x-amz-acl', + body) @s3acl def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self): self._test_put_no_body() - self._test_put_no_body(string_to_md5='test') + self._test_put_no_body(string_to_md5=b'test') self._test_put_no_body(use_content_length=True) - self._test_put_no_body(use_content_length=True, string_to_md5='test') + self._test_put_no_body(use_content_length=True, string_to_md5=b'test') self._test_put_no_body(use_transfer_encoding=True) - self._test_put_no_body(use_transfer_encoding=True, string_to_md5='zz') + self._test_put_no_body(use_transfer_encoding=True, string_to_md5=b'zz') def test_object_acl_GET(self): req = Request.blank('/bucket/object?acl', diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index 85dd7437cd..d530596e88 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -17,6 +17,7 @@ import unittest import cgi import mock +import six from six.moves.urllib.parse import quote from swift.common import swob @@ -62,7 +63,8 @@ class TestS3ApiBucket(S3ApiTestCase): for name, _, _, _ in self.objects: self.swift.register( 'DELETE', - '/v1/AUTH_test/bucket+segments/' + name.encode('utf-8'), + '/v1/AUTH_test/bucket+segments/' + + swob.bytes_to_wsgi(name.encode('utf-8')), swob.HTTPNoContent, {}, json.dumps([])) self.swift.register( 'GET', @@ -118,7 +120,7 @@ class TestS3ApiBucket(S3ApiTestCase): 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '404') - self.assertEqual(body, '') # sanity + self.assertEqual(body, b'') # sanity def test_bucket_HEAD_slash(self): req = Request.blank('/junk/', @@ -168,7 +170,8 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual('2011-01-05T02:19:14.275Z', o.find('./LastModified').text) self.assertEqual(items, [ - (i[0].encode('utf-8'), '"0-N"' if i[0] == 'slo' else '"0"') + (i[0].encode('utf-8') if six.PY2 else i[0], + '"0-N"' if i[0] == 'slo' else '"0"') for i in self.objects]) def test_bucket_GET_url_encoded(self): @@ -519,8 +522,12 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual(elem.findall('./DeleteMarker'), []) versions = elem.findall('./Version') objects = list(self.objects) - self.assertEqual([v.find('./Key').text for v in versions], - [v[0].encode('utf-8') for v in objects]) + if six.PY2: + self.assertEqual([v.find('./Key').text for v in versions], + [v[0].encode('utf-8') for v in objects]) + else: + self.assertEqual([v.find('./Key').text for v in versions], + [v[0] for v in objects]) self.assertEqual([v.find('./IsLatest').text for v in versions], ['true' for v in objects]) self.assertEqual([v.find('./VersionId').text for v in versions], @@ -598,7 +605,7 @@ class TestS3ApiBucket(S3ApiTestCase): headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) - self.assertEqual(body, '') + self.assertEqual(body, b'') self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Location'], '/bucket') @@ -610,7 +617,7 @@ class TestS3ApiBucket(S3ApiTestCase): 'Date': self.get_date_header(), 'Transfer-Encoding': 'chunked'}) status, headers, body = self.call_s3api(req) - self.assertEqual(body, '') + self.assertEqual(body, b'') self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Location'], '/bucket') @@ -622,7 +629,7 @@ class TestS3ApiBucket(S3ApiTestCase): headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) - self.assertEqual(body, '') + self.assertEqual(body, b'') self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Location'], '/bucket') diff --git a/test/unit/common/middleware/s3api/test_etree.py b/test/unit/common/middleware/s3api/test_etree.py index be2249ae02..63cb5c7ffc 100644 --- a/test/unit/common/middleware/s3api/test_etree.py +++ b/test/unit/common/middleware/s3api/test_etree.py @@ -15,6 +15,8 @@ import unittest +import six + from swift.common.middleware.s3api import etree @@ -58,15 +60,18 @@ class TestS3ApiEtree(unittest.TestCase): sub.text = '\xef\xbc\xa1' self.assertTrue(isinstance(sub.text, str)) xml_string = etree.tostring(elem) - self.assertTrue(isinstance(xml_string, str)) + self.assertIsInstance(xml_string, bytes) def test_fromstring_with_nonascii_text(self): - input_str = '\n' \ - '\xef\xbc\xa1' + input_str = b'\n' \ + b'\xef\xbc\xa1' elem = etree.fromstring(input_str) text = elem.find('FOO').text - self.assertEqual(text, '\xef\xbc\xa1') - self.assertTrue(isinstance(text, str)) + if six.PY2: + self.assertEqual(text, b'\xef\xbc\xa1') + else: + self.assertEqual(text, b'\xef\xbc\xa1'.decode('utf8')) + self.assertIsInstance(text, str) if __name__ == '__main__': diff --git a/test/unit/common/middleware/s3api/test_multi_delete.py b/test/unit/common/middleware/s3api/test_multi_delete.py index c8bb7206f6..6ea5f59003 100644 --- a/test/unit/common/middleware/s3api/test_multi_delete.py +++ b/test/unit/common/middleware/s3api/test_multi_delete.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import base64 import json import unittest from datetime import datetime @@ -43,7 +44,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = 'object' body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket/object?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -80,7 +81,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -133,7 +134,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -180,7 +181,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -207,7 +208,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key') body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -232,7 +233,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): SubElement(obj, 'Key').text = key SubElement(obj, 'VersionId').text = 'not-supported' body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -286,7 +287,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = name body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -308,7 +309,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = 'x' * 1000 + str(i) body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -333,7 +334,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key body = tostring(elem, use_s3ns=False) - content_md5 = md5(body).digest().encode('base64').strip() + content_md5 = base64.b64encode(md5(body).digest()).strip() req = Request.blank('/bucket?delete', environ={'REQUEST_METHOD': 'POST'}, @@ -374,8 +375,8 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertEqual(len(elem.findall('Deleted')), len(self.keys)) def _test_no_body(self, use_content_length=False, - use_transfer_encoding=False, string_to_md5=''): - content_md5 = md5(string_to_md5).digest().encode('base64').strip() + use_transfer_encoding=False, string_to_md5=b''): + content_md5 = base64.b64encode(md5(string_to_md5).digest()).strip() with UnreadableInput(self) as fake_input: req = Request.blank( '/bucket?delete', @@ -398,11 +399,11 @@ class TestS3ApiMultiDelete(S3ApiTestCase): @s3acl def test_object_multi_DELETE_empty_body(self): self._test_no_body() - self._test_no_body(string_to_md5='test') + self._test_no_body(string_to_md5=b'test') self._test_no_body(use_content_length=True) - self._test_no_body(use_content_length=True, string_to_md5='test') + self._test_no_body(use_content_length=True, string_to_md5=b'test') self._test_no_body(use_transfer_encoding=True) - self._test_no_body(use_transfer_encoding=True, string_to_md5='test') + self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test') if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py index 2dc3d8db61..cff18c2759 100644 --- a/test/unit/common/middleware/s3api/test_multi_upload.py +++ b/test/unit/common/middleware/s3api/test_multi_upload.py @@ -14,12 +14,13 @@ # limitations under the License. import base64 +import binascii import hashlib from mock import patch import os import time import unittest -from urllib import quote +from six.moves.urllib.parse import quote from swift.common import swob from swift.common.swob import Request @@ -66,9 +67,9 @@ MULTIPARTS_TEMPLATE = \ ('subdir/object/Z/2', '2014-05-07T19:47:58.592270', 'fedcba9876543210', 41)) -S3_ETAG = '"%s-2"' % hashlib.md5(( +S3_ETAG = '"%s-2"' % hashlib.md5(binascii.a2b_hex( '0123456789abcdef0123456789abcdef' - 'fedcba9876543210fedcba9876543210').decode('hex')).hexdigest() + 'fedcba9876543210fedcba9876543210')).hexdigest() class TestS3ApiMultiUpload(S3ApiTestCase): @@ -83,9 +84,9 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.s3api.conf.min_segment_size = 1 - objects = map(lambda item: {'name': item[0], 'last_modified': item[1], - 'hash': item[2], 'bytes': item[3]}, - OBJECTS_TEMPLATE) + objects = [{'name': item[0], 'last_modified': item[1], + 'hash': item[2], 'bytes': item[3]} + for item in OBJECTS_TEMPLATE] object_list = json.dumps(objects) self.swift.register('PUT', segment_bucket, @@ -172,10 +173,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase): multiparts=None): segment_bucket = '/v1/AUTH_test/bucket+segments' objects = multiparts or MULTIPARTS_TEMPLATE - objects = map(lambda item: {'name': item[0], 'last_modified': item[1], - 'hash': item[2], 'bytes': item[3]}, - objects) - object_list = json.dumps(objects) + objects = [{'name': item[0], 'last_modified': item[1], + 'hash': item[2], 'bytes': item[3]} + for item in objects] + object_list = json.dumps(objects).encode('ascii') self.swift.register('GET', segment_bucket, swob.HTTPOk, {}, object_list) @@ -568,7 +569,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self._test_object_multipart_upload_initiate({}) self._test_object_multipart_upload_initiate({'Etag': 'blahblahblah'}) self._test_object_multipart_upload_initiate({ - 'Content-MD5': base64.b64encode('blahblahblahblah').strip()}) + 'Content-MD5': base64.b64encode(b'blahblahblahblah').strip()}) @s3acl(s3acl_only=True) @patch('swift.common.middleware.s3api.controllers.multi_upload.' @@ -667,7 +668,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(self._get_error_code(body), 'NoSuchBucket') def test_object_multipart_upload_complete(self): - content_md5 = base64.b64encode(hashlib.md5(XML).digest()) + content_md5 = base64.b64encode(hashlib.md5( + XML.encode('ascii')).digest()) req = Request.blank('/bucket/object?uploadId=X', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', @@ -701,7 +703,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(headers.get(h), override_etag) def test_object_multipart_upload_invalid_md5(self): - bad_md5 = base64.b64encode(hashlib.md5(XML + 'some junk').digest()) + bad_md5 = base64.b64encode(hashlib.md5( + XML.encode('ascii') + b'some junk').digest()) req = Request.blank('/bucket/object?uploadId=X', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', @@ -726,11 +729,11 @@ class TestS3ApiMultiUpload(S3ApiTestCase): ])) self.swift.register( 'PUT', '/v1/AUTH_test/bucket/heartbeat-ok', - swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({ + swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({ 'Etag': '"slo-etag"', 'Response Status': '201 Created', 'Errors': [], - })]) + }).encode('ascii')]) mock_time.time.side_effect = ( 1, # start_time 12, # first whitespace @@ -748,14 +751,14 @@ class TestS3ApiMultiUpload(S3ApiTestCase): 'Date': self.get_date_header(), }, body=XML) status, headers, body = self.call_s3api(req) - lines = body.split('\n') - self.assertTrue(lines[0].startswith('%s' % S3_ETAG, body) + self.assertIn(('%s' % S3_ETAG).encode('ascii'), body) self.assertEqual(self.swift.calls, [ ('HEAD', '/v1/AUTH_test/bucket'), ('HEAD', '/v1/AUTH_test/bucket+segments/heartbeat-ok/X'), @@ -779,10 +782,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase): ])) self.swift.register( 'PUT', '/v1/AUTH_test/bucket/heartbeat-fail', - swob.HTTPAccepted, {}, [' ', ' ', ' ', json.dumps({ + swob.HTTPAccepted, {}, [b' ', b' ', b' ', json.dumps({ 'Response Status': '400 Bad Request', 'Errors': [['some/object', '403 Forbidden']], - })]) + }).encode('ascii')]) mock_time.time.side_effect = ( 1, # start_time 12, # first whitespace @@ -797,8 +800,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase): 'Date': self.get_date_header(), }, body=XML) status, headers, body = self.call_s3api(req) - lines = body.split('\n') - self.assertTrue(lines[0].startswith('us-east-1') + b'us-east-1') test('test:tester/%s/us-east-1/not-s3/aws4_request' % dt, 'Error parsing the X-Amz-Credential parameter; incorrect service ' '"not-s3". This endpoint belongs to "s3".') @@ -483,9 +485,9 @@ class TestS3ApiMiddleware(S3ApiTestCase): self.assertEqual(req.environ['s3api.auth_details'], { 'access_key': 'test:tester', 'signature': 'hmac', - 'string_to_sign': '\n'.join([ - 'PUT', '', '', date_header, - '/bucket/object?partNumber=1&uploadId=123456789abcdef']), + 'string_to_sign': b'\n'.join([ + b'PUT', b'', b'', date_header.encode('ascii'), + b'/bucket/object?partNumber=1&uploadId=123456789abcdef']), 'check_signature': mock_cs}) def test_invalid_uri(self): @@ -506,8 +508,10 @@ class TestS3ApiMiddleware(S3ApiTestCase): self.assertEqual(self._get_error_code(body), 'InvalidDigest') def test_object_create_bad_md5_too_short(self): - too_short_digest = hashlib.md5('hey').hexdigest()[:-1] - md5_str = too_short_digest.encode('base64').strip() + too_short_digest = hashlib.md5(b'hey').digest()[:-1] + md5_str = base64.b64encode(too_short_digest).strip() + if not six.PY2: + md5_str = md5_str.decode('ascii') req = Request.blank( '/bucket/object', environ={'REQUEST_METHOD': 'PUT', @@ -518,8 +522,10 @@ class TestS3ApiMiddleware(S3ApiTestCase): self.assertEqual(self._get_error_code(body), 'InvalidDigest') def test_object_create_bad_md5_too_long(self): - too_long_digest = hashlib.md5('hey').hexdigest() + 'suffix' - md5_str = too_long_digest.encode('base64').strip() + too_long_digest = hashlib.md5(b'hey').digest() + b'suffix' + md5_str = base64.b64encode(too_long_digest).strip() + if not six.PY2: + md5_str = md5_str.decode('ascii') req = Request.blank( '/bucket/object', environ={'REQUEST_METHOD': 'PUT', @@ -705,13 +711,13 @@ class TestS3ApiMiddleware(S3ApiTestCase): with self.assertRaises(ValueError) as cm: self.s3api.check_pipeline(self.conf) self.assertIn('expected auth between s3api and proxy-server', - cm.exception.message) + cm.exception.args[0]) pipeline.return_value = 'proxy-server' with self.assertRaises(ValueError) as cm: self.s3api.check_pipeline(self.conf) self.assertIn("missing filters ['s3api']", - cm.exception.message) + cm.exception.args[0]) def test_s3api_initialization_with_disabled_pipeline_check(self): with patch("swift.common.middleware.s3api.s3api.loadcontext"), \ @@ -799,7 +805,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): 'Missing required header for this request: x-amz-content-sha256') def test_signature_v4_bad_authorization_string(self): - def test(auth_str, error, msg, extra=''): + def test(auth_str, error, msg, extra=b''): environ = { 'REQUEST_METHOD': 'GET'} headers = { @@ -835,7 +841,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): test(auth_str, 'AuthorizationHeaderMalformed', "The authorization header is malformed; " "the region 'us-west-2' is wrong; expecting 'us-east-1'", - 'us-east-1') + b'us-east-1') auth_str = ('AWS4-HMAC-SHA256 ' 'Credential=test:tester/%s/us-east-1/not-s3/aws4_request, ' @@ -901,8 +907,8 @@ class TestS3ApiMiddleware(S3ApiTestCase): patch.object(swift.common.middleware.s3api.s3request, 'SERVICE', 'host'): req = _get_req(path, environ) - hash_in_sts = req._string_to_sign().split('\n')[3] - self.assertEqual(hash_val, hash_in_sts) + hash_in_sts = req._string_to_sign().split(b'\n')[3] + self.assertEqual(hash_val, hash_in_sts.decode('ascii')) self.assertTrue(req.check_signature( 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY')) @@ -963,7 +969,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): 'validate_bucket_name'): verify('27ba31df5dbc6e063d8f87d62eb07143' 'f7f271c5330a917840586ac1c85b6f6b', - unquote('/%E1%88%B4'), env) + swob.wsgi_unquote('/%E1%88%B4'), env) # get-vanilla-query-order-key env = { @@ -1101,12 +1107,12 @@ class TestS3ApiMiddleware(S3ApiTestCase): swob.HTTPOk, {}, None) with patch.object(self.s3_token, '_json_request') as mock_req: mock_resp = requests.Response() - mock_resp._content = json.dumps(GOOD_RESPONSE_V2) + mock_resp._content = json.dumps(GOOD_RESPONSE_V2).encode('ascii') mock_resp.status_code = 201 mock_req.return_value = mock_resp status, headers, body = self.call_s3api(req) - self.assertEqual(body, '') + self.assertEqual(body, b'') self.assertEqual(1, mock_req.call_count) def test_s3api_with_only_s3_token_v3(self): @@ -1127,12 +1133,12 @@ class TestS3ApiMiddleware(S3ApiTestCase): swob.HTTPOk, {}, None) with patch.object(self.s3_token, '_json_request') as mock_req: mock_resp = requests.Response() - mock_resp._content = json.dumps(GOOD_RESPONSE_V3) + mock_resp._content = json.dumps(GOOD_RESPONSE_V3).encode('ascii') mock_resp.status_code = 200 mock_req.return_value = mock_resp status, headers, body = self.call_s3api(req) - self.assertEqual(body, '') + self.assertEqual(body, b'') self.assertEqual(1, mock_req.call_count) def test_s3api_with_s3_token_and_auth_token(self): @@ -1157,7 +1163,8 @@ class TestS3ApiMiddleware(S3ApiTestCase): with patch.object(self.auth_token, '_do_fetch_token') as mock_fetch: mock_resp = requests.Response() - mock_resp._content = json.dumps(GOOD_RESPONSE_V2) + mock_resp._content = json.dumps( + GOOD_RESPONSE_V2).encode('ascii') mock_resp.status_code = 201 mock_req.return_value = mock_resp @@ -1167,7 +1174,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): mock_fetch.return_value = (MagicMock(), mock_access_info) status, headers, body = self.call_s3api(req) - self.assertEqual(body, '') + self.assertEqual(body, b'') self.assertEqual(1, mock_req.call_count) # With X-Auth-Token, auth_token will call _do_fetch_token to # connect to keystone in auth_token, again @@ -1198,7 +1205,8 @@ class TestS3ApiMiddleware(S3ApiTestCase): no_token_id_good_resp = copy.deepcopy(GOOD_RESPONSE_V2) # delete token id del no_token_id_good_resp['access']['token']['id'] - mock_resp._content = json.dumps(no_token_id_good_resp) + mock_resp._content = json.dumps( + no_token_id_good_resp).encode('ascii') mock_resp.status_code = 201 mock_req.return_value = mock_resp diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py index 3cec613586..f9e4acbf53 100644 --- a/test/unit/common/middleware/s3api/test_s3request.py +++ b/test/unit/common/middleware/s3api/test_s3request.py @@ -143,7 +143,7 @@ class TestRequest(S3ApiTestCase): def test_get_response_without_match_ACL_MAP(self): with self.assertRaises(Exception) as e: self._test_get_response('POST', req_klass=S3AclRequest) - self.assertEqual(e.exception.message, + self.assertEqual(e.exception.args[0], 'No permission to be checked exists') def test_get_response_without_duplication_HEAD_request(self): @@ -215,8 +215,8 @@ class TestRequest(S3ApiTestCase): s3req = create_s3request_with_param('max-keys', '1' * 30) with self.assertRaises(InvalidArgument) as result: s3req.get_validated_param('max-keys', 1) - self.assertTrue( - 'not an integer or within integer range' in result.exception.body) + self.assertIn( + b'not an integer or within integer range', result.exception.body) self.assertEqual( result.exception.headers['content-type'], 'application/xml') @@ -224,8 +224,8 @@ class TestRequest(S3ApiTestCase): s3req = create_s3request_with_param('max-keys', '-1') with self.assertRaises(InvalidArgument) as result: s3req.get_validated_param('max-keys', 1) - self.assertTrue( - 'must be an integer between 0 and' in result.exception.body) + self.assertIn( + b'must be an integer between 0 and', result.exception.body) self.assertEqual( result.exception.headers['content-type'], 'application/xml') @@ -233,8 +233,8 @@ class TestRequest(S3ApiTestCase): s3req = create_s3request_with_param('max-keys', 'invalid') with self.assertRaises(InvalidArgument) as result: s3req.get_validated_param('max-keys', 1) - self.assertTrue( - 'not an integer or within integer range' in result.exception.body) + self.assertIn( + b'not an integer or within integer range', result.exception.body) self.assertEqual( result.exception.headers['content-type'], 'application/xml') @@ -351,7 +351,7 @@ class TestRequest(S3ApiTestCase): headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '403') - self.assertEqual(body, '') + self.assertEqual(body, b'') def test_date_header_expired(self): self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, @@ -363,7 +363,7 @@ class TestRequest(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '403') - self.assertEqual(body, '') + self.assertEqual(body, b'') def test_date_header_with_x_amz_date_valid(self): self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, @@ -376,7 +376,7 @@ class TestRequest(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '404') - self.assertEqual(body, '') + self.assertEqual(body, b'') def test_date_header_with_x_amz_date_expired(self): self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, @@ -390,7 +390,7 @@ class TestRequest(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '403') - self.assertEqual(body, '') + self.assertEqual(body, b'') def _test_request_timestamp_sigv4(self, date_header): # signature v4 here @@ -428,7 +428,7 @@ class TestRequest(S3ApiTestCase): def test_request_timestamp_sigv4(self): access_denied_message = \ - 'AWS authentication requires a valid Date or x-amz-date header' + b'AWS authentication requires a valid Date or x-amz-date header' # normal X-Amz-Date header date_header = {'X-Amz-Date': self.get_v4_amz_date_header()} @@ -443,7 +443,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv4(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # mangled Date header @@ -451,7 +451,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv4(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # Negative timestamp @@ -459,7 +459,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv4(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # far-past Date header @@ -467,7 +467,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv4(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # near-future X-Amz-Date header @@ -481,9 +481,9 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(RequestTimeTooSkewed) as cm: self._test_request_timestamp_sigv4(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) - self.assertIn('The difference between the request time and the ' - 'current time is too large.', cm.exception.body) + self.assertEqual('403 Forbidden', cm.exception.args[0]) + self.assertIn(b'The difference between the request time and the ' + b'current time is too large.', cm.exception.body) def _test_request_timestamp_sigv2(self, date_header): # signature v4 here @@ -505,7 +505,7 @@ class TestRequest(S3ApiTestCase): def test_request_timestamp_sigv2(self): access_denied_message = \ - 'AWS authentication requires a valid Date or x-amz-date header' + b'AWS authentication requires a valid Date or x-amz-date header' # In v2 format, normal X-Amz-Date header is same date_header = {'X-Amz-Date': self.get_date_header()} @@ -520,7 +520,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv2(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # mangled Date header @@ -528,7 +528,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv2(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # Negative timestamp @@ -536,7 +536,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv2(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # far-past Date header @@ -544,7 +544,7 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(AccessDenied) as cm: self._test_request_timestamp_sigv2(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) + self.assertEqual('403 Forbidden', cm.exception.args[0]) self.assertIn(access_denied_message, cm.exception.body) # far-future Date header @@ -552,9 +552,9 @@ class TestRequest(S3ApiTestCase): with self.assertRaises(RequestTimeTooSkewed) as cm: self._test_request_timestamp_sigv2(date_header) - self.assertEqual('403 Forbidden', cm.exception.message) - self.assertIn('The difference between the request time and the ' - 'current time is too large.', cm.exception.body) + self.assertEqual('403 Forbidden', cm.exception.args[0]) + self.assertIn(b'The difference between the request time and the ' + b'current time is too large.', cm.exception.body) def test_headers_to_sign_sigv4(self): environ = { @@ -681,14 +681,14 @@ class TestRequest(S3ApiTestCase): sigv4_req = SigV4Request(req.environ) uri = sigv4_req._canonical_uri() - self.assertEqual(uri, '/') + self.assertEqual(uri, b'/') self.assertEqual(req.environ['PATH_INFO'], '/') req = Request.blank('/obj1', environ=environ, headers=headers) sigv4_req = SigV4Request(req.environ) uri = sigv4_req._canonical_uri() - self.assertEqual(uri, '/obj1') + self.assertEqual(uri, b'/obj1') self.assertEqual(req.environ['PATH_INFO'], '/obj1') environ = { @@ -701,7 +701,7 @@ class TestRequest(S3ApiTestCase): sigv4_req = SigV4Request(req.environ) uri = sigv4_req._canonical_uri() - self.assertEqual(uri, '/') + self.assertEqual(uri, b'/') self.assertEqual(req.environ['PATH_INFO'], '/') req = Request.blank('/bucket/obj1', @@ -710,7 +710,7 @@ class TestRequest(S3ApiTestCase): sigv4_req = SigV4Request(req.environ) uri = sigv4_req._canonical_uri() - self.assertEqual(uri, '/bucket/obj1') + self.assertEqual(uri, b'/bucket/obj1') self.assertEqual(req.environ['PATH_INFO'], '/bucket/obj1') @patch.object(S3Request, '_validate_dates', lambda *a: None) @@ -724,12 +724,12 @@ class TestRequest(S3ApiTestCase): 'bWq2s1WEIj+Ydj0vQ697zp+IXMU='), }) sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com') - expected_sts = '\n'.join([ - 'GET', - '', - '', - 'Tue, 27 Mar 2007 19:36:42 +0000', - '/johnsmith/photos/puppy.jpg', + expected_sts = b'\n'.join([ + b'GET', + b'', + b'', + b'Tue, 27 Mar 2007 19:36:42 +0000', + b'/johnsmith/photos/puppy.jpg', ]) self.assertEqual(expected_sts, sigv2_req._string_to_sign()) self.assertTrue(sigv2_req.check_signature(secret)) @@ -743,12 +743,12 @@ class TestRequest(S3ApiTestCase): 'MyyxeRY7whkBe+bq8fHCL/2kKUg='), }) sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com') - expected_sts = '\n'.join([ - 'PUT', - '', - 'image/jpeg', - 'Tue, 27 Mar 2007 21:15:45 +0000', - '/johnsmith/photos/puppy.jpg', + expected_sts = b'\n'.join([ + b'PUT', + b'', + b'image/jpeg', + b'Tue, 27 Mar 2007 21:15:45 +0000', + b'/johnsmith/photos/puppy.jpg', ]) self.assertEqual(expected_sts, sigv2_req._string_to_sign()) self.assertTrue(sigv2_req.check_signature(secret)) @@ -763,12 +763,12 @@ class TestRequest(S3ApiTestCase): 'htDYFYduRNen8P9ZfE/s9SuKy0U='), }) sigv2_req = S3Request(req.environ, storage_domain='s3.amazonaws.com') - expected_sts = '\n'.join([ - 'GET', - '', - '', - 'Tue, 27 Mar 2007 19:42:41 +0000', - '/johnsmith/', + expected_sts = b'\n'.join([ + b'GET', + b'', + b'', + b'Tue, 27 Mar 2007 19:42:41 +0000', + b'/johnsmith/', ]) self.assertEqual(expected_sts, sigv2_req._string_to_sign()) self.assertTrue(sigv2_req.check_signature(secret)) @@ -846,7 +846,7 @@ class TestHashingInput(S3ApiTestCase): self.assertEqual(b'1234', wrapped.read(4)) self.assertEqual(b'56', wrapped.read(2)) # even though the hash matches, there was more data than we expected - with self.assertRaises(swob.Response) as raised: + with self.assertRaises(swob.HTTPException) as raised: wrapped.read(3) self.assertEqual(raised.exception.status, '422 Unprocessable Entity') # the error causes us to close the input @@ -859,7 +859,7 @@ class TestHashingInput(S3ApiTestCase): self.assertEqual(b'1234', wrapped.read(4)) self.assertEqual(b'56', wrapped.read(2)) # even though the hash matches, there was more data than we expected - with self.assertRaises(swob.Response) as raised: + with self.assertRaises(swob.HTTPException) as raised: wrapped.read(4) self.assertEqual(raised.exception.status, '422 Unprocessable Entity') self.assertTrue(wrapped._input.closed) @@ -870,14 +870,14 @@ class TestHashingInput(S3ApiTestCase): hashlib.md5(raw).hexdigest()) self.assertEqual(b'1234', wrapped.read(4)) self.assertEqual(b'5678', wrapped.read(4)) - with self.assertRaises(swob.Response) as raised: + with self.assertRaises(swob.HTTPException) as raised: wrapped.read(4) self.assertEqual(raised.exception.status, '422 Unprocessable Entity') self.assertTrue(wrapped._input.closed) def test_empty_bad_hash(self): wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope') - with self.assertRaises(swob.Response) as raised: + with self.assertRaises(swob.HTTPException) as raised: wrapped.read(3) self.assertEqual(raised.exception.status, '422 Unprocessable Entity') # the error causes us to close the input diff --git a/test/unit/common/middleware/s3api/test_service.py b/test/unit/common/middleware/s3api/test_service.py index 2f8286ba8b..f32046052a 100644 --- a/test/unit/common/middleware/s3api/test_service.py +++ b/test/unit/common/middleware/s3api/test_service.py @@ -135,7 +135,7 @@ class TestS3ApiService(S3ApiTestCase): self.assertEqual(len(names), len(expected)) for i in expected: - self.assertTrue(i[0] in names) + self.assertIn(i[0], names) def _test_service_GET_for_check_bucket_owner(self, buckets): self.s3api.conf.check_bucket_owner = True diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index 22f88f25ac..96d02401fa 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -1289,7 +1289,7 @@ class TestInternalClient(unittest.TestCase): def fake_app(self, env, start_response): start_response('404 Not Found', []) - return ['one\ntwo\nthree'] + return [b'one\ntwo\nthree'] client = InternalClient() lines = [] diff --git a/test/unit/common/test_swob.py b/test/unit/common/test_swob.py index f97941fa8d..8c57516c45 100644 --- a/test/unit/common/test_swob.py +++ b/test/unit/common/test_swob.py @@ -670,7 +670,7 @@ class TestRequest(unittest.TestCase): def test_app(environ, start_response): start_response('401 Unauthorized', []) - return ['hi'] + return [b'hi'] # Request environment contains valid account in path req = swift.common.swob.Request.blank('/v1/account-name') @@ -692,7 +692,7 @@ class TestRequest(unittest.TestCase): def test_app(environ, start_response): start_response('401 Unauthorized', []) - return ['hi'] + return [b'hi'] # Request environment contains bad path req = swift.common.swob.Request.blank('/random') @@ -706,7 +706,7 @@ class TestRequest(unittest.TestCase): def test_app(environ, start_response): start_response('401 Unauthorized', []) - return ['no creds in request'] + return [b'no creds in request'] # Request to get token req = swift.common.swob.Request.blank('/v1.0/auth') @@ -729,7 +729,7 @@ class TestRequest(unittest.TestCase): def test_app(environ, start_response): start_response('401 Unauthorized', { 'Www-Authenticate': 'Me realm="whatever"'}) - return ['no creds in request'] + return [b'no creds in request'] # Auth middleware sets own Www-Authenticate req = swift.common.swob.Request.blank('/auth/v1.0') @@ -743,7 +743,7 @@ class TestRequest(unittest.TestCase): def test_app(environ, start_response): start_response('401 Unauthorized', []) - return ['hi'] + return [b'hi'] hacker = 'account-name\n\nfoo
' # url injection test quoted_hacker = quote(hacker) @@ -766,7 +766,7 @@ class TestRequest(unittest.TestCase): # Other status codes should not have WWW-Authenticate in response def test_app(environ, start_response): start_response('200 OK', []) - return ['hi'] + return [b'hi'] req = swift.common.swob.Request.blank('/') resp = req.get_response(test_app) @@ -1763,7 +1763,7 @@ class TestConditionalIfMatch(unittest.TestCase): def fake_app_404(environ, start_response): start_response('404 Not Found', []) - return ['hi'] + return [b'hi'] req = swift.common.swob.Request.blank( '/', headers={'If-Match': '*'}) diff --git a/tox.ini b/tox.ini index 5930733b2d..1ee5fc9536 100644 --- a/tox.ini +++ b/tox.ini @@ -40,7 +40,7 @@ commands = test/unit/account \ test/unit/cli \ test/unit/common/middleware/crypto \ - test/unit/common/middleware/s3api/test_s3token.py \ + test/unit/common/middleware/s3api/ \ test/unit/common/middleware/test_account_quotas.py \ test/unit/common/middleware/test_acl.py \ test/unit/common/middleware/test_catch_errors.py \