From b07d87c4be48ee28b936b7fa14ed3d5f20efabf4 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 4 Dec 2023 11:38:17 +0000 Subject: [PATCH] tests: use subclasses for S3Acl tests We remove s3api.FakeSwift and replace it with the "normal" FakeSwift. Additionally the @s3acl decorator is removed and replaced with an inheritance based pattern. This simplifies maintenance using more familiar patterns and improves debugging. Co-Authored-By: Clay Gerrard Change-Id: I55b596a42af01870b49fda22800f7a1293163eb8 --- test/unit/common/middleware/helpers.py | 37 +- test/unit/common/middleware/s3api/__init__.py | 160 +- test/unit/common/middleware/s3api/helpers.py | 81 - test/unit/common/middleware/s3api/test_acl.py | 152 +- .../common/middleware/s3api/test_bucket.py | 676 ++++---- .../common/middleware/s3api/test_helpers.py | 69 - .../middleware/s3api/test_multi_delete.py | 200 ++- .../middleware/s3api/test_multi_upload.py | 1476 ++++++++--------- test/unit/common/middleware/s3api/test_obj.py | 1069 ++++++------ .../common/middleware/s3api/test_s3_acl.py | 181 +- .../common/middleware/s3api/test_s3api.py | 10 +- .../common/middleware/s3api/test_s3request.py | 3 - .../common/middleware/s3api/test_service.py | 40 +- test/unit/common/middleware/test_helpers.py | 141 +- 14 files changed, 2086 insertions(+), 2209 deletions(-) delete mode 100644 test/unit/common/middleware/s3api/test_helpers.py diff --git a/test/unit/common/middleware/helpers.py b/test/unit/common/middleware/helpers.py index 021e2c4f71..3736c47093 100644 --- a/test/unit/common/middleware/helpers.py +++ b/test/unit/common/middleware/helpers.py @@ -119,13 +119,16 @@ class FakeSwift(object): * received ``POST /v1/a/c/o?x=y``, if it matches a registered ``POST``, will update uploaded ``/v1/a/c/o`` """ - ALLOWED_METHODS = [ + DEFAULT_ALLOWED_METHODS = [ 'PUT', 'POST', 'DELETE', 'GET', 'HEAD', 'OPTIONS', 'REPLICATE', 'SSYNC', 'UPDATE'] container_existence_skip_cache = 0.0 account_existence_skip_cache = 0.0 - def __init__(self): + def __init__(self, allowed_methods=None): + self.allowed_methods = set(self.DEFAULT_ALLOWED_METHODS) + if allowed_methods: + self.allowed_methods.update(allowed_methods) self._calls = [] self.req_bodies = [] self._unclosed_req_keys = defaultdict(int) @@ -136,6 +139,7 @@ class FakeSwift(object): self.uploaded = {} # mapping of (method, path) --> (response class, headers, body) self._responses = {} + self._sticky_headers = {} self.logger = debug_logger('fake-swift') self.account_ring = FakeRing() self.container_ring = FakeRing() @@ -192,7 +196,21 @@ class FakeSwift(object): # HEAD resp never has body body = None - return resp_class, HeaderKeyDict(headers), body + try: + is_success = resp_class().is_success + except Exception: + # test_reconciler passes in an exploding response + is_success = False + if is_success and method in ('GET', 'HEAD'): + # update sticky resp headers with headers from registered resp + sticky_headers = self._sticky_headers.get(env['PATH_INFO'], {}) + resp_headers = HeaderKeyDict(sticky_headers) + resp_headers.update(headers) + else: + # error responses don't get sticky resp headers + resp_headers = HeaderKeyDict(headers) + + return resp_class, resp_headers, body def _get_policy_index(self, acc, cont): path = '/v1/%s/%s' % (acc, cont) @@ -219,7 +237,7 @@ class FakeSwift(object): def __call__(self, env, start_response): method = env['REQUEST_METHOD'] - if method not in self.ALLOWED_METHODS: + if method not in self.allowed_methods: raise HTTPNotImplemented() path, acc, cont, obj = self._parse_path(env) @@ -315,6 +333,9 @@ class FakeSwift(object): return LeakTrackingIter(wsgi_iter, self.mark_closed, self.mark_read, (method, path)) + def clear_calls(self): + del self._calls[:] + def mark_opened(self, key): self._unclosed_req_keys[key] += 1 self._unread_req_paths[key] += 1 @@ -353,6 +374,14 @@ class FakeSwift(object): def call_count(self): return len(self._calls) + def update_sticky_response_headers(self, path, headers): + """ + Tests setUp can use this to ensure any successful GET/HEAD response for + a given path will include these headers. + """ + sticky_headers = self._sticky_headers.setdefault(path, {}) + sticky_headers.update(headers) + def register(self, method, path, response_class, headers, body=b''): path = normalize_path(path) self._responses[(method, path)] = [(response_class, headers, body)] diff --git a/test/unit/common/middleware/s3api/__init__.py b/test/unit/common/middleware/s3api/__init__.py index 1fc614fb45..c6613c5d0e 100644 --- a/test/unit/common/middleware/s3api/__init__.py +++ b/test/unit/common/middleware/s3api/__init__.py @@ -12,20 +12,24 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - +import json import unittest from datetime import datetime import email import mock import time +from contextlib import contextmanager from swift.common import swob +from swift.common.http import is_success from swift.common.middleware.s3api.s3api import filter_factory from swift.common.middleware.s3api.etree import fromstring +from swift.common.middleware.s3api.subresource import Owner, encode_acl, \ + Grant, User, ACL, PERMISSIONS, AllUsers, AuthenticatedUsers from test.debug_logger import debug_logger -from test.unit.common.middleware.s3api.helpers import FakeSwift +from test.unit.common.middleware.helpers import FakeSwift class FakeApp(object): @@ -33,8 +37,9 @@ class FakeApp(object): account_existence_skip_cache = 0.0 def __init__(self): + self.remote_user = 'authorized' self._pipeline_final_app = self - self.swift = FakeSwift() + self.swift = FakeSwift(allowed_methods=['TEST']) self.logger = debug_logger() def _update_s3_path_info(self, env): @@ -50,26 +55,38 @@ class FakeApp(object): path = env['PATH_INFO'] env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant) - def __call__(self, env, start_response): + @staticmethod + def authorize_cb(req): + # Assume swift owner, if not yet set + req.environ.setdefault('swift_owner', True) + # But then default to blocking authz, to ensure we've replaced + # the default auth system + return swob.HTTPForbidden(request=req) + + def handle(self, env): if 's3api.auth_details' in env: self._update_s3_path_info(env) + else: + return + + if self.remote_user: + env['REMOTE_USER'] = self.remote_user if env['REQUEST_METHOD'] == 'TEST': + env['swift.authorize'] = self.authorize_cb + else: + env['swift.authorize'] = lambda req: None - def authorize_cb(req): - # Assume swift owner, if not yet set - req.environ.setdefault('REMOTE_USER', 'authorized') - req.environ.setdefault('swift_owner', True) - # But then default to blocking authz, to ensure we've replaced - # the default auth system - return swob.HTTPForbidden(request=req) - - env['swift.authorize'] = authorize_cb + if 'swift.authorize_override' in env: + return + def __call__(self, env, start_response): + self.handle(env) return self.swift(env, start_response) class S3ApiTestCase(unittest.TestCase): + def __init__(self, name): unittest.TestCase.__init__(self, name) @@ -100,6 +117,11 @@ class S3ApiTestCase(unittest.TestCase): self.s3api = filter_factory({}, **self.conf)(self.app) self.logger = self.s3api.logger = self.swift.logger = debug_logger() + # if you change the registered acl response for /bucket or + # /bucket/object tearDown will complain at you; you can set this to + # True in order to indicate you know what you're doing + self.s3acl_response_modified = False + self.swift.register('HEAD', '/v1/AUTH_test', swob.HTTPOk, {}, None) self.swift.register('HEAD', '/v1/AUTH_test/bucket', @@ -110,7 +132,6 @@ class S3ApiTestCase(unittest.TestCase): swob.HTTPNoContent, {}, None) self.swift.register('DELETE', '/v1/AUTH_test/bucket', swob.HTTPNoContent, {}, None) - self.swift.register('GET', '/v1/AUTH_test/bucket/object', swob.HTTPOk, {'etag': 'object etag'}, "") self.swift.register('PUT', '/v1/AUTH_test/bucket/object', @@ -135,7 +156,7 @@ class S3ApiTestCase(unittest.TestCase): # register bucket HEAD response with given policy index header headers = {'X-Backend-Storage-Policy-Index': str(bucket_policy_index)} self.swift.register('HEAD', '/v1/AUTH_test/' + bucket, - swob.HTTPNoContent, headers, None) + swob.HTTPNoContent, headers) def _assert_policy_index(self, req_headers, resp_headers, policy_index): self.assertNotIn('X-Backend-Storage-Policy-Index', req_headers) @@ -213,5 +234,114 @@ class S3ApiTestCase(unittest.TestCase): else: return status[0], headers[0], body + @contextmanager + def stubbed_container_info(self, versioning_enabled=False): + """ + some tests might want to opt-out of container_info HEAD requests; e.g. + + with self.stubbed_container_info(): + status, headers, body = self.call_s3api(req) + """ + fake_info = {'status': 204} + if versioning_enabled: + fake_info['sysmeta'] = { + 'versions-container': '\x00versions\x00bucket', + } + + with mock.patch('swift.common.middleware.s3api.s3request.' + 'get_container_info', return_value=fake_info): + yield + def call_s3api(self, req, **kwargs): return self.call_app(req, app=self.s3api, **kwargs) + + +def _gen_test_headers(owner, grants=[], resource='container'): + if not grants: + grants = [Grant(User('test:tester'), 'FULL_CONTROL')] + return encode_acl(resource, ACL(owner, grants)) + + +def _gen_grant(permission): + # generate Grant with a grantee named by "permission" + account_name = '%s:%s' % ('test', permission.lower()) + return Grant(User(account_name), permission) + + +class S3ApiTestCaseAcl(S3ApiTestCase): + + def setUp(self): + super(S3ApiTestCaseAcl, self).setUp() + self.s3api.conf.s3_acl = True + + # some extra buckets for s3acl tests + buckets = ['bucket', 'public', 'authenticated'] + for bucket in buckets: + path = '/v1/AUTH_test/' + bucket + self.swift.register('HEAD', path, swob.HTTPNoContent, {}, None), + self.swift.register('GET', path, swob.HTTPOk, {}, json.dumps([])), + + for account in ('AUTH_test', 'AUTH_X'): + self.swift.register('TEST', '/v1/' + account, + swob.HTTPMethodNotAllowed, {}, None) + + # setup sticky ACL headers... + grants = [_gen_grant(perm) for perm in PERMISSIONS] + self.default_owner = Owner('test:tester', 'test:tester') + container_headers = _gen_test_headers(self.default_owner, grants) + object_headers = _gen_test_headers( + self.default_owner, grants, 'object') + public_headers = _gen_test_headers( + self.default_owner, [Grant(AllUsers(), 'READ')]) + authenticated_headers = _gen_test_headers( + self.default_owner, [Grant(AuthenticatedUsers(), 'READ')], + 'bucket') + + sticky_s3acl_headers = { + '/v1/AUTH_test/bucket': container_headers, + '/v1/AUTH_test/bucket+segments': container_headers, + '/v1/AUTH_test/bucket/object': object_headers, + '/v1/AUTH_test/public': public_headers, + '/v1/AUTH_test/authenticated': authenticated_headers, + } + for path, headers in sticky_s3acl_headers.items(): + self.swift.update_sticky_response_headers(path, headers) + + def tearDown(self): + # sanity the test didn't break the the ACLs + swift_path_acl_resp_checks = { + '/v1/AUTH_test/bucket': ( + 'X-Container-Sysmeta-S3api-Acl', '/bucket', + swob.HTTPNoContent), + '/v1/AUTH_test/bucket/object': ( + 'X-Object-Sysmeta-S3api-Acl', '/bucket/object', swob.HTTPOk), + } + check_paths = [] + for swift_path, (acl, check, resp_class) in \ + swift_path_acl_resp_checks.items(): + if self.s3acl_response_modified: + # this is expected to reset back to the original sticky headers + self.swift.register('HEAD', swift_path, resp_class, {}, None) + req = swob.Request.blank(swift_path, method='HEAD') + status, headers, body = self.call_app(req) + if is_success(int(status.split()[0])): + self.assertIn(acl, headers, + 'In tearDown it seems the test (accidently?) ' + 'removed the ACL on %s' % swift_path) + check_paths.append(check) + else: + self.fail('test changed resp for %s' % swift_path) + account_expected = { + 'test:tester': 200, + 'test:other': 403, + } + for account, expected in account_expected.items(): + for path in check_paths: + req = swob.Request.blank(path, method='HEAD', headers={ + 'Authorization': 'AWS %s:hmac' % account, + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(int(status.split()[0]), expected, + 'In tearDown it seems the test (accidently?) ' + 'broke ACL access for %s to %s' % ( + account, path)) diff --git a/test/unit/common/middleware/s3api/helpers.py b/test/unit/common/middleware/s3api/helpers.py index 33603c33ac..2de66f03a9 100644 --- a/test/unit/common/middleware/s3api/helpers.py +++ b/test/unit/common/middleware/s3api/helpers.py @@ -15,87 +15,6 @@ # This stuff can't live in test/unit/__init__.py due to its swob dependency. -from swift.common import swob -from swift.common.utils import split_path -from swift.common.request_helpers import is_sys_meta - -from test.unit.common.middleware.helpers import FakeSwift as BaseFakeSwift - - -class FakeSwift(BaseFakeSwift): - """ - A good-enough fake Swift proxy server to use in testing middleware. - """ - ALLOWED_METHODS = BaseFakeSwift.ALLOWED_METHODS + ['TEST'] - - def __init__(self, s3_acl=False): - super(FakeSwift, self).__init__() - self.s3_acl = s3_acl - self.remote_user = 'authorized' - - def _fake_auth_middleware(self, env): - if 'swift.authorize_override' in env: - return - - if 's3api.auth_details' not in env: - return - - tenant_user = env['s3api.auth_details']['access_key'] - tenant, user = tenant_user.rsplit(':', 1) - - path = env['PATH_INFO'] - env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant) - - if self.remote_user: - env['REMOTE_USER'] = self.remote_user - - if env['REQUEST_METHOD'] == 'TEST': - - def authorize_cb(req): - # Assume swift owner, if not yet set - req.environ.setdefault('swift_owner', True) - # But then default to blocking authz, to ensure we've replaced - # the default auth system - return swob.HTTPForbidden(request=req) - - env['swift.authorize'] = authorize_cb - else: - env['swift.authorize'] = lambda req: None - - def __call__(self, env, start_response): - if self.s3_acl: - self._fake_auth_middleware(env) - return super(FakeSwift, self).__call__(env, start_response) - - def register(self, method, path, response_class, headers, body): - # assuming the path format like /v1/account/container/object - resource_map = ['account', 'container', 'object'] - index = len(list(filter(None, split_path(path, 0, 4, True)[1:]))) - 1 - resource = resource_map[index] - if (method, path) in self._responses: - old_headers = self._responses[(method, path)][0][1] - headers = headers.copy() - for key, value in old_headers.items(): - if is_sys_meta(resource, key) and key not in headers: - # keep old sysmeta for s3acl - headers.update({key: value}) - - if body is not None and not isinstance(body, (bytes, list)): - body = body.encode('utf8') - return super(FakeSwift, self).register( - method, path, response_class, headers, body) - - def register_unconditionally(self, method, path, response_class, headers, - body): - # register() keeps old sysmeta around, but - # register_unconditionally() keeps nothing. - if body is not None and not isinstance(body, bytes): - body = body.encode('utf8') - self._responses[(method, path)] = [(response_class, headers, body)] - - def clear_calls(self): - del self._calls[:] - class UnreadableInput(object): # Some clients will send neither a Content-Length nor a Transfer-Encoding diff --git a/test/unit/common/middleware/s3api/test_acl.py b/test/unit/common/middleware/s3api/test_acl.py index 22f284f782..e0fefd060f 100644 --- a/test/unit/common/middleware/s3api/test_acl.py +++ b/test/unit/common/middleware/s3api/test_acl.py @@ -26,15 +26,14 @@ from swift.common.middleware.s3api.s3response import InvalidArgument from swift.common.middleware.s3api.acl_utils import handle_acl_header from swift.common.utils import md5 -from test.unit.common.middleware.s3api import S3ApiTestCase +from test.unit.common.middleware.s3api import S3ApiTestCase, S3ApiTestCaseAcl from test.unit.common.middleware.s3api.helpers import UnreadableInput -from test.unit.common.middleware.s3api.test_s3_acl import s3acl -class TestS3ApiAcl(S3ApiTestCase): +class BaseS3ApiAcl(object): def setUp(self): - super(TestS3ApiAcl, self).setUp() + super(BaseS3ApiAcl, self).setUp() # All ACL API should be called against to existing bucket. self.swift.register('PUT', '/v1/AUTH_test/bucket', HTTPAccepted, {}, None) @@ -46,7 +45,6 @@ class TestS3ApiAcl(S3ApiTestCase): name = elem.find('./AccessControlList/Grant/Grantee/ID').text self.assertEqual(name, owner) - @s3acl def test_bucket_acl_GET(self): req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'GET'}, @@ -58,6 +56,55 @@ class TestS3ApiAcl(S3ApiTestCase): self.assertSetEqual(set((('HEAD', '/v1/AUTH_test/bucket'),)), set(self.swift.calls)) + def _test_put_no_body(self, use_content_length=False, + use_transfer_encoding=False, string_to_md5=b''): + content_md5 = base64.b64encode( + md5(string_to_md5, usedforsecurity=False).digest()).strip() + with UnreadableInput(self) as fake_input: + req = Request.blank( + '/bucket?acl', + environ={ + 'REQUEST_METHOD': 'PUT', + 'wsgi.input': fake_input}, + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-MD5': content_md5}, + body='') + if not use_content_length: + req.environ.pop('CONTENT_LENGTH') + if use_transfer_encoding: + req.environ['HTTP_TRANSFER_ENCODING'] = 'chunked' + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '400 Bad Request') + self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader') + self.assertEqual(self._get_error_message(body), + 'Your request was missing a required header.') + self.assertIn(b'x-amz-acl', + body) + + def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self): + self._test_put_no_body() + self._test_put_no_body(string_to_md5=b'test') + self._test_put_no_body(use_content_length=True) + self._test_put_no_body(use_content_length=True, string_to_md5=b'test') + self._test_put_no_body(use_transfer_encoding=True) + self._test_put_no_body(use_transfer_encoding=True, string_to_md5=b'zz') + + def test_object_acl_GET(self): + req = Request.blank('/bucket/object?acl', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + if not self.s3api.conf.s3_acl: + self._check_acl('test:tester', body) + self.assertSetEqual(set((('HEAD', '/v1/AUTH_test/bucket/object'),)), + set(self.swift.calls)) + + +class TestS3ApiAclNoSetup(BaseS3ApiAcl, S3ApiTestCase): + def test_bucket_acl_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') @@ -99,19 +146,6 @@ class TestS3ApiAcl(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') - @s3acl(s3acl_only=True) - def test_bucket_canned_acl_PUT_with_s3acl(self): - req = Request.blank('/bucket?acl', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'X-AMZ-ACL': 'public-read'}) - with mock.patch('swift.common.middleware.s3api.s3request.' - 'handle_acl_header') as mock_handler: - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '200') - self.assertEqual(mock_handler.call_count, 0) - def test_bucket_fails_with_both_acl_header_and_xml_PUT(self): elem = Element('AccessControlPolicy') owner = SubElement(elem, 'Owner') @@ -135,54 +169,6 @@ class TestS3ApiAcl(S3ApiTestCase): self.assertEqual(self._get_error_code(body), 'UnexpectedContent') - def _test_put_no_body(self, use_content_length=False, - use_transfer_encoding=False, string_to_md5=b''): - content_md5 = base64.b64encode( - md5(string_to_md5, usedforsecurity=False).digest()).strip() - with UnreadableInput(self) as fake_input: - req = Request.blank( - '/bucket?acl', - environ={ - 'REQUEST_METHOD': 'PUT', - 'wsgi.input': fake_input}, - headers={ - 'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'Content-MD5': content_md5}, - body='') - if not use_content_length: - req.environ.pop('CONTENT_LENGTH') - if use_transfer_encoding: - req.environ['HTTP_TRANSFER_ENCODING'] = 'chunked' - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '400 Bad Request') - self.assertEqual(self._get_error_code(body), 'MissingSecurityHeader') - self.assertEqual(self._get_error_message(body), - 'Your request was missing a required header.') - self.assertIn(b'x-amz-acl', - body) - - @s3acl - def test_bucket_fails_with_neither_acl_header_nor_xml_PUT(self): - self._test_put_no_body() - self._test_put_no_body(string_to_md5=b'test') - self._test_put_no_body(use_content_length=True) - self._test_put_no_body(use_content_length=True, string_to_md5=b'test') - self._test_put_no_body(use_transfer_encoding=True) - self._test_put_no_body(use_transfer_encoding=True, string_to_md5=b'zz') - - @s3acl - def test_object_acl_GET(self): - req = Request.blank('/bucket/object?acl', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - if not self.s3api.conf.s3_acl: - self._check_acl('test:tester', body) - self.assertSetEqual(set((('HEAD', '/v1/AUTH_test/bucket/object'),)), - set(self.swift.calls)) - def test_invalid_xml(self): req = Request.blank('/bucket?acl', environ={'REQUEST_METHOD': 'PUT'}, @@ -210,7 +196,30 @@ class TestS3ApiAcl(S3ApiTestCase): [('X-Container-Read', '.'), ('X-Container-Write', '.')]) - @s3acl(s3acl_only=True) + def test_handle_acl_with_invalid_header_string(self): + req = Request.blank('/bucket', headers={'X-Amz-Acl': 'invalid'}) + with self.assertRaises(InvalidArgument) as cm: + handle_acl_header(req) + self.assertTrue('argument_name' in cm.exception.info) + self.assertEqual(cm.exception.info['argument_name'], 'x-amz-acl') + self.assertTrue('argument_value' in cm.exception.info) + self.assertEqual(cm.exception.info['argument_value'], 'invalid') + + +class TestS3ApiAclCommonSetup(BaseS3ApiAcl, S3ApiTestCaseAcl): + + def test_bucket_canned_acl_PUT_with_s3acl(self): + req = Request.blank('/bucket?acl', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'X-AMZ-ACL': 'public-read'}) + with mock.patch('swift.common.middleware.s3api.s3request.' + 'handle_acl_header') as mock_handler: + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + self.assertEqual(mock_handler.call_count, 0) + def test_handle_acl_header_with_s3acl(self): def check_generated_acl_header(acl, targets): req = Request.blank('/bucket', @@ -227,15 +236,6 @@ class TestS3ApiAcl(S3ApiTestCase): check_generated_acl_header('private', ['X-Container-Read', 'X-Container-Write']) - def test_handle_acl_with_invalid_header_string(self): - req = Request.blank('/bucket', headers={'X-Amz-Acl': 'invalid'}) - with self.assertRaises(InvalidArgument) as cm: - handle_acl_header(req) - self.assertTrue('argument_name' in cm.exception.info) - self.assertEqual(cm.exception.info['argument_name'], 'x-amz-acl') - self.assertTrue('argument_value' in cm.exception.info) - self.assertEqual(cm.exception.info['argument_value'], 'invalid') - if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index 6672639158..ac6c5ef2d8 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -33,15 +33,14 @@ from swift.common.middleware.s3api.subresource import Owner, encode_acl, \ from swift.common.middleware.s3api.s3request import MAX_32BIT_INT from test.unit.common.middleware.helpers import normalize_path -from test.unit.common.middleware.s3api import S3ApiTestCase -from test.unit.common.middleware.s3api.test_s3_acl import s3acl +from test.unit.common.middleware.s3api import S3ApiTestCase, S3ApiTestCaseAcl from test.unit.common.middleware.s3api.helpers import UnreadableInput # Example etag from ProxyFS; note that it is already quote-wrapped PFS_ETAG = '"pfsv2/AUTH_test/01234567/89abcdef-32"' -class TestS3ApiBucket(S3ApiTestCase): +class BaseS3ApiBucket(object): def setup_objects(self): self.objects = (('lily', '2011-01-05T02:19:14.275290', '0', '3909'), (u'lily-\u062a', '2011-01-05T02:19:14.275290', 0, 390), @@ -127,9 +126,327 @@ class TestS3ApiBucket(S3ApiTestCase): ])) def setUp(self): - super(TestS3ApiBucket, self).setUp() + super(BaseS3ApiBucket, self).setUp() self.setup_objects() + def _add_versions_request(self, orig_objects=None, versioned_objects=None, + bucket='junk'): + if orig_objects is None: + orig_objects = self.objects_list + if versioned_objects is None: + versioned_objects = self.versioned_objects + all_versions = versioned_objects + [ + dict(i, version_id='null', is_latest=True) + for i in orig_objects] + all_versions.sort(key=lambda o: ( + o['name'], '' if o['version_id'] == 'null' else o['version_id'])) + self.swift.register( + 'GET', '/v1/AUTH_test/%s' % bucket, swob.HTTPOk, + {'Content-Type': 'application/json'}, json.dumps(all_versions)) + + def _assert_delete_markers(self, elem): + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(len(delete_markers), 1) + self.assertEqual(delete_markers[0].find('./IsLatest').text, 'false') + self.assertEqual(delete_markers[0].find('./VersionId').text, '2') + self.assertEqual(delete_markers[0].find('./Key').text, 'rose') + + def _test_bucket_PUT_with_location(self, root_element): + elem = Element(root_element) + SubElement(elem, 'LocationConstraint').text = 'us-east-1' + xml = tostring(elem) + + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body=xml) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + + def _test_method_error_delete(self, path, sw_resp): + self.swift.register('HEAD', '/v1/AUTH_test' + path, sw_resp, {}, None) + return self._test_method_error('DELETE', path, sw_resp) + + def test_bucket_GET_error(self): + code = self._test_method_error('GET', '/bucket', swob.HTTPUnauthorized) + self.assertEqual(code, 'SignatureDoesNotMatch') + code = self._test_method_error('GET', '/bucket', swob.HTTPForbidden) + self.assertEqual(code, 'AccessDenied') + code = self._test_method_error('GET', '/bucket', swob.HTTPNotFound) + self.assertEqual(code, 'NoSuchBucket') + code = self._test_method_error('GET', '/bucket', + swob.HTTPServiceUnavailable) + self.assertEqual(code, 'ServiceUnavailable') + code = self._test_method_error('GET', '/bucket', swob.HTTPServerError) + self.assertEqual(code, 'InternalError') + + def test_bucket_GET_non_json(self): + # Suppose some middleware accidentally makes it return txt instead + resp_body = b'\n'.join([b'obj%d' % i for i in range(100)]) + self.swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPOk, {}, + resp_body) + # When we do our GET... + req = Request.blank('/bucket', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + # ...there isn't much choice but to error... + self.assertEqual(self._get_error_code(body), 'InternalError') + # ... but we should at least log the body to aid in debugging + self.assertIn( + 'Got non-JSON response trying to list /bucket: %r' + % (resp_body[:60] + b'...'), + self.s3api.logger.get_lines_for_level('error')) + + def test_bucket_PUT_error(self): + code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated, + headers={'Content-Length': 'a'}) + self.assertEqual(code, 'InvalidArgument') + code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated, + headers={'Content-Length': '-1'}) + self.assertEqual(code, 'InvalidArgument') + code = self._test_method_error('PUT', '/bucket', swob.HTTPUnauthorized) + self.assertEqual(code, 'SignatureDoesNotMatch') + code = self._test_method_error('PUT', '/bucket', swob.HTTPForbidden) + self.assertEqual(code, 'AccessDenied') + code = self._test_method_error('PUT', '/bucket', swob.HTTPAccepted) + self.assertEqual(code, 'BucketAlreadyOwnedByYou') + with mock.patch( + 'swift.common.middleware.s3api.s3request.get_container_info', + return_value={'sysmeta': {'s3api-acl': '{"Owner": "nope"}'}}): + code = self._test_method_error( + 'PUT', '/bucket', swob.HTTPAccepted) + self.assertEqual(code, 'BucketAlreadyExists') + code = self._test_method_error('PUT', '/bucket', swob.HTTPServerError) + self.assertEqual(code, 'InternalError') + code = self._test_method_error( + 'PUT', '/bucket', swob.HTTPServiceUnavailable) + self.assertEqual(code, 'ServiceUnavailable') + code = self._test_method_error( + 'PUT', '/bucket+bucket', swob.HTTPCreated) + self.assertEqual(code, 'InvalidBucketName') + code = self._test_method_error( + 'PUT', '/192.168.11.1', swob.HTTPCreated) + self.assertEqual(code, 'InvalidBucketName') + code = self._test_method_error( + 'PUT', '/bucket.-bucket', swob.HTTPCreated) + self.assertEqual(code, 'InvalidBucketName') + code = self._test_method_error( + 'PUT', '/bucket-.bucket', swob.HTTPCreated) + self.assertEqual(code, 'InvalidBucketName') + code = self._test_method_error('PUT', '/bucket*', swob.HTTPCreated) + self.assertEqual(code, 'InvalidBucketName') + code = self._test_method_error('PUT', '/b', swob.HTTPCreated) + self.assertEqual(code, 'InvalidBucketName') + code = self._test_method_error( + 'PUT', '/%s' % ''.join(['b' for x in range(64)]), + swob.HTTPCreated) + self.assertEqual(code, 'InvalidBucketName') + + def test_bucket_PUT_bucket_already_owned_by_you(self): + self.swift.register( + 'PUT', '/v1/AUTH_test/bucket', swob.HTTPAccepted, + {'X-Container-Object-Count': 0}, None) + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '409 Conflict') + self.assertIn(b'BucketAlreadyOwnedByYou', body) + + def test_bucket_PUT_first_put_fail(self): + self.swift.register( + 'PUT', '/v1/AUTH_test/bucket', + swob.HTTPServiceUnavailable, + {'X-Container-Object-Count': 0}, None) + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '503 Service Unavailable') + # The last call was PUT not POST for acl set + self.assertEqual(self.swift.calls, [ + ('PUT', '/v1/AUTH_test/bucket'), + ]) + + def test_bucket_PUT(self): + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(body, b'') + self.assertEqual(status.split()[0], '200') + self.assertEqual(headers['Location'], '/bucket') + + # Apparently some clients will include a chunked transfer-encoding + # even with no body + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Transfer-Encoding': 'chunked'}) + status, headers, body = self.call_s3api(req) + self.assertEqual(body, b'') + self.assertEqual(status.split()[0], '200') + self.assertEqual(headers['Location'], '/bucket') + + with UnreadableInput(self) as fake_input: + req = Request.blank( + '/bucket', + environ={'REQUEST_METHOD': 'PUT', + 'wsgi.input': fake_input}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(body, b'') + self.assertEqual(status.split()[0], '200') + self.assertEqual(headers['Location'], '/bucket') + + def test_bucket_PUT_with_location(self): + self._test_bucket_PUT_with_location('CreateBucketConfiguration') + + def test_bucket_PUT_with_ami_location(self): + # ec2-ami-tools apparently uses CreateBucketConstraint instead? + self._test_bucket_PUT_with_location('CreateBucketConstraint') + + def test_bucket_PUT_with_strange_location(self): + # Even crazier: it doesn't seem to matter + self._test_bucket_PUT_with_location('foo') + + def test_bucket_PUT_with_location_error(self): + elem = Element('CreateBucketConfiguration') + SubElement(elem, 'LocationConstraint').text = 'XXX' + xml = tostring(elem) + + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body=xml) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), + 'InvalidLocationConstraint') + + def test_bucket_PUT_with_location_invalid_xml(self): + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='invalid_xml') + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'MalformedXML') + + def test_bucket_DELETE_error(self): + code = self._test_method_error_delete('/bucket', swob.HTTPUnauthorized) + self.assertEqual(code, 'SignatureDoesNotMatch') + code = self._test_method_error_delete('/bucket', swob.HTTPForbidden) + self.assertEqual(code, 'AccessDenied') + code = self._test_method_error_delete('/bucket', swob.HTTPNotFound) + self.assertEqual(code, 'NoSuchBucket') + code = self._test_method_error_delete('/bucket', swob.HTTPServerError) + self.assertEqual(code, 'InternalError') + + # bucket not empty is now validated at s3api + self.swift._responses.get(('HEAD', '/v1/AUTH_test/bucket')) + self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, + {'X-Container-Object-Count': '1'}, None) + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, _headers, body = self.call_s3api(req) + self.assertEqual('409 Conflict', status) + self.assertEqual('BucketNotEmpty', self._get_error_code(body)) + self.assertNotIn('You must delete all versions in the bucket', + self._get_error_message(body)) + + def test_bucket_DELETE_error_with_enabled_versioning(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, + {'X-Container-Object-Count': '1', + 'X-Container-Sysmeta-Versions-Enabled': 'True'}, + None) + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, _headers, body = self.call_s3api(req) + self.assertEqual('409 Conflict', status) + self.assertEqual('BucketNotEmpty', self._get_error_code(body)) + self.assertIn('You must delete all versions in the bucket', + self._get_error_message(body)) + + def test_bucket_DELETE_error_with_suspended_versioning(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, + {'X-Container-Object-Count': '1', + 'X-Container-Sysmeta-Versions-Enabled': 'False'}, + None) + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, _headers, body = self.call_s3api(req) + self.assertEqual('409 Conflict', status) + self.assertEqual('BucketNotEmpty', self._get_error_code(body)) + self.assertIn('You must delete all versions in the bucket', + self._get_error_message(body)) + + def test_bucket_DELETE(self): + # overwrite default HEAD to return x-container-object-count + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, + {'X-Container-Object-Count': 0}, None) + + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + + def test_bucket_DELETE_with_empty_versioning(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket+versioning', + swob.HTTPNoContent, {}, None) + self.swift.register('DELETE', '/v1/AUTH_test/bucket+versioning', + swob.HTTPNoContent, {}, None) + # overwrite default HEAD to return x-container-object-count + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, + {'X-Container-Object-Count': 0}, None) + + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + + def test_bucket_DELETE_error_while_segment_bucket_delete(self): + # An error occurred while deleting segment objects + self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/lily', + swob.HTTPServiceUnavailable, {}, json.dumps([])) + # overwrite default HEAD to return x-container-object-count + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, + {'X-Container-Object-Count': 0}, None) + + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '503') + called = [(method, path) for method, path, _ in + self.swift.calls_with_headers] + # Don't delete original bucket when error occurred in segment container + self.assertNotIn(('DELETE', '/v1/AUTH_test/bucket'), called) + + +class TestS3ApiBucketNoACL(BaseS3ApiBucket, S3ApiTestCase): + def test_bucket_HEAD(self): req = Request.blank('/junk', environ={'REQUEST_METHOD': 'HEAD'}, @@ -194,39 +511,6 @@ class TestS3ApiBucket(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '404') - @s3acl - def test_bucket_GET_error(self): - code = self._test_method_error('GET', '/bucket', swob.HTTPUnauthorized) - self.assertEqual(code, 'SignatureDoesNotMatch') - code = self._test_method_error('GET', '/bucket', swob.HTTPForbidden) - self.assertEqual(code, 'AccessDenied') - code = self._test_method_error('GET', '/bucket', swob.HTTPNotFound) - self.assertEqual(code, 'NoSuchBucket') - code = self._test_method_error('GET', '/bucket', - swob.HTTPServiceUnavailable) - self.assertEqual(code, 'ServiceUnavailable') - code = self._test_method_error('GET', '/bucket', swob.HTTPServerError) - self.assertEqual(code, 'InternalError') - - @s3acl - def test_bucket_GET_non_json(self): - # Suppose some middleware accidentally makes it return txt instead - resp_body = b'\n'.join([b'obj%d' % i for i in range(100)]) - self.swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPOk, {}, - resp_body) - # When we do our GET... - req = Request.blank('/bucket', - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - # ...there isn't much choice but to error... - self.assertEqual(self._get_error_code(body), 'InternalError') - # ... but we should at least log the body to aid in debugging - self.assertIn( - 'Got non-JSON response trying to list /bucket: %r' - % (resp_body[:60] + b'...'), - self.s3api.logger.get_lines_for_level('error')) - def test_bucket_GET(self): bucket_name = 'junk' req = Request.blank('/%s' % bucket_name, @@ -765,28 +1049,6 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual([v.find('./StorageClass').text for v in versions], ['STANDARD' for v in objects]) - def _add_versions_request(self, orig_objects=None, versioned_objects=None, - bucket='junk'): - if orig_objects is None: - orig_objects = self.objects_list - if versioned_objects is None: - versioned_objects = self.versioned_objects - all_versions = versioned_objects + [ - dict(i, version_id='null', is_latest=True) - for i in orig_objects] - all_versions.sort(key=lambda o: ( - o['name'], '' if o['version_id'] == 'null' else o['version_id'])) - self.swift.register( - 'GET', '/v1/AUTH_test/%s' % bucket, swob.HTTPOk, - {'Content-Type': 'application/json'}, json.dumps(all_versions)) - - def _assert_delete_markers(self, elem): - delete_markers = elem.findall('./DeleteMarker') - self.assertEqual(len(delete_markers), 1) - self.assertEqual(delete_markers[0].find('./IsLatest').text, 'false') - self.assertEqual(delete_markers[0].find('./VersionId').text, '2') - self.assertEqual(delete_markers[0].find('./Key').text, 'rose') - def test_bucket_GET_with_versions(self): self._add_versions_request() req = Request.blank('/junk?versions', @@ -1205,150 +1467,6 @@ class TestS3ApiBucket(S3ApiTestCase): '?limit=1001&prefix=subdir/&versions=')), ]) - @s3acl - def test_bucket_PUT_error(self): - code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated, - headers={'Content-Length': 'a'}) - self.assertEqual(code, 'InvalidArgument') - code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated, - headers={'Content-Length': '-1'}) - self.assertEqual(code, 'InvalidArgument') - code = self._test_method_error('PUT', '/bucket', swob.HTTPUnauthorized) - self.assertEqual(code, 'SignatureDoesNotMatch') - code = self._test_method_error('PUT', '/bucket', swob.HTTPForbidden) - self.assertEqual(code, 'AccessDenied') - code = self._test_method_error('PUT', '/bucket', swob.HTTPAccepted) - self.assertEqual(code, 'BucketAlreadyOwnedByYou') - with mock.patch( - 'swift.common.middleware.s3api.s3request.get_container_info', - return_value={'sysmeta': {'s3api-acl': '{"Owner": "nope"}'}}): - code = self._test_method_error( - 'PUT', '/bucket', swob.HTTPAccepted) - self.assertEqual(code, 'BucketAlreadyExists') - code = self._test_method_error('PUT', '/bucket', swob.HTTPServerError) - self.assertEqual(code, 'InternalError') - code = self._test_method_error( - 'PUT', '/bucket', swob.HTTPServiceUnavailable) - self.assertEqual(code, 'ServiceUnavailable') - code = self._test_method_error( - 'PUT', '/bucket+bucket', swob.HTTPCreated) - self.assertEqual(code, 'InvalidBucketName') - code = self._test_method_error( - 'PUT', '/192.168.11.1', swob.HTTPCreated) - self.assertEqual(code, 'InvalidBucketName') - code = self._test_method_error( - 'PUT', '/bucket.-bucket', swob.HTTPCreated) - self.assertEqual(code, 'InvalidBucketName') - code = self._test_method_error( - 'PUT', '/bucket-.bucket', swob.HTTPCreated) - self.assertEqual(code, 'InvalidBucketName') - code = self._test_method_error('PUT', '/bucket*', swob.HTTPCreated) - self.assertEqual(code, 'InvalidBucketName') - code = self._test_method_error('PUT', '/b', swob.HTTPCreated) - self.assertEqual(code, 'InvalidBucketName') - code = self._test_method_error( - 'PUT', '/%s' % ''.join(['b' for x in range(64)]), - swob.HTTPCreated) - self.assertEqual(code, 'InvalidBucketName') - - @s3acl(s3acl_only=True) - def test_bucket_PUT_error_non_swift_owner(self): - code = self._test_method_error('PUT', '/bucket', swob.HTTPAccepted, - env={'swift_owner': False}) - self.assertEqual(code, 'AccessDenied') - - @s3acl - def test_bucket_PUT_bucket_already_owned_by_you(self): - self.swift.register( - 'PUT', '/v1/AUTH_test/bucket', swob.HTTPAccepted, - {'X-Container-Object-Count': 0}, None) - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '409 Conflict') - self.assertIn(b'BucketAlreadyOwnedByYou', body) - - @s3acl - def test_bucket_PUT_first_put_fail(self): - self.swift.register( - 'PUT', '/v1/AUTH_test/bucket', - swob.HTTPServiceUnavailable, - {'X-Container-Object-Count': 0}, None) - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '503 Service Unavailable') - # The last call was PUT not POST for acl set - self.assertEqual(self.swift.calls, [ - ('PUT', '/v1/AUTH_test/bucket'), - ]) - - @s3acl - def test_bucket_PUT(self): - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(body, b'') - self.assertEqual(status.split()[0], '200') - self.assertEqual(headers['Location'], '/bucket') - - # Apparently some clients will include a chunked transfer-encoding - # even with no body - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'Transfer-Encoding': 'chunked'}) - status, headers, body = self.call_s3api(req) - self.assertEqual(body, b'') - self.assertEqual(status.split()[0], '200') - self.assertEqual(headers['Location'], '/bucket') - - with UnreadableInput(self) as fake_input: - req = Request.blank( - '/bucket', - environ={'REQUEST_METHOD': 'PUT', - 'wsgi.input': fake_input}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(body, b'') - self.assertEqual(status.split()[0], '200') - self.assertEqual(headers['Location'], '/bucket') - - def _test_bucket_PUT_with_location(self, root_element): - elem = Element(root_element) - SubElement(elem, 'LocationConstraint').text = 'us-east-1' - xml = tostring(elem) - - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body=xml) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '200') - - @s3acl - def test_bucket_PUT_with_location(self): - self._test_bucket_PUT_with_location('CreateBucketConfiguration') - - @s3acl - def test_bucket_PUT_with_ami_location(self): - # ec2-ami-tools apparently uses CreateBucketConstraint instead? - self._test_bucket_PUT_with_location('CreateBucketConstraint') - - @s3acl - def test_bucket_PUT_with_strange_location(self): - # Even crazier: it doesn't seem to matter - self._test_bucket_PUT_with_location('foo') - def test_bucket_PUT_with_mixed_case_location(self): self.s3api.conf.location = 'RegionOne' elem = Element('CreateBucketConfiguration') @@ -1385,7 +1503,8 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual(headers.get('X-Container-Read'), '.r:*,.rlistings') self.assertNotIn('X-Container-Sysmeta-S3api-Acl', headers) - @s3acl(s3acl_only=True) + +class TestS3ApiBucketAcl(BaseS3ApiBucket, S3ApiTestCaseAcl): def test_bucket_PUT_with_canned_s3acl(self): account = 'test:tester' acl = \ @@ -1403,144 +1522,10 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual(headers.get('X-Container-Sysmeta-S3api-Acl'), acl['x-container-sysmeta-s3api-acl']) - @s3acl - def test_bucket_PUT_with_location_error(self): - elem = Element('CreateBucketConfiguration') - SubElement(elem, 'LocationConstraint').text = 'XXX' - xml = tostring(elem) - - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body=xml) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), - 'InvalidLocationConstraint') - - @s3acl - def test_bucket_PUT_with_location_invalid_xml(self): - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body='invalid_xml') - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'MalformedXML') - - def _test_method_error_delete(self, path, sw_resp): - self.swift.register('HEAD', '/v1/AUTH_test' + path, sw_resp, {}, None) - return self._test_method_error('DELETE', path, sw_resp) - - @s3acl - def test_bucket_DELETE_error(self): - code = self._test_method_error_delete('/bucket', swob.HTTPUnauthorized) - self.assertEqual(code, 'SignatureDoesNotMatch') - code = self._test_method_error_delete('/bucket', swob.HTTPForbidden) + def test_bucket_PUT_error_non_swift_owner(self): + code = self._test_method_error('PUT', '/bucket', swob.HTTPAccepted, + env={'swift_owner': False}) self.assertEqual(code, 'AccessDenied') - code = self._test_method_error_delete('/bucket', swob.HTTPNotFound) - self.assertEqual(code, 'NoSuchBucket') - code = self._test_method_error_delete('/bucket', swob.HTTPServerError) - self.assertEqual(code, 'InternalError') - - # bucket not empty is now validated at s3api - self.swift._responses.get(('HEAD', '/v1/AUTH_test/bucket')) - self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - {'X-Container-Object-Count': '1'}, None) - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, _headers, body = self.call_s3api(req) - self.assertEqual('409 Conflict', status) - self.assertEqual('BucketNotEmpty', self._get_error_code(body)) - self.assertNotIn('You must delete all versions in the bucket', - self._get_error_message(body)) - - @s3acl - def test_bucket_DELETE_error_with_enabled_versioning(self): - self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - {'X-Container-Object-Count': '1', - 'X-Container-Sysmeta-Versions-Enabled': 'True'}, - None) - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, _headers, body = self.call_s3api(req) - self.assertEqual('409 Conflict', status) - self.assertEqual('BucketNotEmpty', self._get_error_code(body)) - self.assertIn('You must delete all versions in the bucket', - self._get_error_message(body)) - - @s3acl - def test_bucket_DELETE_error_with_suspended_versioning(self): - self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - {'X-Container-Object-Count': '1', - 'X-Container-Sysmeta-Versions-Enabled': 'False'}, - None) - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, _headers, body = self.call_s3api(req) - self.assertEqual('409 Conflict', status) - self.assertEqual('BucketNotEmpty', self._get_error_code(body)) - self.assertIn('You must delete all versions in the bucket', - self._get_error_message(body)) - - @s3acl - def test_bucket_DELETE(self): - # overwrite default HEAD to return x-container-object-count - self.swift.register( - 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - {'X-Container-Object-Count': 0}, None) - - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - - @s3acl - def test_bucket_DELETE_with_empty_versioning(self): - self.swift.register('HEAD', '/v1/AUTH_test/bucket+versioning', - swob.HTTPNoContent, {}, None) - self.swift.register('DELETE', '/v1/AUTH_test/bucket+versioning', - swob.HTTPNoContent, {}, None) - # overwrite default HEAD to return x-container-object-count - self.swift.register( - 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - {'X-Container-Object-Count': 0}, None) - - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - - @s3acl - def test_bucket_DELETE_error_while_segment_bucket_delete(self): - # An error occurred while deleting segment objects - self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/lily', - swob.HTTPServiceUnavailable, {}, json.dumps([])) - # overwrite default HEAD to return x-container-object-count - self.swift.register( - 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - {'X-Container-Object-Count': 0}, None) - - req = Request.blank('/bucket', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '503') - called = [(method, path) for method, path, _ in - self.swift.calls_with_headers] - # Don't delete original bucket when error occurred in segment container - self.assertNotIn(('DELETE', '/v1/AUTH_test/bucket'), called) def _test_bucket_for_s3acl(self, method, account): req = Request.blank('/bucket', @@ -1550,25 +1535,21 @@ class TestS3ApiBucket(S3ApiTestCase): return self.call_s3api(req) - @s3acl(s3acl_only=True) def test_bucket_GET_without_permission(self): status, headers, body = self._test_bucket_for_s3acl('GET', 'test:other') self.assertEqual(self._get_error_code(body), 'AccessDenied') - @s3acl(s3acl_only=True) def test_bucket_GET_with_read_permission(self): status, headers, body = self._test_bucket_for_s3acl('GET', 'test:read') self.assertEqual(status.split()[0], '200') - @s3acl(s3acl_only=True) def test_bucket_GET_with_fullcontrol_permission(self): status, headers, body = \ self._test_bucket_for_s3acl('GET', 'test:full_control') self.assertEqual(status.split()[0], '200') - @s3acl(s3acl_only=True) def test_bucket_GET_with_owner_permission(self): status, headers, body = self._test_bucket_for_s3acl('GET', 'test:tester') @@ -1582,18 +1563,15 @@ class TestS3ApiBucket(S3ApiTestCase): return self.call_s3api(req) - @s3acl(s3acl_only=True) def test_bucket_GET_authenticated_users(self): status, headers, body = \ self._test_bucket_GET_canned_acl('authenticated') self.assertEqual(status.split()[0], '200') - @s3acl(s3acl_only=True) def test_bucket_GET_all_users(self): status, headers, body = self._test_bucket_GET_canned_acl('public') self.assertEqual(status.split()[0], '200') - @s3acl(s3acl_only=True) def test_bucket_DELETE_without_permission(self): status, headers, body = self._test_bucket_for_s3acl('DELETE', 'test:other') @@ -1602,7 +1580,6 @@ class TestS3ApiBucket(S3ApiTestCase): called = [method for method, _, _ in self.swift.calls_with_headers] self.assertNotIn('DELETE', called) - @s3acl(s3acl_only=True) def test_bucket_DELETE_with_write_permission(self): status, headers, body = self._test_bucket_for_s3acl('DELETE', 'test:write') @@ -1611,7 +1588,6 @@ class TestS3ApiBucket(S3ApiTestCase): called = [method for method, _, _ in self.swift.calls_with_headers] self.assertNotIn('DELETE', called) - @s3acl(s3acl_only=True) def test_bucket_DELETE_with_fullcontrol_permission(self): status, headers, body = \ self._test_bucket_for_s3acl('DELETE', 'test:full_control') diff --git a/test/unit/common/middleware/s3api/test_helpers.py b/test/unit/common/middleware/s3api/test_helpers.py deleted file mode 100644 index 052218481b..0000000000 --- a/test/unit/common/middleware/s3api/test_helpers.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This stuff can't live in test/unit/__init__.py due to its swob dependency. - -import unittest -from test.unit.common.middleware.s3api.helpers import FakeSwift -from swift.common.middleware.s3api.utils import sysmeta_header -from swift.common.swob import HeaderKeyDict -from mock import MagicMock - - -class S3ApiHelperTestCase(unittest.TestCase): - def setUp(self): - self.method = 'HEAD' - self.path = '/v1/AUTH_test/bucket' - - def _check_headers(self, swift, method, path, headers): - _, response_headers, _ = swift._responses[(method, path)][0] - self.assertEqual(headers, response_headers) - - def test_fake_swift_sysmeta(self): - swift = FakeSwift() - orig_headers = HeaderKeyDict() - orig_headers.update({sysmeta_header('container', 'acl'): 'test', - 'x-container-meta-foo': 'bar'}) - - swift.register(self.method, self.path, MagicMock(), orig_headers, None) - - self._check_headers(swift, self.method, self.path, orig_headers) - - new_headers = orig_headers.copy() - del new_headers[sysmeta_header('container', 'acl').title()] - swift.register(self.method, self.path, MagicMock(), new_headers, None) - - self._check_headers(swift, self.method, self.path, orig_headers) - - def test_fake_swift_sysmeta_overwrite(self): - swift = FakeSwift() - orig_headers = HeaderKeyDict() - orig_headers.update({sysmeta_header('container', 'acl'): 'test', - 'x-container-meta-foo': 'bar'}) - swift.register(self.method, self.path, MagicMock(), orig_headers, None) - - self._check_headers(swift, self.method, self.path, orig_headers) - - new_headers = orig_headers.copy() - new_headers[sysmeta_header('container', 'acl').title()] = 'bar' - - swift.register(self.method, self.path, MagicMock(), new_headers, None) - - self.assertFalse(orig_headers == new_headers) - self._check_headers(swift, self.method, self.path, new_headers) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit/common/middleware/s3api/test_multi_delete.py b/test/unit/common/middleware/s3api/test_multi_delete.py index d40b48f2de..8cf4e5d282 100644 --- a/test/unit/common/middleware/s3api/test_multi_delete.py +++ b/test/unit/common/middleware/s3api/test_multi_delete.py @@ -24,18 +24,17 @@ from swift.common import swob from swift.common.swob import Request from test.unit import make_timestamp_iter -from test.unit.common.middleware.s3api import S3ApiTestCase +from test.unit.common.middleware.s3api import S3ApiTestCase, S3ApiTestCaseAcl from test.unit.common.middleware.s3api.helpers import UnreadableInput from swift.common.middleware.s3api.etree import fromstring, tostring, \ Element, SubElement from swift.common.utils import md5 -from test.unit.common.middleware.s3api.test_s3_acl import s3acl -class TestS3ApiMultiDelete(S3ApiTestCase): +class BaseS3ApiMultiDelete(object): def setUp(self): - super(TestS3ApiMultiDelete, self).setUp() + super(BaseS3ApiMultiDelete, self).setUp() self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key1', swob.HTTPOk, {}, None) self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key2', @@ -45,7 +44,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): swob.HTTPOk, {}, None) self.ts = make_timestamp_iter() - @s3acl def test_object_multi_DELETE_to_object(self): elem = Element('Delete') obj = SubElement(elem, 'Object') @@ -64,7 +62,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') - @s3acl def test_object_multi_DELETE(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) @@ -109,7 +106,8 @@ class TestS3ApiMultiDelete(S3ApiTestCase): 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) - status, headers, body = self.call_s3api(req) + with self.stubbed_container_info(): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) @@ -130,7 +128,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): ('DELETE', '/v1/AUTH_test/bucket/business/caf\xc3\xa9'), ]) - @s3acl def test_object_multi_DELETE_with_error(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) @@ -170,7 +167,8 @@ class TestS3ApiMultiDelete(S3ApiTestCase): 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) - status, headers, body = self.call_s3api(req) + with self.stubbed_container_info(): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) @@ -196,7 +194,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): ('DELETE', '/v1/AUTH_test/bucket/Key4?multipart-manifest=delete'), ]) - @s3acl def test_object_multi_DELETE_with_non_json(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) @@ -242,7 +239,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): 'Could not parse SLO delete response (200 OK): %s: ' % b'asdf']) self.s3api.logger.clear() - @s3acl def test_object_multi_DELETE_quiet(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) @@ -272,7 +268,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), 0) - @s3acl def test_object_multi_DELETE_no_key(self): self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', swob.HTTPNoContent, {}, None) @@ -297,7 +292,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'UserKeyMustBeSpecified') - @s3acl def test_object_multi_DELETE_versioned_enabled(self): self.swift.register( 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, { @@ -344,7 +338,9 @@ class TestS3ApiMultiDelete(S3ApiTestCase): 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) - status, headers, body = self.call_s3api(req) + # XXX versioning_enabled=True not required? + with self.stubbed_container_info(): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') self.assertEqual(self.swift.calls, [ @@ -363,7 +359,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertEqual({'Key1', 'Key2', 'Key3', 'Key4'}, set( e.findtext('Key') for e in elem.findall('Deleted'))) - @s3acl def test_object_multi_DELETE_versioned_suspended(self): self.swift.register( 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, {}, None) @@ -402,7 +397,9 @@ class TestS3ApiMultiDelete(S3ApiTestCase): 'Date': self.get_date_header(), 'Content-MD5': content_md5}, body=body) - status, headers, body = self.call_s3api(req) + # XXX versioning_enabled=True not required? + with self.stubbed_container_info(): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body) self.assertEqual(len(elem.findall('Deleted')), 3) @@ -421,7 +418,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): ('DELETE', '/v1/AUTH_test/bucket/Key3'), ]) - @s3acl def test_object_multi_DELETE_with_invalid_md5(self): elem = Element('Delete') for key in ['Key1', 'Key2']: @@ -438,7 +434,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidDigest') - @s3acl def test_object_multi_DELETE_without_md5(self): elem = Element('Delete') for key in ['Key1', 'Key2']: @@ -454,7 +449,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidRequest') - @s3acl def test_object_multi_DELETE_lots_of_keys(self): elem = Element('Delete') for i in range(self.s3api.conf.max_multi_delete_objects): @@ -483,7 +477,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertEqual(len(elem.findall('Deleted')), self.s3api.conf.max_multi_delete_objects) - @s3acl def test_object_multi_DELETE_too_many_keys(self): elem = Element('Delete') for i in range(self.s3api.conf.max_multi_delete_objects + 1): @@ -502,7 +495,6 @@ class TestS3ApiMultiDelete(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'MalformedXML') - @s3acl def test_object_multi_DELETE_unhandled_exception(self): exploding_resp = mock.MagicMock( side_effect=Exception('kaboom')) @@ -525,61 +517,40 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertEqual(status.split()[0], '200') self.assertIn(b'Key1Server Error', body) - def _test_object_multi_DELETE(self, account): - self.keys = ['Key1', 'Key2'] - self.swift.register( - 'DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[0], - swob.HTTPNoContent, {}, None) - self.swift.register( - 'DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[1], - swob.HTTPNotFound, {}, None) - - elem = Element('Delete') - for key in self.keys: - obj = SubElement(elem, 'Object') - SubElement(obj, 'Key').text = key - body = tostring(elem, use_s3ns=False) - content_md5 = ( - base64.b64encode(md5(body, usedforsecurity=False).digest()) + def _test_no_body(self, use_content_length=False, + use_transfer_encoding=False, string_to_md5=b''): + content_md5 = (base64.b64encode( + md5(string_to_md5, usedforsecurity=False).digest()) .strip()) + with UnreadableInput(self) as fake_input: + req = Request.blank( + '/bucket?delete', + environ={ + 'REQUEST_METHOD': 'POST', + 'wsgi.input': fake_input}, + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-MD5': content_md5}, + body='') + if not use_content_length: + req.environ.pop('CONTENT_LENGTH') + if use_transfer_encoding: + req.environ['HTTP_TRANSFER_ENCODING'] = 'chunked' + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '400 Bad Request') + self.assertEqual(self._get_error_code(body), 'MissingRequestBodyError') - req = Request.blank('/bucket?delete', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Authorization': 'AWS %s:hmac' % account, - 'Date': self.get_date_header(), - 'Content-MD5': content_md5}, - body=body) - req.date = datetime.now() - req.content_type = 'text/plain' + def test_object_multi_DELETE_empty_body(self): + self._test_no_body() + self._test_no_body(string_to_md5=b'test') + self._test_no_body(use_content_length=True) + self._test_no_body(use_content_length=True, string_to_md5=b'test') + self._test_no_body(use_transfer_encoding=True) + self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test') - return self.call_s3api(req) - @s3acl(s3acl_only=True) - def test_object_multi_DELETE_without_permission(self): - status, headers, body = self._test_object_multi_DELETE('test:other') - self.assertEqual(status.split()[0], '200') - elem = fromstring(body) - errors = elem.findall('Error') - self.assertEqual(len(errors), len(self.keys)) - for e in errors: - self.assertTrue(e.find('Key').text in self.keys) - self.assertEqual(e.find('Code').text, 'AccessDenied') - self.assertEqual(e.find('Message').text, 'Access Denied.') - - @s3acl(s3acl_only=True) - def test_object_multi_DELETE_with_write_permission(self): - status, headers, body = self._test_object_multi_DELETE('test:write') - self.assertEqual(status.split()[0], '200') - elem = fromstring(body) - self.assertEqual(len(elem.findall('Deleted')), len(self.keys)) - - @s3acl(s3acl_only=True) - def test_object_multi_DELETE_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_object_multi_DELETE('test:full_control') - self.assertEqual(status.split()[0], '200') - elem = fromstring(body) - self.assertEqual(len(elem.findall('Deleted')), len(self.keys)) +class TestS3ApiMultiDeleteNoAcl(BaseS3ApiMultiDelete, S3ApiTestCase): def test_object_multi_DELETE_with_system_entity(self): self.keys = ['Key1', 'Key2'] @@ -620,38 +591,61 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertNotIn(b'root:/root', body) self.assertIn(b'Key1', body) - def _test_no_body(self, use_content_length=False, - use_transfer_encoding=False, string_to_md5=b''): - content_md5 = (base64.b64encode( - md5(string_to_md5, usedforsecurity=False).digest()) - .strip()) - with UnreadableInput(self) as fake_input: - req = Request.blank( - '/bucket?delete', - environ={ - 'REQUEST_METHOD': 'POST', - 'wsgi.input': fake_input}, - headers={ - 'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'Content-MD5': content_md5}, - body='') - if not use_content_length: - req.environ.pop('CONTENT_LENGTH') - if use_transfer_encoding: - req.environ['HTTP_TRANSFER_ENCODING'] = 'chunked' - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '400 Bad Request') - self.assertEqual(self._get_error_code(body), 'MissingRequestBodyError') - @s3acl - def test_object_multi_DELETE_empty_body(self): - self._test_no_body() - self._test_no_body(string_to_md5=b'test') - self._test_no_body(use_content_length=True) - self._test_no_body(use_content_length=True, string_to_md5=b'test') - self._test_no_body(use_transfer_encoding=True) - self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test') +class TestS3ApiMultiDeleteAcl(BaseS3ApiMultiDelete, S3ApiTestCaseAcl): + + def _test_object_multi_DELETE(self, account): + self.keys = ['Key1', 'Key2'] + self.swift.register( + 'DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[0], + swob.HTTPNoContent, {}, None) + self.swift.register( + 'DELETE', '/v1/AUTH_test/bucket/%s' % self.keys[1], + swob.HTTPNotFound, {}, None) + + elem = Element('Delete') + for key in self.keys: + obj = SubElement(elem, 'Object') + SubElement(obj, 'Key').text = key + body = tostring(elem, use_s3ns=False) + content_md5 = ( + base64.b64encode(md5(body, usedforsecurity=False).digest()) + .strip()) + + req = Request.blank('/bucket?delete', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS %s:hmac' % account, + 'Date': self.get_date_header(), + 'Content-MD5': content_md5}, + body=body) + req.date = datetime.now() + req.content_type = 'text/plain' + + return self.call_s3api(req) + + def test_object_multi_DELETE_without_permission(self): + status, headers, body = self._test_object_multi_DELETE('test:other') + self.assertEqual(status.split()[0], '200') + elem = fromstring(body) + errors = elem.findall('Error') + self.assertEqual(len(errors), len(self.keys)) + for e in errors: + self.assertTrue(e.find('Key').text in self.keys) + self.assertEqual(e.find('Code').text, 'AccessDenied') + self.assertEqual(e.find('Message').text, 'Access Denied.') + + def test_object_multi_DELETE_with_write_permission(self): + status, headers, body = self._test_object_multi_DELETE('test:write') + self.assertEqual(status.split()[0], '200') + elem = fromstring(body) + self.assertEqual(len(elem.findall('Deleted')), len(self.keys)) + + def test_object_multi_DELETE_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_object_multi_DELETE('test:full_control') + self.assertEqual(status.split()[0], '200') + elem = fromstring(body) + self.assertEqual(len(elem.findall('Deleted')), len(self.keys)) if __name__ == '__main__': diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py index 58e4e46c6d..7a91011848 100644 --- a/test/unit/common/middleware/s3api/test_multi_upload.py +++ b/test/unit/common/middleware/s3api/test_multi_upload.py @@ -27,12 +27,11 @@ from swift.common.swob import Request from swift.common.utils import json, md5, Timestamp from test.unit import FakeMemcache, patch_policies -from test.unit.common.middleware.s3api import S3ApiTestCase +from test.unit.common.middleware.s3api import S3ApiTestCase, S3ApiTestCaseAcl from test.unit.common.middleware.s3api.helpers import UnreadableInput from swift.common.middleware.s3api.etree import fromstring, tostring from swift.common.middleware.s3api.subresource import Owner, Grant, User, \ ACL, encode_acl, decode_acl, ACLPublicRead -from test.unit.common.middleware.s3api.test_s3_acl import s3acl from swift.common.middleware.s3api.utils import sysmeta_header, mktime, \ S3Timestamp from swift.common.middleware.s3api.s3request import MAX_32BIT_INT @@ -92,10 +91,10 @@ S3_ETAG = '"%s-2"' % md5(binascii.a2b_hex( 'fedcba9876543210fedcba9876543210'), usedforsecurity=False).hexdigest() -class TestS3ApiMultiUpload(S3ApiTestCase): +class BaseS3ApiMultiUpload(object): def setUp(self): - super(TestS3ApiMultiUpload, self).setUp() + super(BaseS3ApiMultiUpload, self).setUp() self.segment_bucket = '/v1/AUTH_test/bucket+segments' self.etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' @@ -149,7 +148,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.swift.register('DELETE', self.segment_bucket + '/object/X/2', swob.HTTPNoContent, {}, None) - @s3acl def test_bucket_upload_part_missing_key(self): req = Request.blank('/bucket?partNumber=1&uploadId=x', environ={'REQUEST_METHOD': 'PUT'}, @@ -160,63 +158,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual([], self.swift.calls) self.assertNotIn('X-Backend-Storage-Policy-Index', headers) - def _do_test_bucket_upload_part_success(self, bucket_policy_index, - segment_bucket_policy_index): - self._register_bucket_policy_index_head('bucket', bucket_policy_index) - self._register_bucket_policy_index_head('bucket+segments', - segment_bucket_policy_index) - req = Request.blank('/bucket/object?partNumber=1&uploadId=X', - method='PUT', - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - with patch('swift.common.middleware.s3api.s3request.' - 'get_container_info', - lambda env, app, swift_source: {'status': 204}): - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '200 OK') - self.assertEqual([ - ('HEAD', '/v1/AUTH_test/bucket+segments/object/X'), - ('PUT', '/v1/AUTH_test/bucket+segments/object/X/1'), - ], self.swift.calls) - self.assertEqual(req.environ.get('swift.backend_path'), - '/v1/AUTH_test/bucket+segments/object/X/1') - self._assert_policy_index(req.headers, headers, - segment_bucket_policy_index) - - def test_bucket_upload_part_success(self): - self._do_test_bucket_upload_part_success(0, 0) - - def test_bucket_upload_part_success_mixed_policy(self): - self._do_test_bucket_upload_part_success(0, 1) - - def test_bucket_upload_part_v4_bad_hash(self): - authz_header = 'AWS4-HMAC-SHA256 ' + ', '.join([ - 'Credential=test:tester/%s/us-east-1/s3/aws4_request' % - self.get_v4_amz_date_header().split('T', 1)[0], - 'SignedHeaders=host;x-amz-date', - 'Signature=X', - ]) - req = Request.blank( - '/bucket/object?partNumber=1&uploadId=X', - method='PUT', - headers={'Authorization': authz_header, - 'X-Amz-Date': self.get_v4_amz_date_header(), - 'X-Amz-Content-SHA256': 'not_the_hash'}, - body=b'test') - with patch('swift.common.middleware.s3api.s3request.' - 'get_container_info', - lambda env, app, swift_source: {'status': 204}): - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '400 Bad Request') - self.assertEqual(self._get_error_code(body), 'BadDigest') - self.assertEqual([ - ('HEAD', '/v1/AUTH_test/bucket+segments/object/X'), - ('PUT', '/v1/AUTH_test/bucket+segments/object/X/1'), - ], self.swift.calls) - self.assertEqual('/v1/AUTH_test/bucket+segments/object/X/1', - req.environ.get('swift.backend_path')) - - @s3acl def test_object_multipart_uploads_list(self): req = Request.blank('/bucket/object?uploads', environ={'REQUEST_METHOD': 'GET'}, @@ -225,7 +166,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidRequest') - @s3acl def test_bucket_multipart_uploads_initiate(self): req = Request.blank('/bucket?uploads', environ={'REQUEST_METHOD': 'POST'}, @@ -234,7 +174,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidRequest') - @s3acl def test_bucket_list_parts(self): req = Request.blank('/bucket?uploadId=x', environ={'REQUEST_METHOD': 'GET'}, @@ -243,7 +182,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidRequest') - @s3acl def test_bucket_multipart_uploads_abort(self): req = Request.blank('/bucket?uploadId=x', environ={'REQUEST_METHOD': 'DELETE'}, @@ -254,7 +192,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(self._get_error_message(body), 'A key must be specified') - @s3acl def test_bucket_multipart_uploads_complete(self): req = Request.blank('/bucket?uploadId=x', environ={'REQUEST_METHOD': 'POST'}, @@ -300,60 +237,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): 'Date': self.get_date_header()}) return self.call_s3api(req) - def test_bucket_multipart_uploads_GET_paginated(self): - uploads = [ - ['object/abc'] + ['object/abc/%d' % i for i in range(1, 1000)], - ['object/def'] + ['object/def/%d' % i for i in range(1, 1000)], - ['object/ghi'] + ['object/ghi/%d' % i for i in range(1, 1000)], - ] - - objects = [ - {'name': name, 'last_modified': '2014-05-07T19:47:50.592270', - 'hash': 'HASH', 'bytes': 42} - for upload in uploads for name in upload - ] - end = 1000 - while True: - if end == 1000: - self.swift.register( - 'GET', '%s?format=json' % (self.segment_bucket), - swob.HTTPOk, {}, json.dumps(objects[:end])) - else: - self.swift.register( - 'GET', '%s?format=json&marker=%s' % ( - self.segment_bucket, objects[end - 1001]['name']), - swob.HTTPOk, {}, json.dumps(objects[end - 1000:end])) - if not objects[end - 1000:end]: - break - end += 1000 - req = Request.blank('/bucket/?uploads', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - elem = fromstring(body, 'ListMultipartUploadsResult') - self.assertEqual(elem.find('Bucket').text, 'bucket') - self.assertIsNone(elem.find('KeyMarker').text) - self.assertIsNone(elem.find('UploadIdMarker').text) - self.assertEqual(elem.find('NextUploadIdMarker').text, 'ghi') - self.assertEqual(elem.find('MaxUploads').text, '1000') - self.assertEqual(elem.find('IsTruncated').text, 'false') - self.assertEqual(len(elem.findall('Upload')), len(uploads)) - expected_uploads = [(upload[0], '2014-05-07T19:47:51.000Z') - for upload in uploads] - for u in elem.findall('Upload'): - name = u.find('Key').text + '/' + u.find('UploadId').text - initiated = u.find('Initiated').text - self.assertIn((name, initiated), expected_uploads) - self.assertEqual(u.find('Initiator/ID').text, 'test:tester') - self.assertEqual(u.find('Initiator/DisplayName').text, - 'test:tester') - self.assertEqual(u.find('Owner/ID').text, 'test:tester') - self.assertEqual(u.find('Owner/DisplayName').text, 'test:tester') - self.assertEqual(u.find('StorageClass').text, 'STANDARD') - self.assertEqual(status.split()[0], '200') - - @s3acl def test_bucket_multipart_uploads_GET(self): status, headers, body = self._test_bucket_multipart_uploads_GET() elem = fromstring(body, 'ListMultipartUploadsResult') @@ -377,7 +260,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(u.find('StorageClass').text, 'STANDARD') self.assertEqual(status.split()[0], '200') - @s3acl def test_bucket_multipart_uploads_GET_without_segment_bucket(self): segment_bucket = '/v1/AUTH_test/bucket+segments' self.swift.register('GET', segment_bucket, swob.HTTPNotFound, {}, '') @@ -399,10 +281,10 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(len(elem.findall('Upload')), 0) - @s3acl @patch('swift.common.middleware.s3api.s3request.get_container_info', lambda env, app, swift_source: {'status': 404}) def test_bucket_multipart_uploads_GET_without_bucket(self): + self.s3acl_response_modified = True self.swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNotFound, {}, '') req = Request.blank('/bucket?uploads', @@ -413,14 +295,12 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(status.split()[0], '404') self.assertEqual(self._get_error_code(body), 'NoSuchBucket') - @s3acl def test_bucket_multipart_uploads_GET_encoding_type_error(self): query = 'encoding-type=xml' status, headers, body = \ self._test_bucket_multipart_uploads_GET(query) self.assertEqual(self._get_error_code(body), 'InvalidArgument') - @s3acl def test_bucket_multipart_uploads_GET_maxuploads(self): query = 'max-uploads=2' status, headers, body = \ @@ -433,21 +313,18 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(elem.find('IsTruncated').text, 'true') self.assertEqual(status.split()[0], '200') - @s3acl def test_bucket_multipart_uploads_GET_str_maxuploads(self): query = 'max-uploads=invalid' status, headers, body = \ self._test_bucket_multipart_uploads_GET(query) self.assertEqual(self._get_error_code(body), 'InvalidArgument') - @s3acl def test_bucket_multipart_uploads_GET_negative_maxuploads(self): query = 'max-uploads=-1' status, headers, body = \ self._test_bucket_multipart_uploads_GET(query) self.assertEqual(self._get_error_code(body), 'InvalidArgument') - @s3acl def test_bucket_multipart_uploads_GET_maxuploads_over_default(self): query = 'max-uploads=1001' status, headers, body = \ @@ -460,14 +337,12 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(elem.find('IsTruncated').text, 'false') self.assertEqual(status.split()[0], '200') - @s3acl def test_bucket_multipart_uploads_GET_maxuploads_over_max_32bit_int(self): query = 'max-uploads=%s' % (MAX_32BIT_INT + 1) status, headers, body = \ self._test_bucket_multipart_uploads_GET(query) self.assertEqual(self._get_error_code(body), 'InvalidArgument') - @s3acl def test_bucket_multipart_uploads_GET_with_id_and_key_marker(self): query = 'upload-id-marker=Y&key-marker=object' multiparts = \ @@ -500,7 +375,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(query['format'], 'json') self.assertEqual(query['marker'], quote_plus('object/Y/2')) - @s3acl def test_bucket_multipart_uploads_GET_with_key_marker(self): query = 'key-marker=object' multiparts = \ @@ -539,7 +413,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(query['format'], 'json') self.assertEqual(query['marker'], quote_plus('object/Y/2')) - @s3acl def test_bucket_multipart_uploads_GET_with_prefix(self): query = 'prefix=X' multiparts = \ @@ -569,7 +442,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(query['format'], 'json') self.assertEqual(query['prefix'], 'X') - @s3acl def test_bucket_multipart_uploads_GET_with_delimiter(self): query = 'delimiter=/' multiparts = \ @@ -637,7 +509,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(query['format'], 'json') self.assertTrue(query.get('delimiter') is None) - @s3acl def test_bucket_multipart_uploads_GET_with_multi_chars_delimiter(self): query = 'delimiter=subdir' multiparts = \ @@ -698,7 +569,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(query['format'], 'json') self.assertTrue(query.get('delimiter') is None) - @s3acl def test_bucket_multipart_uploads_GET_with_prefix_and_delimiter(self): query = 'prefix=dir/&delimiter=/' multiparts = \ @@ -742,6 +612,427 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(query['prefix'], quote_plus('dir/')) self.assertTrue(query.get('delimiter') is None) + def test_object_multipart_upload_complete_error(self): + malformed_xml = 'malformed_XML' + req = Request.blank('/bucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body=malformed_xml) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'MalformedXML') + + # without target bucket + req = Request.blank('/nobucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), }, + body=XML) + with patch( + 'swift.common.middleware.s3api.s3request.get_container_info', + lambda env, app, swift_source: {'status': 404}): + self.swift.register('HEAD', '/v1/AUTH_test/nobucket', + swob.HTTPNotFound, {}, None) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'NoSuchBucket') + + def test_object_multipart_upload_abort_error(self): + req = Request.blank('/bucket/object?uploadId=invalid', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'NoSuchUpload') + + # without target bucket + req = Request.blank('/nobucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with patch( + 'swift.common.middleware.s3api.s3request.get_container_info', + lambda env, app, swift_source: {'status': 404}): + self.swift.register('HEAD', '/v1/AUTH_test/nobucket', + swob.HTTPNotFound, {}, None) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'NoSuchBucket') + + def test_object_multipart_upload_abort(self): + req = Request.blank('/bucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + + @patch('swift.common.middleware.s3api.s3request.get_container_info', + lambda env, app, swift_source: {'status': 204}) + def test_object_upload_part_error(self): + # without upload id + req = Request.blank('/bucket/object?partNumber=1', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='part object') + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + + # invalid part number + req = Request.blank('/bucket/object?partNumber=invalid&uploadId=X', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='part object') + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + + # part number must be > 0 + req = Request.blank('/bucket/object?partNumber=0&uploadId=X', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='part object') + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + + # part number must be < 10001 + req = Request.blank('/bucket/object?partNumber=10001&uploadId=X', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='part object') + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + + # without target bucket + req = Request.blank('/nobucket/object?partNumber=1&uploadId=X', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='part object') + with patch( + 'swift.common.middleware.s3api.s3request.get_container_info', + lambda env, app, swift_source: {'status': 404}): + self.swift.register('HEAD', '/v1/AUTH_test/nobucket', + swob.HTTPNotFound, {}, None) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'NoSuchBucket') + + def test_object_upload_part(self): + req = Request.blank('/bucket/object?partNumber=1&uploadId=X', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='part object') + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + + def test_object_list_parts_error(self): + req = Request.blank('/bucket/object?uploadId=invalid', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'NoSuchUpload') + + # without target bucket + req = Request.blank('/nobucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with patch( + 'swift.common.middleware.s3api.s3request.get_container_info', + lambda env, app, swift_source: {'status': 404}): + self.swift.register('HEAD', '/v1/AUTH_test/nobucket', + swob.HTTPNotFound, {}, None) + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'NoSuchBucket') + + def test_object_list_parts(self): + swift_parts = [ + {'name': 'object/X/%d' % i, + 'last_modified': '2014-05-07T19:47:%02d.592270' % (i % 60), + 'hash': hex(i), + 'bytes': 100 * i} + for i in range(1, 2000)] + ceil_last_modified = ['2014-05-07T19:%02d:%02d.000Z' + % (47 if (i + 1) % 60 else 48, (i + 1) % 60) + for i in range(1, 2000)] + swift_sorted = sorted(swift_parts, key=lambda part: part['name']) + self.swift.register('GET', + "%s?delimiter=/&format=json&marker=&" + "prefix=object/X/" % self.segment_bucket, + swob.HTTPOk, {}, json.dumps(swift_sorted)) + self.swift.register('GET', + "%s?delimiter=/&format=json&marker=object/X/999&" + "prefix=object/X/" % self.segment_bucket, + swob.HTTPOk, {}, json.dumps({})) + req = Request.blank('/bucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + elem = fromstring(body, 'ListPartsResult') + self.assertEqual(elem.find('Bucket').text, 'bucket') + self.assertEqual(elem.find('Key').text, 'object') + self.assertEqual(elem.find('UploadId').text, 'X') + self.assertEqual(elem.find('Initiator/ID').text, 'test:tester') + self.assertEqual(elem.find('Initiator/ID').text, 'test:tester') + self.assertEqual(elem.find('Owner/ID').text, 'test:tester') + self.assertEqual(elem.find('Owner/ID').text, 'test:tester') + self.assertEqual(elem.find('StorageClass').text, 'STANDARD') + self.assertEqual(elem.find('PartNumberMarker').text, '0') + self.assertEqual(elem.find('NextPartNumberMarker').text, '1000') + self.assertEqual(elem.find('MaxParts').text, '1000') + self.assertEqual(elem.find('IsTruncated').text, 'true') + self.assertEqual(len(elem.findall('Part')), 1000) + s3_parts = [] + for p in elem.findall('Part'): + partnum = int(p.find('PartNumber').text) + s3_parts.append(partnum) + self.assertEqual( + p.find('LastModified').text, + ceil_last_modified[partnum - 1]) + self.assertEqual(p.find('ETag').text.strip(), + '"%s"' % swift_parts[partnum - 1]['hash']) + self.assertEqual(p.find('Size').text, + str(swift_parts[partnum - 1]['bytes'])) + self.assertEqual(status.split()[0], '200') + self.assertEqual(s3_parts, list(range(1, 1001))) + + def _test_copy_for_s3acl(self, account, src_permission=None, + src_path='/src_bucket/src_obj', src_headers=None, + head_resp=swob.HTTPOk, put_header=None, + timestamp=None): + owner = 'test:tester' + grants = [Grant(User(account), src_permission)] \ + if src_permission else [Grant(User(owner), 'FULL_CONTROL')] + src_o_headers = encode_acl('object', ACL(Owner(owner, owner), grants)) + src_o_headers.update({'last-modified': self.last_modified}) + src_o_headers.update(src_headers or {}) + self.swift.register('HEAD', '/v1/AUTH_test/%s' % src_path.lstrip('/'), + head_resp, src_o_headers, None) + put_header = put_header or {} + put_headers = {'Authorization': 'AWS %s:hmac' % account, + 'Date': self.get_date_header(), + 'X-Amz-Copy-Source': src_path} + put_headers.update(put_header) + req = Request.blank( + '/bucket/object?partNumber=1&uploadId=X', + environ={'REQUEST_METHOD': 'PUT'}, + headers=put_headers) + timestamp = timestamp or time.time() + with patch('swift.common.middleware.s3api.utils.time.time', + return_value=timestamp): + return self.call_s3api(req) + + def test_upload_part_copy(self): + date_header = self.get_date_header() + timestamp = mktime(date_header) + last_modified = S3Timestamp(timestamp).s3xmlformat + status, headers, body = self._test_copy_for_s3acl( + 'test:tester', put_header={'Date': date_header}, + timestamp=timestamp) + self.assertEqual(status.split()[0], '200') + self.assertEqual(headers['Content-Type'], 'application/xml') + self.assertTrue(headers.get('etag') is None) + elem = fromstring(body, 'CopyPartResult') + self.assertEqual(elem.find('LastModified').text, last_modified) + self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) + + _, _, headers = self.swift.calls_with_headers[-1] + self.assertEqual(headers['X-Copy-From'], '/src_bucket/src_obj') + self.assertEqual(headers['Content-Length'], '0') + # Some headers *need* to get cleared in case we're copying from + # another multipart upload + for header in ( + 'X-Object-Sysmeta-S3api-Etag', + 'X-Object-Sysmeta-Slo-Etag', + 'X-Object-Sysmeta-Slo-Size', + 'X-Object-Sysmeta-Container-Update-Override-Etag', + 'X-Object-Sysmeta-Swift3-Etag', + ): + self.assertEqual(headers[header], '') + + def test_upload_part_copy_headers_error(self): + account = 'test:tester' + etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' + last_modified_since = 'Fri, 01 Apr 2014 12:00:00 GMT' + + header = {'X-Amz-Copy-Source-If-Match': etag} + status, header, body = \ + self._test_copy_for_s3acl(account, + head_resp=swob.HTTPPreconditionFailed, + put_header=header) + self.assertEqual(self._get_error_code(body), 'PreconditionFailed') + + header = {'X-Amz-Copy-Source-If-None-Match': etag} + status, header, body = \ + self._test_copy_for_s3acl(account, + head_resp=swob.HTTPNotModified, + put_header=header) + self.assertEqual(self._get_error_code(body), 'PreconditionFailed') + + header = {'X-Amz-Copy-Source-If-Modified-Since': last_modified_since} + status, header, body = \ + self._test_copy_for_s3acl(account, + head_resp=swob.HTTPNotModified, + put_header=header) + self.assertEqual(self._get_error_code(body), 'PreconditionFailed') + + header = \ + {'X-Amz-Copy-Source-If-Unmodified-Since': last_modified_since} + status, header, body = \ + self._test_copy_for_s3acl(account, + head_resp=swob.HTTPPreconditionFailed, + put_header=header) + self.assertEqual(self._get_error_code(body), 'PreconditionFailed') + + def _test_no_body(self, use_content_length=False, + use_transfer_encoding=False, string_to_md5=b''): + raw_md5 = md5(string_to_md5, usedforsecurity=False).digest() + content_md5 = base64.b64encode(raw_md5).strip() + with UnreadableInput(self) as fake_input: + req = Request.blank( + '/bucket/object?uploadId=X', + environ={ + 'REQUEST_METHOD': 'POST', + 'wsgi.input': fake_input}, + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-MD5': content_md5}, + body='') + if not use_content_length: + req.environ.pop('CONTENT_LENGTH') + if use_transfer_encoding: + req.environ['HTTP_TRANSFER_ENCODING'] = 'chunked' + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '400 Bad Request') + self.assertEqual(self._get_error_code(body), 'InvalidRequest') + self.assertEqual(self._get_error_message(body), + 'You must specify at least one part') + + def test_object_multi_upload_empty_body(self): + self._test_no_body() + self._test_no_body(string_to_md5=b'test') + self._test_no_body(use_content_length=True) + self._test_no_body(use_content_length=True, string_to_md5=b'test') + self._test_no_body(use_transfer_encoding=True) + self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test') + + +class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): + + def _do_test_bucket_upload_part_success(self, bucket_policy_index, + segment_bucket_policy_index): + self._register_bucket_policy_index_head('bucket', bucket_policy_index) + self._register_bucket_policy_index_head('bucket+segments', + segment_bucket_policy_index) + req = Request.blank('/bucket/object?partNumber=1&uploadId=X', + method='PUT', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with patch('swift.common.middleware.s3api.s3request.' + 'get_container_info', + lambda env, app, swift_source: {'status': 204}): + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '200 OK') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket+segments/object/X'), + ('PUT', '/v1/AUTH_test/bucket+segments/object/X/1'), + ], self.swift.calls) + self.assertEqual(req.environ.get('swift.backend_path'), + '/v1/AUTH_test/bucket+segments/object/X/1') + self._assert_policy_index(req.headers, headers, + segment_bucket_policy_index) + + def test_bucket_upload_part_success(self): + self._do_test_bucket_upload_part_success(0, 0) + + def test_bucket_upload_part_success_mixed_policy(self): + self._do_test_bucket_upload_part_success(0, 1) + + def test_bucket_upload_part_v4_bad_hash(self): + authz_header = 'AWS4-HMAC-SHA256 ' + ', '.join([ + 'Credential=test:tester/%s/us-east-1/s3/aws4_request' % + self.get_v4_amz_date_header().split('T', 1)[0], + 'SignedHeaders=host;x-amz-date', + 'Signature=X', + ]) + req = Request.blank( + '/bucket/object?partNumber=1&uploadId=X', + method='PUT', + headers={'Authorization': authz_header, + 'X-Amz-Date': self.get_v4_amz_date_header(), + 'X-Amz-Content-SHA256': 'not_the_hash'}, + body=b'test') + with patch('swift.common.middleware.s3api.s3request.' + 'get_container_info', + lambda env, app, swift_source: {'status': 204}): + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '400 Bad Request') + self.assertEqual(self._get_error_code(body), 'BadDigest') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket+segments/object/X'), + ('PUT', '/v1/AUTH_test/bucket+segments/object/X/1'), + ], self.swift.calls) + self.assertEqual('/v1/AUTH_test/bucket+segments/object/X/1', + req.environ.get('swift.backend_path')) + + def test_bucket_multipart_uploads_GET_paginated(self): + uploads = [ + ['object/abc'] + ['object/abc/%d' % i for i in range(1, 1000)], + ['object/def'] + ['object/def/%d' % i for i in range(1, 1000)], + ['object/ghi'] + ['object/ghi/%d' % i for i in range(1, 1000)], + ] + + objects = [ + {'name': name, 'last_modified': '2014-05-07T19:47:50.592270', + 'hash': 'HASH', 'bytes': 42} + for upload in uploads for name in upload + ] + end = 1000 + while True: + if end == 1000: + self.swift.register( + 'GET', '%s?format=json' % (self.segment_bucket), + swob.HTTPOk, {}, json.dumps(objects[:end])) + else: + self.swift.register( + 'GET', '%s?format=json&marker=%s' % ( + self.segment_bucket, objects[end - 1001]['name']), + swob.HTTPOk, {}, json.dumps(objects[end - 1000:end])) + if not objects[end - 1000:end]: + break + end += 1000 + req = Request.blank('/bucket/?uploads', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + elem = fromstring(body, 'ListMultipartUploadsResult') + self.assertEqual(elem.find('Bucket').text, 'bucket') + self.assertIsNone(elem.find('KeyMarker').text) + self.assertIsNone(elem.find('UploadIdMarker').text) + self.assertEqual(elem.find('NextUploadIdMarker').text, 'ghi') + self.assertEqual(elem.find('MaxUploads').text, '1000') + self.assertEqual(elem.find('IsTruncated').text, 'false') + self.assertEqual(len(elem.findall('Upload')), len(uploads)) + expected_uploads = [(upload[0], '2014-05-07T19:47:51.000Z') + for upload in uploads] + for u in elem.findall('Upload'): + name = u.find('Key').text + '/' + u.find('UploadId').text + initiated = u.find('Initiated').text + self.assertIn((name, initiated), expected_uploads) + self.assertEqual(u.find('Initiator/ID').text, 'test:tester') + self.assertEqual(u.find('Initiator/DisplayName').text, + 'test:tester') + self.assertEqual(u.find('Owner/ID').text, 'test:tester') + self.assertEqual(u.find('Owner/DisplayName').text, 'test:tester') + self.assertEqual(u.find('StorageClass').text, 'STANDARD') + self.assertEqual(status.split()[0], '200') + @patch('swift.common.middleware.s3api.controllers.' 'multi_upload.unique_id', lambda: 'X') def _test_object_multipart_upload_initiate( @@ -933,9 +1224,8 @@ class TestS3ApiMultiUpload(S3ApiTestCase): segment_bucket_policy_index=None): if segment_bucket_policy_index is None: segment_bucket_policy_index = bucket_policy_index - # mostly inlining stuff from @s3acl(s3_acl_only=True) + # N.B. this s3acl test does NOT use the common S3ApiTestAcl setUp self.s3api.conf.s3_acl = True - self.swift.s3_acl = True container_headers = encode_acl('container', ACL( Owner('test:tester', 'test:tester'), [Grant(User('test:tester'), 'FULL_CONTROL')])) @@ -1058,33 +1348,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): fake_memcache, bucket_policy_index=0, segment_bucket_policy_index=0, **kwargs) - @s3acl(s3acl_only=True) - @patch('swift.common.middleware.s3api.controllers.' - 'multi_upload.unique_id', lambda: 'X') - def test_object_multipart_upload_initiate_no_content_type(self): - req = Request.blank('/bucket/object?uploads', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Authorization': - 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'x-amz-acl': 'public-read', - 'x-amz-meta-foo': 'bar'}) - status, headers, body = self.call_s3api(req) - fromstring(body, 'InitiateMultipartUploadResult') - self.assertEqual(status.split()[0], '200') - - _, _, req_headers = self.swift.calls_with_headers[-1] - self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar') - self.assertEqual(req_headers.get( - 'X-Object-Sysmeta-S3api-Has-Content-Type'), 'no') - tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl')) - self.assertTrue(tmpacl_header) - acl_header = encode_acl('object', - ACLPublicRead(Owner('test:tester', - 'test:tester'))) - self.assertEqual(acl_header.get(sysmeta_header('object', 'acl')), - tmpacl_header) - @patch('swift.common.middleware.s3api.controllers.' 'multi_upload.unique_id', lambda: 'X') def test_object_multipart_upload_initiate_without_bucket(self): @@ -1099,31 +1362,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(status.split()[0], '404') self.assertEqual(self._get_error_code(body), 'NoSuchBucket') - @s3acl - def test_object_multipart_upload_complete_error(self): - malformed_xml = 'malformed_XML' - req = Request.blank('/bucket/object?uploadId=X', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body=malformed_xml) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'MalformedXML') - - # without target bucket - req = Request.blank('/nobucket/object?uploadId=X', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), }, - body=XML) - with patch( - 'swift.common.middleware.s3api.s3request.get_container_info', - lambda env, app, swift_source: {'status': 404}): - self.swift.register('HEAD', '/v1/AUTH_test/nobucket', - swob.HTTPNotFound, {}, None) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NoSuchBucket') - def _do_test_object_multipart_upload_complete( self, bucket_policy_index=int(POLICIES.default), segment_bucket_policy_index=None): @@ -1708,7 +1946,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual('ServiceUnavailable', self._get_error_code(body)) def test_object_multipart_upload_complete_old_content_type(self): - self.swift.register_unconditionally( + self.swift.register( 'HEAD', '/v1/AUTH_test/bucket+segments/object/X', swob.HTTPOk, {"Content-Type": "thingy/dingy"}, None) @@ -1725,7 +1963,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(headers.get('Content-Type'), 'thingy/dingy') def test_object_multipart_upload_complete_no_content_type(self): - self.swift.register_unconditionally( + self.swift.register( 'HEAD', '/v1/AUTH_test/bucket+segments/object/X', swob.HTTPOk, {"X-Object-Sysmeta-S3api-Has-Content-Type": "no"}, None) @@ -1976,204 +2214,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): h = 'X-Object-Sysmeta-Container-Update-Override-Etag' self.assertEqual(headers.get(h), override_etag) - @s3acl(s3acl_only=True) - def test_object_multipart_upload_complete_s3acl(self): - acl_headers = encode_acl('object', ACLPublicRead(Owner('test:tester', - 'test:tester'))) - headers = {} - headers[sysmeta_header('object', 'tmpacl')] = \ - acl_headers.get(sysmeta_header('object', 'acl')) - headers['X-Object-Meta-Foo'] = 'bar' - headers['Content-Type'] = 'baz/quux' - self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object/X', - swob.HTTPOk, headers, None) - req = Request.blank('/bucket/object?uploadId=X', - environ={'REQUEST_METHOD': 'POST'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body=XML) - status, headers, body = self.call_s3api(req) - fromstring(body, 'CompleteMultipartUploadResult') - self.assertEqual(status.split()[0], '200') - - _, _, headers = self.swift.calls_with_headers[-2] - self.assertEqual(headers.get('X-Object-Meta-Foo'), 'bar') - self.assertEqual(headers.get('Content-Type'), 'baz/quux') - self.assertEqual( - tostring(ACLPublicRead(Owner('test:tester', - 'test:tester')).elem()), - tostring(decode_acl('object', headers, False).elem())) - - @s3acl - def test_object_multipart_upload_abort_error(self): - req = Request.blank('/bucket/object?uploadId=invalid', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NoSuchUpload') - - # without target bucket - req = Request.blank('/nobucket/object?uploadId=X', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - with patch( - 'swift.common.middleware.s3api.s3request.get_container_info', - lambda env, app, swift_source: {'status': 404}): - self.swift.register('HEAD', '/v1/AUTH_test/nobucket', - swob.HTTPNotFound, {}, None) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NoSuchBucket') - - @s3acl - def test_object_multipart_upload_abort(self): - req = Request.blank('/bucket/object?uploadId=X', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - - @s3acl - @patch('swift.common.middleware.s3api.s3request.get_container_info', - lambda env, app, swift_source: {'status': 204}) - def test_object_upload_part_error(self): - # without upload id - req = Request.blank('/bucket/object?partNumber=1', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body='part object') - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'InvalidArgument') - - # invalid part number - req = Request.blank('/bucket/object?partNumber=invalid&uploadId=X', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body='part object') - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'InvalidArgument') - - # part number must be > 0 - req = Request.blank('/bucket/object?partNumber=0&uploadId=X', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body='part object') - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'InvalidArgument') - - # part number must be < 10001 - req = Request.blank('/bucket/object?partNumber=10001&uploadId=X', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body='part object') - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'InvalidArgument') - - # without target bucket - req = Request.blank('/nobucket/object?partNumber=1&uploadId=X', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body='part object') - with patch( - 'swift.common.middleware.s3api.s3request.get_container_info', - lambda env, app, swift_source: {'status': 404}): - self.swift.register('HEAD', '/v1/AUTH_test/nobucket', - swob.HTTPNotFound, {}, None) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NoSuchBucket') - - @s3acl - def test_object_upload_part(self): - req = Request.blank('/bucket/object?partNumber=1&uploadId=X', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}, - body='part object') - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '200') - - @s3acl - def test_object_list_parts_error(self): - req = Request.blank('/bucket/object?uploadId=invalid', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NoSuchUpload') - - # without target bucket - req = Request.blank('/nobucket/object?uploadId=X', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - with patch( - 'swift.common.middleware.s3api.s3request.get_container_info', - lambda env, app, swift_source: {'status': 404}): - self.swift.register('HEAD', '/v1/AUTH_test/nobucket', - swob.HTTPNotFound, {}, None) - status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NoSuchBucket') - - @s3acl - def test_object_list_parts(self): - swift_parts = [ - {'name': 'object/X/%d' % i, - 'last_modified': '2014-05-07T19:47:%02d.592270' % (i % 60), - 'hash': hex(i), - 'bytes': 100 * i} - for i in range(1, 2000)] - ceil_last_modified = ['2014-05-07T19:%02d:%02d.000Z' - % (47 if (i + 1) % 60 else 48, (i + 1) % 60) - for i in range(1, 2000)] - swift_sorted = sorted(swift_parts, key=lambda part: part['name']) - self.swift.register('GET', - "%s?delimiter=/&format=json&marker=&" - "prefix=object/X/" % self.segment_bucket, - swob.HTTPOk, {}, json.dumps(swift_sorted)) - self.swift.register('GET', - "%s?delimiter=/&format=json&marker=object/X/999&" - "prefix=object/X/" % self.segment_bucket, - swob.HTTPOk, {}, json.dumps({})) - req = Request.blank('/bucket/object?uploadId=X', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - elem = fromstring(body, 'ListPartsResult') - self.assertEqual(elem.find('Bucket').text, 'bucket') - self.assertEqual(elem.find('Key').text, 'object') - self.assertEqual(elem.find('UploadId').text, 'X') - self.assertEqual(elem.find('Initiator/ID').text, 'test:tester') - self.assertEqual(elem.find('Initiator/ID').text, 'test:tester') - self.assertEqual(elem.find('Owner/ID').text, 'test:tester') - self.assertEqual(elem.find('Owner/ID').text, 'test:tester') - self.assertEqual(elem.find('StorageClass').text, 'STANDARD') - self.assertEqual(elem.find('PartNumberMarker').text, '0') - self.assertEqual(elem.find('NextPartNumberMarker').text, '1000') - self.assertEqual(elem.find('MaxParts').text, '1000') - self.assertEqual(elem.find('IsTruncated').text, 'true') - self.assertEqual(len(elem.findall('Part')), 1000) - s3_parts = [] - for p in elem.findall('Part'): - partnum = int(p.find('PartNumber').text) - s3_parts.append(partnum) - self.assertEqual( - p.find('LastModified').text, - ceil_last_modified[partnum - 1]) - self.assertEqual(p.find('ETag').text.strip(), - '"%s"' % swift_parts[partnum - 1]['hash']) - self.assertEqual(p.find('Size').text, - str(swift_parts[partnum - 1]['bytes'])) - self.assertEqual(status.split()[0], '200') - self.assertEqual(s3_parts, list(range(1, 1001))) - def test_object_list_parts_encoding_type(self): self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object@@/X', swob.HTTPOk, {}, None) @@ -2344,283 +2384,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual(len(elem.findall('Part')), 2) self.assertEqual(status.split()[0], '200') - def _test_for_s3acl(self, method, query, account, hasObj=True, body=None): - path = '/bucket%s' % ('/object' + query if hasObj else query) - req = Request.blank(path, - environ={'REQUEST_METHOD': method}, - headers={'Authorization': 'AWS %s:hmac' % account, - 'Date': self.get_date_header()}, - body=body) - return self.call_s3api(req) - - @s3acl(s3acl_only=True) - def test_upload_part_acl_without_permission(self): - status, headers, body = \ - self._test_for_s3acl('PUT', '?partNumber=1&uploadId=X', - 'test:other') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_upload_part_acl_with_write_permission(self): - status, headers, body = \ - self._test_for_s3acl('PUT', '?partNumber=1&uploadId=X', - 'test:write') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_upload_part_acl_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_for_s3acl('PUT', '?partNumber=1&uploadId=X', - 'test:full_control') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_list_multipart_uploads_acl_without_permission(self): - status, headers, body = \ - self._test_for_s3acl('GET', '?uploads', 'test:other', - hasObj=False) - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_list_multipart_uploads_acl_with_read_permission(self): - status, headers, body = \ - self._test_for_s3acl('GET', '?uploads', 'test:read', - hasObj=False) - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_list_multipart_uploads_acl_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_for_s3acl('GET', '?uploads', 'test:full_control', - hasObj=False) - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - @patch('swift.common.middleware.s3api.controllers.' - 'multi_upload.unique_id', lambda: 'X') - def test_initiate_multipart_upload_acl_without_permission(self): - status, headers, body = \ - self._test_for_s3acl('POST', '?uploads', 'test:other') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - @patch('swift.common.middleware.s3api.controllers.' - 'multi_upload.unique_id', lambda: 'X') - def test_initiate_multipart_upload_acl_with_write_permission(self): - status, headers, body = \ - self._test_for_s3acl('POST', '?uploads', 'test:write') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - @patch('swift.common.middleware.s3api.controllers.' - 'multi_upload.unique_id', lambda: 'X') - def test_initiate_multipart_upload_acl_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_for_s3acl('POST', '?uploads', 'test:full_control') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_list_parts_acl_without_permission(self): - status, headers, body = \ - self._test_for_s3acl('GET', '?uploadId=X', 'test:other') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_list_parts_acl_with_read_permission(self): - status, headers, body = \ - self._test_for_s3acl('GET', '?uploadId=X', 'test:read') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_list_parts_acl_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_for_s3acl('GET', '?uploadId=X', 'test:full_control') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_abort_multipart_upload_acl_without_permission(self): - status, headers, body = \ - self._test_for_s3acl('DELETE', '?uploadId=X', 'test:other') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_abort_multipart_upload_acl_with_write_permission(self): - status, headers, body = \ - self._test_for_s3acl('DELETE', '?uploadId=X', 'test:write') - self.assertEqual(status.split()[0], '204') - - @s3acl(s3acl_only=True) - def test_abort_multipart_upload_acl_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_for_s3acl('DELETE', '?uploadId=X', 'test:full_control') - self.assertEqual(status.split()[0], '204') - self.assertEqual([ - path for method, path in self.swift.calls if method == 'DELETE' - ], [ - '/v1/AUTH_test/bucket+segments/object/X', - '/v1/AUTH_test/bucket+segments/object/X/1', - '/v1/AUTH_test/bucket+segments/object/X/2', - ]) - - @s3acl(s3acl_only=True) - def test_complete_multipart_upload_acl_without_permission(self): - status, headers, body = \ - self._test_for_s3acl('POST', '?uploadId=X', 'test:other', - body=XML) - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_complete_multipart_upload_acl_with_write_permission(self): - status, headers, body = \ - self._test_for_s3acl('POST', '?uploadId=X', 'test:write', - body=XML) - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_complete_multipart_upload_acl_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_for_s3acl('POST', '?uploadId=X', 'test:full_control', - body=XML) - self.assertEqual(status.split()[0], '200') - - def _test_copy_for_s3acl(self, account, src_permission=None, - src_path='/src_bucket/src_obj', src_headers=None, - head_resp=swob.HTTPOk, put_header=None, - timestamp=None): - owner = 'test:tester' - grants = [Grant(User(account), src_permission)] \ - if src_permission else [Grant(User(owner), 'FULL_CONTROL')] - src_o_headers = encode_acl('object', ACL(Owner(owner, owner), grants)) - src_o_headers.update({'last-modified': self.last_modified}) - src_o_headers.update(src_headers or {}) - self.swift.register('HEAD', '/v1/AUTH_test/%s' % src_path.lstrip('/'), - head_resp, src_o_headers, None) - put_header = put_header or {} - put_headers = {'Authorization': 'AWS %s:hmac' % account, - 'Date': self.get_date_header(), - 'X-Amz-Copy-Source': src_path} - put_headers.update(put_header) - req = Request.blank( - '/bucket/object?partNumber=1&uploadId=X', - environ={'REQUEST_METHOD': 'PUT'}, - headers=put_headers) - timestamp = timestamp or time.time() - with patch('swift.common.middleware.s3api.utils.time.time', - return_value=timestamp): - return self.call_s3api(req) - - @s3acl - def test_upload_part_copy(self): - date_header = self.get_date_header() - timestamp = mktime(date_header) - last_modified = S3Timestamp(timestamp).s3xmlformat - status, headers, body = self._test_copy_for_s3acl( - 'test:tester', put_header={'Date': date_header}, - timestamp=timestamp) - self.assertEqual(status.split()[0], '200') - self.assertEqual(headers['Content-Type'], 'application/xml') - self.assertTrue(headers.get('etag') is None) - elem = fromstring(body, 'CopyPartResult') - self.assertEqual(elem.find('LastModified').text, last_modified) - self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) - - _, _, headers = self.swift.calls_with_headers[-1] - self.assertEqual(headers['X-Copy-From'], '/src_bucket/src_obj') - self.assertEqual(headers['Content-Length'], '0') - # Some headers *need* to get cleared in case we're copying from - # another multipart upload - for header in ( - 'X-Object-Sysmeta-S3api-Etag', - 'X-Object-Sysmeta-Slo-Etag', - 'X-Object-Sysmeta-Slo-Size', - 'X-Object-Sysmeta-Container-Update-Override-Etag', - 'X-Object-Sysmeta-Swift3-Etag', - ): - self.assertEqual(headers[header], '') - - @s3acl(s3acl_only=True) - def test_upload_part_copy_acl_with_owner_permission(self): - status, headers, body = \ - self._test_copy_for_s3acl('test:tester') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_upload_part_copy_acl_without_permission(self): - status, headers, body = \ - self._test_copy_for_s3acl('test:other', 'READ') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_upload_part_copy_acl_with_write_permission(self): - status, headers, body = \ - self._test_copy_for_s3acl('test:write', 'READ') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_upload_part_copy_acl_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_copy_for_s3acl('test:full_control', 'READ') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_upload_part_copy_acl_without_src_permission(self): - status, headers, body = \ - self._test_copy_for_s3acl('test:write', 'WRITE') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_upload_part_copy_acl_invalid_source(self): - status, headers, body = \ - self._test_copy_for_s3acl('test:write', 'WRITE', '') - self.assertEqual(status.split()[0], '400') - - status, headers, body = \ - self._test_copy_for_s3acl('test:write', 'WRITE', '/') - self.assertEqual(status.split()[0], '400') - - status, headers, body = \ - self._test_copy_for_s3acl('test:write', 'WRITE', '/bucket') - self.assertEqual(status.split()[0], '400') - - status, headers, body = \ - self._test_copy_for_s3acl('test:write', 'WRITE', '/bucket/') - self.assertEqual(status.split()[0], '400') - - @s3acl - def test_upload_part_copy_headers_error(self): - account = 'test:tester' - etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' - last_modified_since = 'Fri, 01 Apr 2014 12:00:00 GMT' - - header = {'X-Amz-Copy-Source-If-Match': etag} - status, header, body = \ - self._test_copy_for_s3acl(account, - head_resp=swob.HTTPPreconditionFailed, - put_header=header) - self.assertEqual(self._get_error_code(body), 'PreconditionFailed') - - header = {'X-Amz-Copy-Source-If-None-Match': etag} - status, header, body = \ - self._test_copy_for_s3acl(account, - head_resp=swob.HTTPNotModified, - put_header=header) - self.assertEqual(self._get_error_code(body), 'PreconditionFailed') - - header = {'X-Amz-Copy-Source-If-Modified-Since': last_modified_since} - status, header, body = \ - self._test_copy_for_s3acl(account, - head_resp=swob.HTTPNotModified, - put_header=header) - self.assertEqual(self._get_error_code(body), 'PreconditionFailed') - - header = \ - {'X-Amz-Copy-Source-If-Unmodified-Since': last_modified_since} - status, header, body = \ - self._test_copy_for_s3acl(account, - head_resp=swob.HTTPPreconditionFailed, - put_header=header) - self.assertEqual(self._get_error_code(body), 'PreconditionFailed') - def test_upload_part_copy_headers_with_match(self): account = 'test:tester' etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' @@ -2650,35 +2413,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertTrue(headers.get('If-Match') is None) self.assertTrue(headers.get('If-Modified-Since') is None) - @s3acl(s3acl_only=True) - def test_upload_part_copy_headers_with_match_and_s3acl(self): - account = 'test:tester' - etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' - last_modified_since = 'Fri, 01 Apr 2014 11:00:00 GMT' - - header = {'X-Amz-Copy-Source-If-Match': etag, - 'X-Amz-Copy-Source-If-Modified-Since': last_modified_since} - status, header, body = \ - self._test_copy_for_s3acl(account, put_header=header) - - self.assertEqual(status.split()[0], '200') - self.assertEqual(len(self.swift.calls_with_headers), 4) - # Before the check of the copy source in the case of s3acl is valid, - # s3api check the bucket write permissions and the object existence - # of the destination. - _, _, headers = self.swift.calls_with_headers[-3] - self.assertTrue(headers.get('If-Match') is None) - self.assertTrue(headers.get('If-Modified-Since') is None) - _, _, headers = self.swift.calls_with_headers[-2] - self.assertEqual(headers['If-Match'], etag) - self.assertEqual(headers['If-Modified-Since'], last_modified_since) - _, _, headers = self.swift.calls_with_headers[-1] - self.assertTrue(headers.get('If-Match') is None) - self.assertTrue(headers.get('If-Modified-Since') is None) - _, _, headers = self.swift.calls_with_headers[0] - self.assertTrue(headers.get('If-Match') is None) - self.assertTrue(headers.get('If-Modified-Since') is None) - def test_upload_part_copy_headers_with_not_match(self): account = 'test:tester' etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' @@ -2707,35 +2441,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertTrue(headers.get('If-None-Match') is None) self.assertTrue(headers.get('If-Unmodified-Since') is None) - @s3acl(s3acl_only=True) - def test_upload_part_copy_headers_with_not_match_and_s3acl(self): - account = 'test:tester' - etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' - last_modified_since = 'Fri, 01 Apr 2014 12:00:00 GMT' - - header = {'X-Amz-Copy-Source-If-None-Match': etag, - 'X-Amz-Copy-Source-If-Unmodified-Since': last_modified_since} - status, header, body = \ - self._test_copy_for_s3acl(account, put_header=header) - - self.assertEqual(status.split()[0], '200') - self.assertEqual(len(self.swift.calls_with_headers), 4) - # Before the check of the copy source in the case of s3acl is valid, - # s3api check the bucket write permissions and the object existence - # of the destination. - _, _, headers = self.swift.calls_with_headers[-3] - self.assertTrue(headers.get('If-Match') is None) - self.assertTrue(headers.get('If-Modified-Since') is None) - _, _, headers = self.swift.calls_with_headers[-2] - self.assertEqual(headers['If-None-Match'], etag) - self.assertEqual(headers['If-Unmodified-Since'], last_modified_since) - self.assertTrue(headers.get('If-Match') is None) - self.assertTrue(headers.get('If-Modified-Since') is None) - _, _, headers = self.swift.calls_with_headers[-1] - self.assertTrue(headers.get('If-None-Match') is None) - self.assertTrue(headers.get('If-Unmodified-Since') is None) - _, _, headers = self.swift.calls_with_headers[0] - def test_upload_part_copy_range_unsatisfiable(self): account = 'test:tester' @@ -2789,40 +2494,6 @@ class TestS3ApiMultiUpload(S3ApiTestCase): self.assertEqual('bytes=0-9', put_headers['Range']) self.assertEqual('/src_bucket/src_obj', put_headers['X-Copy-From']) - def _test_no_body(self, use_content_length=False, - use_transfer_encoding=False, string_to_md5=b''): - raw_md5 = md5(string_to_md5, usedforsecurity=False).digest() - content_md5 = base64.b64encode(raw_md5).strip() - with UnreadableInput(self) as fake_input: - req = Request.blank( - '/bucket/object?uploadId=X', - environ={ - 'REQUEST_METHOD': 'POST', - 'wsgi.input': fake_input}, - headers={ - 'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'Content-MD5': content_md5}, - body='') - if not use_content_length: - req.environ.pop('CONTENT_LENGTH') - if use_transfer_encoding: - req.environ['HTTP_TRANSFER_ENCODING'] = 'chunked' - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '400 Bad Request') - self.assertEqual(self._get_error_code(body), 'InvalidRequest') - self.assertEqual(self._get_error_message(body), - 'You must specify at least one part') - - @s3acl - def test_object_multi_upload_empty_body(self): - self._test_no_body() - self._test_no_body(string_to_md5=b'test') - self._test_no_body(use_content_length=True) - self._test_no_body(use_content_length=True, string_to_md5=b'test') - self._test_no_body(use_transfer_encoding=True) - self._test_no_body(use_transfer_encoding=True, string_to_md5=b'test') - class TestS3ApiMultiUploadNonUTC(TestS3ApiMultiUpload): def setUp(self): @@ -2837,5 +2508,296 @@ class TestS3ApiMultiUploadNonUTC(TestS3ApiMultiUpload): time.tzset() +class TestS3ApiMultiUploadAcl(BaseS3ApiMultiUpload, S3ApiTestCaseAcl): + + @patch('swift.common.middleware.s3api.controllers.' + 'multi_upload.unique_id', lambda: 'X') + def test_object_multipart_upload_initiate_no_content_type(self): + req = Request.blank('/bucket/object?uploads', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': + 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'x-amz-acl': 'public-read', + 'x-amz-meta-foo': 'bar'}) + status, headers, body = self.call_s3api(req) + fromstring(body, 'InitiateMultipartUploadResult') + self.assertEqual(status.split()[0], '200') + + _, _, req_headers = self.swift.calls_with_headers[-1] + self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar') + self.assertEqual(req_headers.get( + 'X-Object-Sysmeta-S3api-Has-Content-Type'), 'no') + tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl')) + self.assertTrue(tmpacl_header) + acl_header = encode_acl('object', + ACLPublicRead(Owner('test:tester', + 'test:tester'))) + self.assertEqual(acl_header.get(sysmeta_header('object', 'acl')), + tmpacl_header) + + def test_object_multipart_upload_complete_s3acl(self): + acl_headers = encode_acl('object', ACLPublicRead(Owner('test:tester', + 'test:tester'))) + headers = {} + headers[sysmeta_header('object', 'tmpacl')] = \ + acl_headers.get(sysmeta_header('object', 'acl')) + headers['X-Object-Meta-Foo'] = 'bar' + headers['Content-Type'] = 'baz/quux' + self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object/X', + swob.HTTPOk, headers, None) + req = Request.blank('/bucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body=XML) + status, headers, body = self.call_s3api(req) + fromstring(body, 'CompleteMultipartUploadResult') + self.assertEqual(status.split()[0], '200') + + _, _, headers = self.swift.calls_with_headers[-2] + self.assertEqual(headers.get('X-Object-Meta-Foo'), 'bar') + self.assertEqual(headers.get('Content-Type'), 'baz/quux') + self.assertEqual( + tostring(ACLPublicRead(Owner('test:tester', + 'test:tester')).elem()), + tostring(decode_acl('object', headers, False).elem())) + + def _test_for_s3acl(self, method, query, account, hasObj=True, body=None): + path = '/bucket%s' % ('/object' + query if hasObj else query) + req = Request.blank(path, + environ={'REQUEST_METHOD': method}, + headers={'Authorization': 'AWS %s:hmac' % account, + 'Date': self.get_date_header()}, + body=body) + return self.call_s3api(req) + + def test_upload_part_acl_without_permission(self): + status, headers, body = \ + self._test_for_s3acl('PUT', '?partNumber=1&uploadId=X', + 'test:other') + self.assertEqual(status.split()[0], '403') + + def test_upload_part_acl_with_write_permission(self): + status, headers, body = \ + self._test_for_s3acl('PUT', '?partNumber=1&uploadId=X', + 'test:write') + self.assertEqual(status.split()[0], '200') + + def test_upload_part_acl_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_for_s3acl('PUT', '?partNumber=1&uploadId=X', + 'test:full_control') + self.assertEqual(status.split()[0], '200') + + def test_list_multipart_uploads_acl_without_permission(self): + status, headers, body = \ + self._test_for_s3acl('GET', '?uploads', 'test:other', + hasObj=False) + self.assertEqual(status.split()[0], '403') + + def test_list_multipart_uploads_acl_with_read_permission(self): + status, headers, body = \ + self._test_for_s3acl('GET', '?uploads', 'test:read', + hasObj=False) + self.assertEqual(status.split()[0], '200') + + def test_list_multipart_uploads_acl_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_for_s3acl('GET', '?uploads', 'test:full_control', + hasObj=False) + self.assertEqual(status.split()[0], '200') + + @patch('swift.common.middleware.s3api.controllers.' + 'multi_upload.unique_id', lambda: 'X') + def test_initiate_multipart_upload_acl_without_permission(self): + status, headers, body = \ + self._test_for_s3acl('POST', '?uploads', 'test:other') + self.assertEqual(status.split()[0], '403') + + @patch('swift.common.middleware.s3api.controllers.' + 'multi_upload.unique_id', lambda: 'X') + def test_initiate_multipart_upload_acl_with_write_permission(self): + status, headers, body = \ + self._test_for_s3acl('POST', '?uploads', 'test:write') + self.assertEqual(status.split()[0], '200') + + @patch('swift.common.middleware.s3api.controllers.' + 'multi_upload.unique_id', lambda: 'X') + def test_initiate_multipart_upload_acl_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_for_s3acl('POST', '?uploads', 'test:full_control') + self.assertEqual(status.split()[0], '200') + + def test_list_parts_acl_without_permission(self): + status, headers, body = \ + self._test_for_s3acl('GET', '?uploadId=X', 'test:other') + self.assertEqual(status.split()[0], '403') + + def test_list_parts_acl_with_read_permission(self): + status, headers, body = \ + self._test_for_s3acl('GET', '?uploadId=X', 'test:read') + self.assertEqual(status.split()[0], '200') + + def test_list_parts_acl_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_for_s3acl('GET', '?uploadId=X', 'test:full_control') + self.assertEqual(status.split()[0], '200') + + def test_abort_multipart_upload_acl_without_permission(self): + status, headers, body = \ + self._test_for_s3acl('DELETE', '?uploadId=X', 'test:other') + self.assertEqual(status.split()[0], '403') + + def test_abort_multipart_upload_acl_with_write_permission(self): + status, headers, body = \ + self._test_for_s3acl('DELETE', '?uploadId=X', 'test:write') + self.assertEqual(status.split()[0], '204') + + def test_abort_multipart_upload_acl_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_for_s3acl('DELETE', '?uploadId=X', 'test:full_control') + self.assertEqual(status.split()[0], '204') + self.assertEqual([ + path for method, path in self.swift.calls if method == 'DELETE' + ], [ + '/v1/AUTH_test/bucket+segments/object/X', + '/v1/AUTH_test/bucket+segments/object/X/1', + '/v1/AUTH_test/bucket+segments/object/X/2', + ]) + + def test_complete_multipart_upload_acl_without_permission(self): + status, headers, body = \ + self._test_for_s3acl('POST', '?uploadId=X', 'test:other', + body=XML) + self.assertEqual(status.split()[0], '403') + + def test_complete_multipart_upload_acl_with_write_permission(self): + status, headers, body = \ + self._test_for_s3acl('POST', '?uploadId=X', 'test:write', + body=XML) + self.assertEqual(status.split()[0], '200') + + def test_complete_multipart_upload_acl_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_for_s3acl('POST', '?uploadId=X', 'test:full_control', + body=XML) + self.assertEqual(status.split()[0], '200') + + def test_upload_part_copy_acl_with_owner_permission(self): + status, headers, body = \ + self._test_copy_for_s3acl('test:tester') + self.assertEqual(status.split()[0], '200') + + def test_upload_part_copy_acl_without_permission(self): + status, headers, body = \ + self._test_copy_for_s3acl('test:other', 'READ') + self.assertEqual(status.split()[0], '403') + + def test_upload_part_copy_acl_with_write_permission(self): + status, headers, body = \ + self._test_copy_for_s3acl('test:write', 'READ') + self.assertEqual(status.split()[0], '200') + + def test_upload_part_copy_acl_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_copy_for_s3acl('test:full_control', 'READ') + self.assertEqual(status.split()[0], '200') + + def test_upload_part_copy_acl_without_src_permission(self): + status, headers, body = \ + self._test_copy_for_s3acl('test:write', 'WRITE') + self.assertEqual(status.split()[0], '403') + + def test_upload_part_copy_acl_invalid_source(self): + self.s3acl_response_modified = True + status, headers, body = \ + self._test_copy_for_s3acl('test:write', 'WRITE', '') + self.assertEqual(status.split()[0], '400') + + status, headers, body = \ + self._test_copy_for_s3acl('test:write', 'WRITE', '/') + self.assertEqual(status.split()[0], '400') + + status, headers, body = \ + self._test_copy_for_s3acl('test:write', 'WRITE', '/bucket') + self.assertEqual(status.split()[0], '400') + + status, headers, body = \ + self._test_copy_for_s3acl('test:write', 'WRITE', '/bucket/') + self.assertEqual(status.split()[0], '400') + + def test_upload_part_copy_headers_with_match_and_s3acl(self): + account = 'test:tester' + etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' + last_modified_since = 'Fri, 01 Apr 2014 11:00:00 GMT' + + header = {'X-Amz-Copy-Source-If-Match': etag, + 'X-Amz-Copy-Source-If-Modified-Since': last_modified_since} + with self.stubbed_container_info(): + status, header, body = \ + self._test_copy_for_s3acl(account, put_header=header) + + self.assertEqual(status.split()[0], '200') + self.assertEqual(len(self.swift.calls_with_headers), 4) + # Before the check of the copy source in the case of s3acl is valid, + # s3api check the bucket write permissions and the object existence + # of the destination. + _, _, headers = self.swift.calls_with_headers[-3] + self.assertTrue(headers.get('If-Match') is None) + self.assertTrue(headers.get('If-Modified-Since') is None) + _, _, headers = self.swift.calls_with_headers[-2] + self.assertEqual(headers['If-Match'], etag) + self.assertEqual(headers['If-Modified-Since'], last_modified_since) + _, _, headers = self.swift.calls_with_headers[-1] + self.assertTrue(headers.get('If-Match') is None) + self.assertTrue(headers.get('If-Modified-Since') is None) + _, _, headers = self.swift.calls_with_headers[0] + self.assertTrue(headers.get('If-Match') is None) + self.assertTrue(headers.get('If-Modified-Since') is None) + + def test_upload_part_copy_headers_with_not_match_and_s3acl(self): + account = 'test:tester' + etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' + last_modified_since = 'Fri, 01 Apr 2014 12:00:00 GMT' + + header = {'X-Amz-Copy-Source-If-None-Match': etag, + 'X-Amz-Copy-Source-If-Unmodified-Since': last_modified_since} + with self.stubbed_container_info(): + status, header, body = \ + self._test_copy_for_s3acl(account, put_header=header) + + self.assertEqual(status.split()[0], '200') + self.assertEqual(len(self.swift.calls_with_headers), 4) + # Before the check of the copy source in the case of s3acl is valid, + # s3api check the bucket write permissions and the object existence + # of the destination. + _, _, headers = self.swift.calls_with_headers[-3] + self.assertTrue(headers.get('If-Match') is None) + self.assertTrue(headers.get('If-Modified-Since') is None) + _, _, headers = self.swift.calls_with_headers[-2] + self.assertEqual(headers['If-None-Match'], etag) + self.assertEqual(headers['If-Unmodified-Since'], last_modified_since) + self.assertTrue(headers.get('If-Match') is None) + self.assertTrue(headers.get('If-Modified-Since') is None) + _, _, headers = self.swift.calls_with_headers[-1] + self.assertTrue(headers.get('If-None-Match') is None) + self.assertTrue(headers.get('If-Unmodified-Since') is None) + _, _, headers = self.swift.calls_with_headers[0] + + +class TestS3ApiMultiUploadAclNonUTC(TestS3ApiMultiUploadAcl): + def setUp(self): + self.orig_tz = os.environ.get('TZ', '') + os.environ['TZ'] = 'EST+05EDT,M4.1.0,M10.5.0' + time.tzset() + super(TestS3ApiMultiUploadAclNonUTC, self).setUp() + + def tearDown(self): + super(TestS3ApiMultiUploadAclNonUTC, self).tearDown() + os.environ['TZ'] = self.orig_tz + time.tzset() + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/s3api/test_obj.py b/test/unit/common/middleware/s3api/test_obj.py index 6f0c64a24a..180de31a88 100644 --- a/test/unit/common/middleware/s3api/test_obj.py +++ b/test/unit/common/middleware/s3api/test_obj.py @@ -31,8 +31,7 @@ from swift.common.swob import Request from swift.common.middleware.proxy_logging import ProxyLoggingMiddleware from test.unit import mock_timestamp_now, patch_policies -from test.unit.common.middleware.s3api import S3ApiTestCase -from test.unit.common.middleware.s3api.test_s3_acl import s3acl +from test.unit.common.middleware.s3api import S3ApiTestCase, S3ApiTestCaseAcl from swift.common.middleware.s3api.s3request import SigV4Request from swift.common.middleware.s3api.subresource import ACL, User, encode_acl, \ Owner, Grant @@ -43,10 +42,10 @@ from swift.common.middleware.versioned_writes.object_versioning import \ from swift.common.utils import md5 -class TestS3ApiObj(S3ApiTestCase): +class BaseS3ApiObj(object): def setUp(self): - super(TestS3ApiObj, self).setUp() + super(BaseS3ApiObj, self).setUp() self.object_body = b'hello' self.etag = md5(self.object_body, usedforsecurity=False).hexdigest() @@ -77,9 +76,11 @@ class TestS3ApiObj(S3ApiTestCase): 'x-object-meta-something': 'oh hai'}, None) + self.bucket_policy_index = 1 + self._register_bucket_policy_index_head( + 'bucket', self.bucket_policy_index) + def _test_object_GETorHEAD(self, method): - bucket_policy_index = 1 - self._register_bucket_policy_index_head('bucket', bucket_policy_index) req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': method}, headers={'Authorization': 'AWS test:tester:hmac', @@ -87,7 +88,8 @@ class TestS3ApiObj(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') # we'll want this for logging - self._assert_policy_index(req.headers, headers, bucket_policy_index) + self._assert_policy_index(req.headers, headers, + self.bucket_policy_index) unexpected_headers = [] for key, val in self.response_headers.items(): @@ -117,7 +119,6 @@ class TestS3ApiObj(S3ApiTestCase): if method == 'GET': self.assertEqual(body, self.object_body) - @s3acl def test_object_HEAD_error(self): # HEAD does not return the body even an error response in the # specifications of the REST API. @@ -126,6 +127,7 @@ class TestS3ApiObj(S3ApiTestCase): environ={'REQUEST_METHOD': 'HEAD'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) + self.s3acl_response_modified = True self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPUnauthorized, {}, None) status, headers, body = self.call_s3api(req) @@ -182,12 +184,8 @@ class TestS3ApiObj(S3ApiTestCase): self.assertEqual(status.split()[0], '503') self.assertEqual(body, b'') # sanity - def test_object_HEAD(self): - self._test_object_GETorHEAD('HEAD') - def _do_test_object_policy_index_logging(self, bucket_policy_index): self.logger.clear() - self._register_bucket_policy_index_head('bucket', bucket_policy_index) req = Request.blank('/bucket/object', headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) @@ -203,13 +201,6 @@ class TestS3ApiObj(S3ApiTestCase): 'GET /bucket/object HTTP/1.0 200') self.assertEqual(parts[-1], str(bucket_policy_index)) - @patch_policies([ - StoragePolicy(0, 'gold', is_default=True), - StoragePolicy(1, 'silver')]) - def test_object_policy_index_logging(self): - self._do_test_object_policy_index_logging(0) - self._do_test_object_policy_index_logging(1) - def _test_object_HEAD_Range(self, range_value): req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'HEAD'}, @@ -218,7 +209,6 @@ class TestS3ApiObj(S3ApiTestCase): 'Date': self.get_date_header()}) return self.call_s3api(req) - @s3acl def test_object_HEAD_Range_with_invalid_value(self): range_value = '' status, headers, body = self._test_object_HEAD_Range(range_value) @@ -259,7 +249,6 @@ class TestS3ApiObj(S3ApiTestCase): status, headers, body = self._test_object_HEAD_Range(range_value) self.assertEqual(status.split()[0], '416') - @s3acl def test_object_HEAD_Range(self): # update response headers self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', @@ -305,8 +294,8 @@ class TestS3ApiObj(S3ApiTestCase): self.assertTrue('x-amz-meta-test' in headers) self.assertEqual('swift', headers['x-amz-meta-test']) - @s3acl def test_object_GET_error(self): + self.s3acl_response_modified = True code = self._test_method_error('GET', '/bucket/object', swob.HTTPUnauthorized) self.assertEqual(code, 'SignatureDoesNotMatch') @@ -342,40 +331,9 @@ class TestS3ApiObj(S3ApiTestCase): expected_status='429 Slow Down') self.assertEqual(code, 'SlowDown') - @s3acl def test_object_GET(self): self._test_object_GETorHEAD('GET') - @s3acl(s3acl_only=True) - def test_object_GET_with_s3acl_and_unknown_user(self): - self.swift.remote_user = None - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'GET'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status, '403 Forbidden') - self.assertEqual(self._get_error_code(body), 'SignatureDoesNotMatch') - - @s3acl(s3acl_only=True) - def test_object_GET_with_s3acl_and_keystone(self): - # for passing keystone authentication root - orig_auth = self.swift._fake_auth_middleware - calls = [] - - def wrapped_auth(env): - calls.append((env['REQUEST_METHOD'], 's3api.auth_details' in env)) - orig_auth(env) - - with patch.object(self.swift, '_fake_auth_middleware', wrapped_auth): - self._test_object_GETorHEAD('GET') - self.assertEqual(calls, [ - ('TEST', True), - ('HEAD', False), - ('GET', False), - ]) - - @s3acl def test_object_GET_Range(self): req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'GET'}, @@ -388,13 +346,6 @@ class TestS3ApiObj(S3ApiTestCase): self.assertTrue('content-range' in headers) self.assertTrue(headers['content-range'].startswith('bytes 0-3')) - @s3acl - def test_object_GET_Range_error(self): - code = self._test_method_error('GET', '/bucket/object', - swob.HTTPRequestedRangeNotSatisfiable) - self.assertEqual(code, 'InvalidRange') - - @s3acl def test_object_GET_Response(self): req = Request.blank('/bucket/object', environ={'REQUEST_METHOD': 'GET', @@ -431,7 +382,6 @@ class TestS3ApiObj(S3ApiTestCase): self.assertTrue('content-encoding' in headers) self.assertEqual(headers['content-encoding'], 'gzip') - @s3acl def test_object_GET_version_id_not_implemented(self): # GET version that is not null req = Request.blank('/bucket/object?versionId=2', @@ -457,7 +407,6 @@ class TestS3ApiObj(S3ApiTestCase): self.assertTrue('accept-ranges' in headers) self.assertEqual(headers['accept-ranges'], 'bytes') - @s3acl def test_object_GET_version_id(self): # GET current version req = Request.blank('/bucket/object?versionId=null', @@ -473,7 +422,8 @@ class TestS3ApiObj(S3ApiTestCase): environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) + with self.stubbed_container_info(versioning_enabled=True): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200', body) self.assertEqual(body, self.object_body) self.assertTrue('accept-ranges' in headers) @@ -496,7 +446,8 @@ class TestS3ApiObj(S3ApiTestCase): environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) + with self.stubbed_container_info(versioning_enabled=True): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200', body) self.assertEqual(body, b'hello1') @@ -508,20 +459,17 @@ class TestS3ApiObj(S3ApiTestCase): environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) + with self.stubbed_container_info(versioning_enabled=True): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '404') - @s3acl(versioning_enabled=False) def test_object_GET_with_version_id_but_not_enabled(self): - # Version not found - self.swift.register( - 'HEAD', '/v1/AUTH_test/bucket', - swob.HTTPNoContent, {}, None) req = Request.blank('/bucket/object?versionId=A', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) + with self.stubbed_container_info(): + status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '404') elem = fromstring(body, 'Error') self.assertEqual(elem.find('Code').text, 'NoSuchVersion') @@ -531,7 +479,6 @@ class TestS3ApiObj(S3ApiTestCase): # NB: No actual backend GET! self.assertEqual(expected_calls, self.swift.calls) - @s3acl def test_object_PUT_error(self): code = self._test_method_error('PUT', '/bucket/object', swob.HTTPUnauthorized) @@ -608,36 +555,6 @@ class TestS3ApiObj(S3ApiTestCase): {}) self.assertEqual(code, 'RequestTimeout') - def test_object_PUT_with_version(self): - self.swift.register('GET', - '/v1/AUTH_test/bucket/src_obj?version-id=foo', - swob.HTTPOk, self.response_headers, - self.object_body) - self.swift.register('PUT', '/v1/AUTH_test/bucket/object', - swob.HTTPCreated, { - 'etag': self.etag, - 'last-modified': self.last_modified, - }, None) - - req = Request.blank('/bucket/object', method='PUT', body='', headers={ - 'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'X-Amz-Copy-Source': '/bucket/src_obj?versionId=foo', - }) - status, headers, body = self.call_s3api(req) - - self.assertEqual('200 OK', status) - elem = fromstring(body, 'CopyObjectResult') - self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) - - self.assertEqual(self.swift.calls, [ - ('HEAD', '/v1/AUTH_test/bucket/src_obj?version-id=foo'), - ('PUT', '/v1/AUTH_test/bucket/object?version-id=foo'), - ]) - _, _, headers = self.swift.calls_with_headers[-1] - self.assertEqual(headers['x-copy-from'], '/bucket/src_obj') - - @s3acl def test_object_PUT(self): etag = self.response_headers['etag'] content_md5 = binascii.b2a_base64(binascii.a2b_hex(etag)).strip() @@ -663,7 +580,6 @@ class TestS3ApiObj(S3ApiTestCase): # Check that s3api converts a Content-MD5 header into an etag. self.assertEqual(headers['etag'], etag) - @s3acl def test_object_PUT_quota_exceeded(self): etag = self.response_headers['etag'] content_md5 = binascii.b2a_base64(binascii.a2b_hex(etag)).strip() @@ -688,7 +604,6 @@ class TestS3ApiObj(S3ApiTestCase): self.assertIn(b'EntityTooLarge', body) self.assertIn(b'Upload exceeds quota.') + req = Request.blank('/bucket/object', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-Type': 'foo/bar'}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual(body, b'') + + self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), + self.swift.calls) + self.assertIn(('DELETE', '/v1/AUTH_test/bucket/object' + '?multipart-manifest=delete'), + self.swift.calls) + _, path, headers = self.swift.calls_with_headers[-1] + path, query_string = path.split('?', 1) + query = {} + for q in query_string.split('&'): + key, arg = q.split('=') + query[key] = arg + self.assertEqual(query['multipart-manifest'], 'delete') + # HEAD did not indicate that it was an S3 MPU, so no async delete + self.assertNotIn('async', query) + self.assertNotIn('Content-Type', headers) + + def test_slo_object_async_DELETE(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, + {'x-static-large-object': 'True', + 'x-object-sysmeta-s3api-etag': 's3-style-etag'}, + None) + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object', + swob.HTTPNoContent, {}, '') + req = Request.blank('/bucket/object', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-Type': 'foo/bar'}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual(body, b'') + + self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), + self.swift.calls) + self.assertIn(('DELETE', '/v1/AUTH_test/bucket/object' + '?async=on&multipart-manifest=delete'), + self.swift.calls) + _, path, headers = self.swift.calls_with_headers[-1] + path, query_string = path.split('?', 1) + query = {} + for q in query_string.split('&'): + key, arg = q.split('=') + query[key] = arg + self.assertEqual(query['multipart-manifest'], 'delete') + self.assertEqual(query['async'], 'on') + self.assertNotIn('Content-Type', headers) + + def _test_set_container_permission(self, account, permission): + self.s3acl_response_modified = True + grants = [Grant(User(account), permission)] + headers = \ + encode_acl('container', + ACL(Owner('test:tester', 'test:tester'), grants)) + self.swift.register('HEAD', '/v1/AUTH_test/bucket', + swob.HTTPNoContent, headers, None) + + +class TestS3ApiObj(BaseS3ApiObj, S3ApiTestCase): + + def test_object_GET_Range_error(self): + code = self._test_method_error('GET', '/bucket/object', + swob.HTTPRequestedRangeNotSatisfiable) + self.assertEqual(code, 'InvalidRange') + + def test_object_HEAD(self): + self._test_object_GETorHEAD('HEAD') + + @patch_policies([ + StoragePolicy(0, 'gold', is_default=True), + StoragePolicy(1, 'silver')]) + def test_object_policy_index_logging(self): + self._do_test_object_policy_index_logging(self.bucket_policy_index) + self._register_bucket_policy_index_head('bucket', 0) + self._do_test_object_policy_index_logging(0) + + def test_object_PUT_with_version(self): + self.swift.register('GET', + '/v1/AUTH_test/bucket/src_obj?version-id=foo', + swob.HTTPOk, self.response_headers, + self.object_body) + self.swift.register('PUT', '/v1/AUTH_test/bucket/object', + swob.HTTPCreated, { + 'etag': self.etag, + 'last-modified': self.last_modified, + }, None) + + req = Request.blank('/bucket/object', method='PUT', body='', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'X-Amz-Copy-Source': '/bucket/src_obj?versionId=foo', + }) + status, headers, body = self.call_s3api(req) + + self.assertEqual('200 OK', status) + elem = fromstring(body, 'CopyObjectResult') + self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) + + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test/bucket/src_obj?version-id=foo'), + ('PUT', '/v1/AUTH_test/bucket/object?version-id=foo'), + ]) + _, _, headers = self.swift.calls_with_headers[-1] + self.assertEqual(headers['x-copy-from'], '/bucket/src_obj') + + def test_object_PUT_headers(self): + content_md5 = binascii.b2a_base64(binascii.a2b_hex(self.etag)).strip() + if not six.PY2: + content_md5 = content_md5.decode('ascii') + + self.swift.register('HEAD', '/v1/AUTH_test/some/source', + swob.HTTPOk, {'last-modified': self.last_modified}, + None) + req = Request.blank( + '/bucket/object', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'X-Amz-Storage-Class': 'STANDARD', + 'X-Amz-Meta-Something': 'oh hai', + 'X-Amz-Meta-Unreadable-Prefix': '\x04w', + 'X-Amz-Meta-Unreadable-Suffix': 'h\x04', + 'X-Amz-Meta-Lots-Of-Unprintable': 5 * '\x04', + 'X-Amz-Copy-Source': '/some/source', + 'Content-MD5': content_md5, + 'Date': self.get_date_header()}, + body=self.object_body) + req.date = datetime.now() + req.content_type = 'text/plain' + status, headers, body = self.call_s3api(req) + self.assertEqual('200 ', status[:4], body) + # Check that s3api does not return an etag header, + # specified copy source. + self.assertNotIn('etag', headers) + # Check that s3api does not return custom metadata in response + self.assertNotIn('x-amz-meta-something', headers) + + _, _, headers = self.swift.calls_with_headers[-1] + # Check that s3api converts a Content-MD5 header into an etag. + self.assertEqual(headers['ETag'], self.etag) + # Check that metadata is omited if no directive is specified + self.assertIsNone(headers.get('X-Object-Meta-Something')) + self.assertIsNone(headers.get('X-Object-Meta-Unreadable-Prefix')) + self.assertIsNone(headers.get('X-Object-Meta-Unreadable-Suffix')) + self.assertIsNone(headers.get('X-Object-Meta-Lots-Of-Unprintable')) + + self.assertEqual(headers['X-Copy-From'], '/some/source') + self.assertEqual(headers['Content-Length'], '0') + + @patch_policies([ + StoragePolicy(0, 'gold', is_default=True), + StoragePolicy(1, 'silver')]) + def test_simple_object_copy(self): + src_policy_index = 0 + self._register_bucket_policy_index_head('some', src_policy_index) + dst_policy_index = 1 + self._register_bucket_policy_index_head('bucket', dst_policy_index) + self.swift.register('HEAD', '/v1/AUTH_test/some/source', + swob.HTTPOk, {}, None) + req = Request.blank( + '/bucket/object', method='PUT', + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'X-Amz-Copy-Source': '/some/source', + 'Date': self.get_date_header(), + }, + ) + timestamp = time.time() + with patch('swift.common.middleware.s3api.utils.time.time', + return_value=timestamp): + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + self._assert_policy_index(req.headers, headers, dst_policy_index) + self.assertEqual('/v1/AUTH_test/bucket/object', + req.environ['swift.backend_path']) + + head_call, put_call = self.swift.calls_with_headers + self.assertNotIn('x-backend-storage-policy-index', head_call.headers) + self.assertNotIn('x-backend-storage-policy-index', put_call.headers) + self.assertEqual(put_call.headers['x-copy-from'], '/some/source') + def test_object_PUT_copy_headers_with_match(self): etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' last_modified_since = 'Fri, 01 Apr 2014 11:00:00 GMT' @@ -1126,31 +1238,6 @@ class TestS3ApiObj(S3ApiTestCase): self.assertEqual(headers['If-Match'], etag) self.assertEqual(headers['If-Modified-Since'], last_modified_since) - @s3acl(s3acl_only=True) - def test_object_PUT_copy_headers_with_match_and_s3acl(self): - etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' - last_modified_since = 'Fri, 01 Apr 2014 11:00:00 GMT' - - header = {'X-Amz-Copy-Source-If-Match': etag, - 'X-Amz-Copy-Source-If-Modified-Since': last_modified_since, - 'Date': self.get_date_header()} - status, header, body = \ - self._test_object_PUT_copy(swob.HTTPOk, header) - - self.assertEqual(status.split()[0], '200') - self.assertEqual(len(self.swift.calls_with_headers), 3) - # After the check of the copy source in the case of s3acl is valid, - # s3api check the bucket write permissions of the destination. - _, _, headers = self.swift.calls_with_headers[-2] - self.assertTrue(headers.get('If-Match') is None) - self.assertTrue(headers.get('If-Modified-Since') is None) - _, _, headers = self.swift.calls_with_headers[-1] - self.assertTrue(headers.get('If-Match') is None) - self.assertTrue(headers.get('If-Modified-Since') is None) - _, _, headers = self.swift.calls_with_headers[0] - self.assertEqual(headers['If-Match'], etag) - self.assertEqual(headers['If-Modified-Since'], last_modified_since) - def test_object_PUT_copy_headers_with_not_match(self): etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' last_modified_since = 'Fri, 01 Apr 2014 12:00:00 GMT' @@ -1170,71 +1257,6 @@ class TestS3ApiObj(S3ApiTestCase): self.assertEqual(headers['If-None-Match'], etag) self.assertEqual(headers['If-Unmodified-Since'], last_modified_since) - @s3acl(s3acl_only=True) - def test_object_PUT_copy_headers_with_not_match_and_s3acl(self): - etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' - last_modified_since = 'Fri, 01 Apr 2014 12:00:00 GMT' - - header = {'X-Amz-Copy-Source-If-None-Match': etag, - 'X-Amz-Copy-Source-If-Unmodified-Since': last_modified_since, - 'Date': self.get_date_header()} - status, header, body = \ - self._test_object_PUT_copy(swob.HTTPOk, header) - self.assertEqual(status.split()[0], '200') - # After the check of the copy source in the case of s3acl is valid, - # s3api check the bucket write permissions of the destination. - self.assertEqual(len(self.swift.calls_with_headers), 3) - _, _, headers = self.swift.calls_with_headers[-1] - self.assertTrue(headers.get('If-None-Match') is None) - self.assertTrue(headers.get('If-Unmodified-Since') is None) - _, _, headers = self.swift.calls_with_headers[0] - self.assertEqual(headers['If-None-Match'], etag) - self.assertEqual(headers['If-Unmodified-Since'], last_modified_since) - - @s3acl - def test_object_POST_error(self): - code = self._test_method_error('POST', '/bucket/object', None) - self.assertEqual(code, 'NotImplemented') - - @s3acl - def test_object_DELETE_error(self): - code = self._test_method_error('DELETE', '/bucket/object', - swob.HTTPUnauthorized) - self.assertEqual(code, 'SignatureDoesNotMatch') - code = self._test_method_error('DELETE', '/bucket/object', - swob.HTTPForbidden) - self.assertEqual(code, 'AccessDenied') - code = self._test_method_error('DELETE', '/bucket/object', - swob.HTTPServerError) - self.assertEqual(code, 'InternalError') - code = self._test_method_error('DELETE', '/bucket/object', - swob.HTTPServiceUnavailable) - self.assertEqual(code, 'ServiceUnavailable') - - with patch( - 'swift.common.middleware.s3api.s3request.get_container_info', - return_value={'status': 404}): - code = self._test_method_error('DELETE', '/bucket/object', - swob.HTTPNotFound) - self.assertEqual(code, 'NoSuchBucket') - - @s3acl - def test_object_DELETE_no_multipart(self): - self.s3api.conf.allow_multipart_uploads = False - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - - self.assertNotIn(('HEAD', '/v1/AUTH_test/bucket/object'), - self.swift.calls) - self.assertIn(('DELETE', '/v1/AUTH_test/bucket/object'), - self.swift.calls) - _, path = self.swift.calls[-1] - self.assertEqual(path.count('?'), 0) - def test_object_DELETE_old_version_id(self): self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPOk, self.response_headers, None) @@ -1246,14 +1268,8 @@ class TestS3ApiObj(S3ApiTestCase): method='DELETE', headers={ 'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) - fake_info = { - 'status': 204, - 'sysmeta': { - 'versions-container': '\x00versions\x00bucket', - } - } - with patch('swift.common.middleware.s3api.s3request.' - 'get_container_info', return_value=fake_info): + + with self.stubbed_container_info(versioning_enabled=True): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '204') self.assertEqual([ @@ -1306,20 +1322,6 @@ class TestS3ApiObj(S3ApiTestCase): '?version-id=1574341899.21751'), ], self.swift.calls) - @s3acl(versioning_enabled=False) - def test_object_DELETE_with_version_id_but_not_enabled(self): - self.swift.register('HEAD', '/v1/AUTH_test/bucket', - swob.HTTPNoContent, {}, None) - req = Request.blank('/bucket/object?versionId=1574358170.12293', - method='DELETE', headers={ - 'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - expected_calls = [] - # NB: No actual backend DELETE! - self.assertEqual(expected_calls, self.swift.calls) - def test_object_DELETE_version_id_not_implemented(self): req = Request.blank('/bucket/object?versionId=1574358170.12293', method='DELETE', headers={ @@ -1550,253 +1552,6 @@ class TestS3ApiObj(S3ApiTestCase): self.assertEqual('1574701081.61553', headers.get('x-amz-version-id')) - @s3acl - def test_object_DELETE_multipart(self): - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - - self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), - self.swift.calls) - self.assertEqual(('DELETE', '/v1/AUTH_test/bucket/object'), - self.swift.calls[-1]) - _, path = self.swift.calls[-1] - self.assertEqual(path.count('?'), 0) - - @s3acl - def test_object_DELETE_missing(self): - self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', - swob.HTTPNotFound, {}, None) - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - - self.assertEqual(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), - self.swift.calls[0]) - # the s3acl retests w/ a get_container_info HEAD @ self.swift.calls[1] - self.assertEqual(('DELETE', '/v1/AUTH_test/bucket/object'), - self.swift.calls[-1]) - - @s3acl - def test_slo_object_DELETE(self): - self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', - swob.HTTPOk, - {'x-static-large-object': 'True'}, - None) - self.swift.register('DELETE', '/v1/AUTH_test/bucket/object', - swob.HTTPOk, {}, '') - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'Content-Type': 'foo/bar'}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - self.assertEqual(body, b'') - - self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), - self.swift.calls) - self.assertIn(('DELETE', '/v1/AUTH_test/bucket/object' - '?multipart-manifest=delete'), - self.swift.calls) - _, path, headers = self.swift.calls_with_headers[-1] - path, query_string = path.split('?', 1) - query = {} - for q in query_string.split('&'): - key, arg = q.split('=') - query[key] = arg - self.assertEqual(query['multipart-manifest'], 'delete') - # HEAD did not indicate that it was an S3 MPU, so no async delete - self.assertNotIn('async', query) - self.assertNotIn('Content-Type', headers) - - @s3acl - def test_slo_object_async_DELETE(self): - self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', - swob.HTTPOk, - {'x-static-large-object': 'True', - 'x-object-sysmeta-s3api-etag': 's3-style-etag'}, - None) - self.swift.register('DELETE', '/v1/AUTH_test/bucket/object', - swob.HTTPNoContent, {}, '') - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': 'DELETE'}, - headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header(), - 'Content-Type': 'foo/bar'}) - status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '204') - self.assertEqual(body, b'') - - self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), - self.swift.calls) - self.assertIn(('DELETE', '/v1/AUTH_test/bucket/object' - '?async=on&multipart-manifest=delete'), - self.swift.calls) - _, path, headers = self.swift.calls_with_headers[-1] - path, query_string = path.split('?', 1) - query = {} - for q in query_string.split('&'): - key, arg = q.split('=') - query[key] = arg - self.assertEqual(query['multipart-manifest'], 'delete') - self.assertEqual(query['async'], 'on') - self.assertNotIn('Content-Type', headers) - - def _test_object_for_s3acl(self, method, account): - req = Request.blank('/bucket/object', - environ={'REQUEST_METHOD': method}, - headers={'Authorization': 'AWS %s:hmac' % account, - 'Date': self.get_date_header()}) - return self.call_s3api(req) - - def _test_set_container_permission(self, account, permission): - grants = [Grant(User(account), permission)] - headers = \ - encode_acl('container', - ACL(Owner('test:tester', 'test:tester'), grants)) - self.swift.register('HEAD', '/v1/AUTH_test/bucket', - swob.HTTPNoContent, headers, None) - - @s3acl(s3acl_only=True) - def test_object_GET_without_permission(self): - status, headers, body = self._test_object_for_s3acl('GET', - 'test:other') - self.assertEqual(self._get_error_code(body), 'AccessDenied') - - @s3acl(s3acl_only=True) - def test_object_GET_with_read_permission(self): - status, headers, body = self._test_object_for_s3acl('GET', - 'test:read') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_GET_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_object_for_s3acl('GET', 'test:full_control') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_PUT_without_permission(self): - status, headers, body = self._test_object_for_s3acl('PUT', - 'test:other') - self.assertEqual(self._get_error_code(body), 'AccessDenied') - - @s3acl(s3acl_only=True) - def test_object_PUT_with_owner_permission(self): - status, headers, body = self._test_object_for_s3acl('PUT', - 'test:tester') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_PUT_with_write_permission(self): - account = 'test:other' - self._test_set_container_permission(account, 'WRITE') - status, headers, body = self._test_object_for_s3acl('PUT', account) - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_PUT_with_fullcontrol_permission(self): - account = 'test:other' - self._test_set_container_permission(account, 'FULL_CONTROL') - status, headers, body = \ - self._test_object_for_s3acl('PUT', account) - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_DELETE_without_permission(self): - account = 'test:other' - status, headers, body = self._test_object_for_s3acl('DELETE', - account) - self.assertEqual(self._get_error_code(body), 'AccessDenied') - - @s3acl(s3acl_only=True) - def test_object_DELETE_with_owner_permission(self): - status, headers, body = self._test_object_for_s3acl('DELETE', - 'test:tester') - self.assertEqual(status.split()[0], '204') - - @s3acl(s3acl_only=True) - def test_object_DELETE_with_write_permission(self): - account = 'test:other' - self._test_set_container_permission(account, 'WRITE') - status, headers, body = self._test_object_for_s3acl('DELETE', - account) - self.assertEqual(status.split()[0], '204') - - @s3acl(s3acl_only=True) - def test_object_DELETE_with_fullcontrol_permission(self): - account = 'test:other' - self._test_set_container_permission(account, 'FULL_CONTROL') - status, headers, body = self._test_object_for_s3acl('DELETE', account) - self.assertEqual(status.split()[0], '204') - - def _test_object_copy_for_s3acl(self, account, src_permission=None, - src_path='/src_bucket/src_obj'): - owner = 'test:tester' - grants = [Grant(User(account), src_permission)] \ - if src_permission else [Grant(User(owner), 'FULL_CONTROL')] - src_o_headers = \ - encode_acl('object', ACL(Owner(owner, owner), grants)) - src_o_headers.update({'last-modified': self.last_modified}) - self.swift.register( - 'HEAD', join('/v1/AUTH_test', src_path.lstrip('/')), - swob.HTTPOk, src_o_headers, None) - - req = Request.blank( - '/bucket/object', - environ={'REQUEST_METHOD': 'PUT'}, - headers={'Authorization': 'AWS %s:hmac' % account, - 'X-Amz-Copy-Source': src_path, - 'Date': self.get_date_header()}) - - return self.call_s3api(req) - - @s3acl(s3acl_only=True) - def test_object_PUT_copy_with_owner_permission(self): - status, headers, body = \ - self._test_object_copy_for_s3acl('test:tester') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_PUT_copy_with_fullcontrol_permission(self): - status, headers, body = \ - self._test_object_copy_for_s3acl('test:full_control', - 'FULL_CONTROL') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_PUT_copy_with_grantee_permission(self): - status, headers, body = \ - self._test_object_copy_for_s3acl('test:write', 'READ') - self.assertEqual(status.split()[0], '200') - - @s3acl(s3acl_only=True) - def test_object_PUT_copy_without_src_obj_permission(self): - status, headers, body = \ - self._test_object_copy_for_s3acl('test:write') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_object_PUT_copy_without_dst_container_permission(self): - status, headers, body = \ - self._test_object_copy_for_s3acl('test:other', 'READ') - self.assertEqual(status.split()[0], '403') - - @s3acl(s3acl_only=True) - def test_object_PUT_copy_empty_src_path(self): - self.swift.register('PUT', '/v1/AUTH_test/bucket/object', - swob.HTTPPreconditionFailed, {}, None) - status, headers, body = self._test_object_copy_for_s3acl( - 'test:write', 'READ', src_path='') - self.assertEqual(status.split()[0], '400') - def test_cors_preflight(self): req = Request.blank( '/bucket/cors-object', @@ -1960,5 +1715,225 @@ class TestS3ApiObjNonUTC(TestS3ApiObj): time.tzset() +class TestS3ApiObjAcl(BaseS3ApiObj, S3ApiTestCaseAcl): + + def _test_object_for_s3acl(self, method, account): + req = Request.blank('/bucket/object', + environ={'REQUEST_METHOD': method}, + headers={'Authorization': 'AWS %s:hmac' % account, + 'Date': self.get_date_header()}) + return self.call_s3api(req) + + def _test_object_copy_for_s3acl(self, account, src_permission=None, + src_path='/src_bucket/src_obj'): + owner = 'test:tester' + grants = [Grant(User(account), src_permission)] \ + if src_permission else [Grant(User(owner), 'FULL_CONTROL')] + src_o_headers = \ + encode_acl('object', ACL(Owner(owner, owner), grants)) + src_o_headers.update({'last-modified': self.last_modified}) + self.swift.register( + 'HEAD', join('/v1/AUTH_test', src_path.lstrip('/')), + swob.HTTPOk, src_o_headers, None) + + req = Request.blank( + '/bucket/object', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS %s:hmac' % account, + 'X-Amz-Copy-Source': src_path, + 'Date': self.get_date_header()}) + + return self.call_s3api(req) + + def test_object_GET_with_s3acl_and_unknown_user(self): + req = Request.blank('/bucket/object', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with patch.object(self.app, 'remote_user', None): + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '403 Forbidden') + self.assertEqual(self._get_error_code(body), 'SignatureDoesNotMatch') + + def test_object_GET_with_s3acl_and_keystone(self): + # for passing keystone authentication root + orig_auth = self.app.handle + calls = [] + + def wrapped_auth(env): + calls.append((env['REQUEST_METHOD'], 's3api.auth_details' in env)) + orig_auth(env) + + with patch.object(self.app, 'handle', wrapped_auth): + self._test_object_GETorHEAD('GET') + self.assertEqual(calls, [ + ('TEST', True), + ('HEAD', False), + ('GET', False), + ]) + + def test_object_PUT_copy_headers_with_match_and_s3acl(self): + etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' + last_modified_since = 'Fri, 01 Apr 2014 11:00:00 GMT' + + header = {'X-Amz-Copy-Source-If-Match': etag, + 'X-Amz-Copy-Source-If-Modified-Since': last_modified_since, + 'Date': self.get_date_header()} + status, header, body = \ + self._test_object_PUT_copy(swob.HTTPOk, header) + + self.assertEqual(status.split()[0], '200') + self.assertEqual(len(self.swift.calls_with_headers), 3) + # After the check of the copy source in the case of s3acl is valid, + # s3api check the bucket write permissions of the destination. + _, _, headers = self.swift.calls_with_headers[-2] + self.assertTrue(headers.get('If-Match') is None) + self.assertTrue(headers.get('If-Modified-Since') is None) + _, _, headers = self.swift.calls_with_headers[-1] + self.assertTrue(headers.get('If-Match') is None) + self.assertTrue(headers.get('If-Modified-Since') is None) + _, _, headers = self.swift.calls_with_headers[0] + self.assertEqual(headers['If-Match'], etag) + self.assertEqual(headers['If-Modified-Since'], last_modified_since) + + def test_object_PUT_copy_headers_with_not_match_and_s3acl(self): + etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1' + last_modified_since = 'Fri, 01 Apr 2014 12:00:00 GMT' + + header = {'X-Amz-Copy-Source-If-None-Match': etag, + 'X-Amz-Copy-Source-If-Unmodified-Since': last_modified_since, + 'Date': self.get_date_header()} + status, header, body = \ + self._test_object_PUT_copy(swob.HTTPOk, header) + self.assertEqual(status.split()[0], '200') + # After the check of the copy source in the case of s3acl is valid, + # s3api check the bucket write permissions of the destination. + self.assertEqual(len(self.swift.calls_with_headers), 3) + _, _, headers = self.swift.calls_with_headers[-1] + self.assertTrue(headers.get('If-None-Match') is None) + self.assertTrue(headers.get('If-Unmodified-Since') is None) + _, _, headers = self.swift.calls_with_headers[0] + self.assertEqual(headers['If-None-Match'], etag) + self.assertEqual(headers['If-Unmodified-Since'], last_modified_since) + + def test_object_GET_Range_error(self): + # needed for pre-flight ACL HEAD request, FakeSwift finds the 416 + # for the GET and returns it for HEAD but s3api won't error + # correctly to 416 on HEAD + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, {}, None), + code = self._test_method_error('GET', '/bucket/object', + swob.HTTPRequestedRangeNotSatisfiable) + self.assertEqual(code, 'InvalidRange') + + def test_object_GET_without_permission(self): + status, headers, body = self._test_object_for_s3acl('GET', + 'test:other') + self.assertEqual(self._get_error_code(body), 'AccessDenied') + + def test_object_GET_with_read_permission(self): + status, headers, body = self._test_object_for_s3acl('GET', + 'test:read') + self.assertEqual(status.split()[0], '200') + + def test_object_GET_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_object_for_s3acl('GET', 'test:full_control') + self.assertEqual(status.split()[0], '200') + + def test_object_PUT_without_permission(self): + status, headers, body = self._test_object_for_s3acl('PUT', + 'test:other') + self.assertEqual(self._get_error_code(body), 'AccessDenied') + + def test_object_PUT_with_owner_permission(self): + status, headers, body = self._test_object_for_s3acl('PUT', + 'test:tester') + self.assertEqual(status.split()[0], '200') + + def test_object_PUT_with_write_permission(self): + account = 'test:other' + self._test_set_container_permission(account, 'WRITE') + status, headers, body = self._test_object_for_s3acl('PUT', account) + self.assertEqual(status.split()[0], '200') + + def test_object_PUT_with_fullcontrol_permission(self): + account = 'test:other' + self._test_set_container_permission(account, 'FULL_CONTROL') + status, headers, body = \ + self._test_object_for_s3acl('PUT', account) + self.assertEqual(status.split()[0], '200') + + def test_object_DELETE_without_permission(self): + account = 'test:other' + status, headers, body = self._test_object_for_s3acl('DELETE', + account) + self.assertEqual(self._get_error_code(body), 'AccessDenied') + + def test_object_DELETE_with_owner_permission(self): + status, headers, body = self._test_object_for_s3acl('DELETE', + 'test:tester') + self.assertEqual(status.split()[0], '204') + + def test_object_DELETE_with_write_permission(self): + account = 'test:other' + self._test_set_container_permission(account, 'WRITE') + status, headers, body = self._test_object_for_s3acl('DELETE', + account) + self.assertEqual(status.split()[0], '204') + + def test_object_DELETE_with_fullcontrol_permission(self): + account = 'test:other' + self._test_set_container_permission(account, 'FULL_CONTROL') + status, headers, body = self._test_object_for_s3acl('DELETE', account) + self.assertEqual(status.split()[0], '204') + + def test_object_PUT_copy_with_owner_permission(self): + status, headers, body = \ + self._test_object_copy_for_s3acl('test:tester') + self.assertEqual(status.split()[0], '200') + + def test_object_PUT_copy_with_fullcontrol_permission(self): + status, headers, body = \ + self._test_object_copy_for_s3acl('test:full_control', + 'FULL_CONTROL') + self.assertEqual(status.split()[0], '200') + + def test_object_PUT_copy_with_grantee_permission(self): + status, headers, body = \ + self._test_object_copy_for_s3acl('test:write', 'READ') + self.assertEqual(status.split()[0], '200') + + def test_object_PUT_copy_without_src_obj_permission(self): + status, headers, body = \ + self._test_object_copy_for_s3acl('test:write') + self.assertEqual(status.split()[0], '403') + + def test_object_PUT_copy_without_dst_container_permission(self): + status, headers, body = \ + self._test_object_copy_for_s3acl('test:other', 'READ') + self.assertEqual(status.split()[0], '403') + + def test_object_PUT_copy_empty_src_path(self): + self.swift.register('PUT', '/v1/AUTH_test/bucket/object', + swob.HTTPPreconditionFailed, {}, None) + status, headers, body = self._test_object_copy_for_s3acl( + 'test:write', 'READ', src_path='') + self.assertEqual(status.split()[0], '400') + + +class TestS3ApiObjNonUTCAcl(TestS3ApiObjAcl): + def setUp(self): + self.orig_tz = os.environ.get('TZ', '') + os.environ['TZ'] = 'EST+05EDT,M4.1.0,M10.5.0' + time.tzset() + super(TestS3ApiObjNonUTCAcl, self).setUp() + + def tearDown(self): + super(TestS3ApiObjNonUTCAcl, self).tearDown() + os.environ['TZ'] = self.orig_tz + time.tzset() + + if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/s3api/test_s3_acl.py b/test/unit/common/middleware/s3api/test_s3_acl.py index 3c4eeae2fd..0bd2af4be8 100644 --- a/test/unit/common/middleware/s3api/test_s3_acl.py +++ b/test/unit/common/middleware/s3api/test_s3_acl.py @@ -14,94 +14,17 @@ # limitations under the License. import unittest -import functools -import sys -import traceback -from mock import patch, MagicMock -from swift.common import swob from swift.common.swob import Request -from swift.common.utils import json - from swift.common.middleware.s3api.etree import tostring, Element, SubElement from swift.common.middleware.s3api.subresource import ACL, ACLPrivate, User, \ - encode_acl, AuthenticatedUsers, AllUsers, Owner, Grant, PERMISSIONS -from test.unit.common.middleware.s3api.test_s3api import S3ApiTestCase -from test.unit.common.middleware.s3api.exceptions import NotMethodException -from test.unit.common.middleware.s3api import FakeSwift + Owner, Grant +from test.unit.common.middleware.s3api import S3ApiTestCaseAcl XMLNS_XSI = 'http://www.w3.org/2001/XMLSchema-instance' -def s3acl(func=None, s3acl_only=False, versioning_enabled=True): - """ - NOTE: s3acl decorator needs an instance of s3api testing framework. - (i.e. An instance for first argument is necessary) - """ - if func is None: - return functools.partial( - s3acl, - s3acl_only=s3acl_only, - versioning_enabled=versioning_enabled) - - @functools.wraps(func) - def s3acl_decorator(*args, **kwargs): - if not args and not kwargs: - raise NotMethodException('Use s3acl decorator for a method') - - def call_func(failing_point=''): - try: - # For maintainability, we patch 204 status for every - # get_container_info. if you want, we can rewrite the - # statement easily with nested decorator like as: - # - # @s3acl - # @patch(xxx) - # def test_xxxx(self) - - fake_info = {'status': 204} - if versioning_enabled: - fake_info['sysmeta'] = { - 'versions-container': '\x00versions\x00bucket', - } - - with patch('swift.common.middleware.s3api.s3request.' - 'get_container_info', return_value=fake_info): - func(*args, **kwargs) - except AssertionError: - # Make traceback message to clarify the assertion - exc_type, exc_instance, exc_traceback = sys.exc_info() - formatted_traceback = ''.join(traceback.format_tb( - exc_traceback)) - message = '\n%s\n%s' % (formatted_traceback, - exc_type.__name__) - if exc_instance.args: - message += ':\n%s' % (exc_instance.args[0],) - message += failing_point - raise exc_type(message) - - instance = args[0] - - if not s3acl_only: - call_func() - instance.swift._calls = [] - - instance.s3api.conf.s3_acl = True - instance.swift.s3_acl = True - owner = Owner('test:tester', 'test:tester') - generate_s3acl_environ('test', instance.swift, owner) - call_func(' (fail at s3_acl)') - - return s3acl_decorator - - -def _gen_test_headers(owner, grants=[], resource='container'): - if not grants: - grants = [Grant(User('test:tester'), 'FULL_CONTROL')] - return encode_acl(resource, ACL(owner, grants)) - - def _make_xml(grantee): owner = 'test:tester' permission = 'READ' @@ -116,69 +39,7 @@ def _make_xml(grantee): return tostring(elem) -def generate_s3acl_environ(account, swift, owner): - - def gen_grant(permission): - # generate Grant with a grantee named by "permission" - account_name = '%s:%s' % (account, permission.lower()) - return Grant(User(account_name), permission) - - grants = [gen_grant(perm) for perm in PERMISSIONS] - container_headers = _gen_test_headers(owner, grants) - object_headers = _gen_test_headers(owner, grants, 'object') - object_body = 'hello' - object_headers['Content-Length'] = len(object_body) - - # TEST method is used to resolve a tenant name - swift.register('TEST', '/v1/AUTH_test', swob.HTTPMethodNotAllowed, - {}, None) - swift.register('TEST', '/v1/AUTH_X', swob.HTTPMethodNotAllowed, - {}, None) - - # for bucket - swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - container_headers, None) - swift.register('HEAD', '/v1/AUTH_test/bucket+segments', swob.HTTPNoContent, - container_headers, None) - swift.register('PUT', '/v1/AUTH_test/bucket', - swob.HTTPCreated, {}, None) - swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPNoContent, - container_headers, json.dumps([])) - swift.register('POST', '/v1/AUTH_test/bucket', - swob.HTTPNoContent, {}, None) - swift.register('DELETE', '/v1/AUTH_test/bucket', - swob.HTTPNoContent, {}, None) - - # necessary for canned-acl tests - public_headers = _gen_test_headers(owner, [Grant(AllUsers(), 'READ')]) - swift.register('GET', '/v1/AUTH_test/public', swob.HTTPNoContent, - public_headers, json.dumps([])) - authenticated_headers = _gen_test_headers( - owner, [Grant(AuthenticatedUsers(), 'READ')], 'bucket') - swift.register('GET', '/v1/AUTH_test/authenticated', - swob.HTTPNoContent, authenticated_headers, - json.dumps([])) - - # for object - swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPOk, - object_headers, None) - - -class TestS3ApiS3Acl(S3ApiTestCase): - - def setUp(self): - super(TestS3ApiS3Acl, self).setUp() - - self.s3api.conf.s3_acl = True - self.swift.s3_acl = True - - account = 'test' - owner_name = '%s:tester' % account - self.default_owner = Owner(owner_name, owner_name) - generate_s3acl_environ(account, self.swift, self.default_owner) - - def tearDown(self): - self.s3api.conf.s3_acl = False +class TestS3ApiS3Acl(S3ApiTestCaseAcl): def test_bucket_acl_PUT_with_other_owner(self): req = Request.blank('/bucket?acl', @@ -521,42 +382,6 @@ class TestS3ApiS3Acl(S3ApiTestCase): status, headers, body = self._test_object_acl_PUT('test:tester') self.assertEqual(status.split()[0], '200') - def test_s3acl_decorator(self): - @s3acl - def non_class_s3acl_error(): - raise TypeError() - - class FakeClass(object): - def __init__(self): - self.s3api = MagicMock() - self.swift = FakeSwift() - - @s3acl - def s3acl_error(self): - raise TypeError() - - @s3acl - def s3acl_assert_fail(self): - assert False - - @s3acl(s3acl_only=True) - def s3acl_s3only_error(self): - if self.s3api.conf.s3_acl: - raise TypeError() - - @s3acl(s3acl_only=True) - def s3acl_s3only_no_error(self): - if not self.s3api.conf.s3_acl: - raise TypeError() - - fake_class = FakeClass() - - self.assertRaises(NotMethodException, non_class_s3acl_error) - self.assertRaises(TypeError, fake_class.s3acl_error) - self.assertRaises(AssertionError, fake_class.s3acl_assert_fail) - self.assertRaises(TypeError, fake_class.s3acl_s3only_error) - self.assertIsNone(fake_class.s3acl_s3only_no_error()) - if __name__ == '__main__': unittest.main() diff --git a/test/unit/common/middleware/s3api/test_s3api.py b/test/unit/common/middleware/s3api/test_s3api.py index f7473253a6..a547505d1c 100644 --- a/test/unit/common/middleware/s3api/test_s3api.py +++ b/test/unit/common/middleware/s3api/test_s3api.py @@ -40,7 +40,7 @@ from keystoneauth1.access import AccessInfoV2 from test.debug_logger import debug_logger, FakeStatsdClient from test.unit.common.middleware.s3api import S3ApiTestCase -from test.unit.common.middleware.s3api.helpers import FakeSwift +from test.unit.common.middleware.helpers import FakeSwift from test.unit.common.middleware.s3api.test_s3token import \ GOOD_RESPONSE_V2, GOOD_RESPONSE_V3 from swift.common.middleware.s3api.s3request import SigV4Request, S3Request @@ -1440,7 +1440,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): self.s3api.logger.logger.statsd_client.get_increment_counts()) def test_s3api_with_only_s3_token(self): - self.swift = FakeSwift() + self.swift = FakeSwift(allowed_methods=['TEST']) self.keystone_auth = KeystoneAuth( self.swift, {'operator_roles': 'swift-user'}) self.s3_token = S3Token( @@ -1470,7 +1470,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): req.environ['swift.backend_path']) def test_s3api_with_only_s3_token_v3(self): - self.swift = FakeSwift() + self.swift = FakeSwift(allowed_methods=['TEST']) self.keystone_auth = KeystoneAuth( self.swift, {'operator_roles': 'swift-user'}) self.s3_token = S3Token( @@ -1500,7 +1500,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): req.environ['swift.backend_path']) def test_s3api_with_s3_token_and_auth_token(self): - self.swift = FakeSwift() + self.swift = FakeSwift(allowed_methods=['TEST']) self.keystone_auth = KeystoneAuth( self.swift, {'operator_roles': 'swift-user'}) self.auth_token = AuthProtocol( @@ -1555,7 +1555,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): statsd_client.get_increment_counts()) def test_s3api_with_only_s3_token_in_s3acl(self): - self.swift = FakeSwift() + self.swift = FakeSwift(allowed_methods=['TEST']) self.keystone_auth = KeystoneAuth( self.swift, {'operator_roles': 'swift-user'}) self.s3_token = S3Token( diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py index 78689af332..7391b35f82 100644 --- a/test/unit/common/middleware/s3api/test_s3request.py +++ b/test/unit/common/middleware/s3api/test_s3request.py @@ -95,7 +95,6 @@ class TestRequest(S3ApiTestCase): def setUp(self): super(TestRequest, self).setUp() self.s3api.conf.s3_acl = True - self.swift.s3_acl = True @patch('swift.common.middleware.s3api.acl_handlers.ACL_MAP', Fake_ACL_MAP) @patch('swift.common.middleware.s3api.s3request.S3AclRequest.authenticate', @@ -122,7 +121,6 @@ class TestRequest(S3ApiTestCase): def test_get_response_without_s3_acl(self): self.s3api.conf.s3_acl = False - self.swift.s3_acl = False mock_get_resp, m_check_permission, s3_resp = \ self._test_get_response('HEAD') self.assertFalse(hasattr(s3_resp, 'bucket_acl')) @@ -1005,7 +1003,6 @@ class TestSigV4Request(S3ApiTestCase): def setUp(self): super(TestSigV4Request, self).setUp() self.s3api.conf.s3_acl = True - self.swift.s3_acl = True def test_init_header_authorization(self): environ = { diff --git a/test/unit/common/middleware/s3api/test_service.py b/test/unit/common/middleware/s3api/test_service.py index fbb999dc35..799da683d9 100644 --- a/test/unit/common/middleware/s3api/test_service.py +++ b/test/unit/common/middleware/s3api/test_service.py @@ -19,8 +19,7 @@ from swift.common import swob from swift.common.swob import Request from swift.common.utils import json -from test.unit.common.middleware.s3api.test_s3_acl import s3acl -from test.unit.common.middleware.s3api import S3ApiTestCase +from test.unit.common.middleware.s3api import S3ApiTestCase, S3ApiTestCaseAcl from swift.common.middleware.s3api.etree import fromstring from swift.common.middleware.s3api.subresource import ACL, Owner, encode_acl @@ -36,7 +35,7 @@ def create_bucket_list_json(buckets): return json.dumps(bucket_list) -class TestS3ApiService(S3ApiTestCase): +class BaseS3ApiService(object): def setup_buckets(self): self.buckets = (('apple', 1, 200), ('orange', 3, 430)) bucket_list = create_bucket_list_json(self.buckets) @@ -44,22 +43,10 @@ class TestS3ApiService(S3ApiTestCase): bucket_list) def setUp(self): - super(TestS3ApiService, self).setUp() + super(BaseS3ApiService, self).setUp() self.setup_buckets() - def test_service_GET_error(self): - code = self._test_method_error( - 'GET', '', swob.HTTPUnauthorized, expected_xml_tags=( - 'Code', 'Message', 'AWSAccessKeyId', 'StringToSign', - 'StringToSignBytes', 'SignatureProvided')) - self.assertEqual(code, 'SignatureDoesNotMatch') - code = self._test_method_error('GET', '', swob.HTTPForbidden) - self.assertEqual(code, 'AccessDenied') - code = self._test_method_error('GET', '', swob.HTTPServerError) - self.assertEqual(code, 'InternalError') - - @s3acl def test_service_GET(self): req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, @@ -83,7 +70,6 @@ class TestS3ApiService(S3ApiTestCase): for i in self.buckets: self.assertTrue(i[0] in names) - @s3acl def test_service_GET_subresource(self): req = Request.blank('/?acl', environ={'REQUEST_METHOD': 'GET'}, @@ -107,6 +93,20 @@ class TestS3ApiService(S3ApiTestCase): for i in self.buckets: self.assertTrue(i[0] in names) + +class TestS3ApiServiceNoAcl(BaseS3ApiService, S3ApiTestCase): + + def test_service_GET_error(self): + code = self._test_method_error( + 'GET', '', swob.HTTPUnauthorized, expected_xml_tags=( + 'Code', 'Message', 'AWSAccessKeyId', 'StringToSign', + 'StringToSignBytes', 'SignatureProvided')) + self.assertEqual(code, 'SignatureDoesNotMatch') + code = self._test_method_error('GET', '', swob.HTTPForbidden) + self.assertEqual(code, 'AccessDenied') + code = self._test_method_error('GET', '', swob.HTTPServerError) + self.assertEqual(code, 'InternalError') + def test_service_GET_with_blind_resource(self): buckets = (('apple', 1, 200), ('orange', 3, 430), ('apple+segment', 1, 200)) @@ -137,6 +137,9 @@ class TestS3ApiService(S3ApiTestCase): for i in expected: self.assertIn(i[0], names) + +class TestS3ApiServiceAcl(BaseS3ApiService, S3ApiTestCaseAcl): + def _test_service_GET_for_check_bucket_owner(self, buckets): self.s3api.conf.check_bucket_owner = True bucket_list = create_bucket_list_json(buckets) @@ -149,7 +152,6 @@ class TestS3ApiService(S3ApiTestCase): 'Date': self.get_date_header()}) return self.call_s3api(req) - @s3acl(s3acl_only=True) def test_service_GET_without_bucket(self): bucket_list = [] for var in range(0, 10): @@ -168,7 +170,6 @@ class TestS3ApiService(S3ApiTestCase): buckets = resp_buckets.iterchildren('Bucket') self.assertEqual(len(list(buckets)), 0) - @s3acl(s3acl_only=True) def test_service_GET_without_owner_bucket(self): bucket_list = [] for var in range(0, 10): @@ -190,7 +191,6 @@ class TestS3ApiService(S3ApiTestCase): buckets = resp_buckets.iterchildren('Bucket') self.assertEqual(len(list(buckets)), 0) - @s3acl(s3acl_only=True) def test_service_GET_bucket_list(self): bucket_list = [] for var in range(0, 10): diff --git a/test/unit/common/middleware/test_helpers.py b/test/unit/common/middleware/test_helpers.py index 64c115b577..cf055f36f4 100644 --- a/test/unit/common/middleware/test_helpers.py +++ b/test/unit/common/middleware/test_helpers.py @@ -16,12 +16,40 @@ import unittest from swift.common.storage_policy import POLICIES -from swift.common.swob import Request, HTTPOk, HTTPNotFound, HTTPCreated +from swift.common.swob import Request, HTTPOk, HTTPNotFound, \ + HTTPCreated, HeaderKeyDict, HTTPException from swift.common import request_helpers as rh +from swift.common.middleware.s3api.utils import sysmeta_header from test.unit.common.middleware.helpers import FakeSwift class TestFakeSwift(unittest.TestCase): + def test_allowed_methods(self): + + def assert_allowed(swift, method): + path = '/v1/a/c/o' + swift.register(method, path, HTTPOk, {}, None) + req = Request.blank(path) + req.method = method + self.assertEqual(200, req.get_response(swift).status_int) + + def assert_disallowed(swift, method): + path = '/v1/a/c/o' + swift.register(method, path, HTTPOk, {}, None) + req = Request.blank(path) + req.method = method + with self.assertRaises(HTTPException) as cm: + req.get_response(swift) + self.assertEqual(501, cm.exception.status_int) + + for method in ('PUT', 'POST', 'DELETE', 'GET', 'HEAD', 'OPTIONS', + 'REPLICATE', 'SSYNC', 'UPDATE'): + assert_allowed(FakeSwift(), method) + assert_allowed(FakeSwift(allowed_methods=['TEST']), 'TEST') + + assert_disallowed(FakeSwift(), 'TEST') + assert_allowed(FakeSwift(allowed_methods=['TEST']), 'TEST') + def test_not_registered(self): swift = FakeSwift() @@ -692,3 +720,114 @@ class TestFakeSwiftMultipleResponses(unittest.TestCase): resp = req.get_response(swift) self.assertEqual(200, resp.status_int) self.assertEqual('Baz', resp.headers['X-Foo']) + + +class TestFakeSwiftStickyHeaders(unittest.TestCase): + def setUp(self): + self.swift = FakeSwift() + self.path = '/v1/AUTH_test/bucket' + + def _check_headers(self, method, path, exp_headers): + captured_headers = {} + + def start_response(status, resp_headers): + self.assertEqual(status, '200 OK') + captured_headers.update(resp_headers) + + env = {'REQUEST_METHOD': method, 'PATH_INFO': path} + body_iter = self.swift(env, start_response) + b''.join(body_iter) + captured_headers.pop('Content-Type') + self.assertEqual(exp_headers, captured_headers) + + def test_sticky_headers(self): + sticky_headers = HeaderKeyDict({ + sysmeta_header('container', 'acl'): 'test', + 'x-container-meta-foo': 'bar', + }) + self.swift.update_sticky_response_headers(self.path, sticky_headers) + # register a response for this path with no headers + self.swift.register('GET', self.path, HTTPOk, {}, None) + self._check_headers('HEAD', self.path, sticky_headers) + self._check_headers('GET', self.path, sticky_headers) + + # sticky headers are not applied to PUT, POST, DELETE + self.swift.register('PUT', self.path, HTTPOk, {}, None) + self._check_headers('PUT', self.path, {}) + self.swift.register('POST', self.path, HTTPOk, {}, None) + self._check_headers('POST', self.path, {}) + self.swift.register('DELETE', self.path, HTTPOk, {}, None) + self._check_headers('DELETE', self.path, {}) + + def test_sticky_headers_match_path(self): + other_path = self.path + '-other' + sticky_headers = HeaderKeyDict({ + sysmeta_header('container', 'acl'): 'test', + 'x-container-meta-foo': 'bar', + }) + sticky_headers_other = HeaderKeyDict({ + 'x-container-meta-foo': 'other', + }) + self.swift.update_sticky_response_headers(self.path, sticky_headers) + self.swift.update_sticky_response_headers(other_path, + sticky_headers_other) + self.swift.register('GET', self.path, HTTPOk, {}, None) + self.swift.register('GET', other_path, HTTPOk, {}, None) + self._check_headers('HEAD', self.path, sticky_headers) + self._check_headers('GET', other_path, sticky_headers_other) + + def test_sticky_headers_update(self): + sticky_headers = HeaderKeyDict({ + sysmeta_header('container', 'acl'): 'test', + 'x-container-meta-foo': 'bar' + }) + exp_headers = sticky_headers.copy() + self.swift.update_sticky_response_headers(self.path, sticky_headers) + self.swift.register('HEAD', self.path, HTTPOk, {}, None) + self._check_headers('HEAD', self.path, exp_headers) + + # check that FakeSwift made a *copy* + sticky_headers['x-container-meta-foo'] = 'changed' + self._check_headers('HEAD', self.path, exp_headers) + + # check existing are updated not replaced + sticky_headers = HeaderKeyDict({ + sysmeta_header('container', 'acl'): 'test-modified', + 'x-container-meta-bar': 'foo' + }) + exp_headers.update(sticky_headers) + self.swift.update_sticky_response_headers(self.path, sticky_headers) + self._check_headers('HEAD', self.path, exp_headers) + + def test_sticky_headers_add_to_response_headers(self): + sticky_headers = HeaderKeyDict({ + 'x-container-meta-foo': 'bar', + }) + self.swift.update_sticky_response_headers(self.path, sticky_headers) + # register a response with another header + self.swift.register('HEAD', self.path, HTTPOk, { + 'x-backend-storage-policy-index': '1', + }, None) + self._check_headers('HEAD', self.path, HeaderKeyDict({ + 'x-container-meta-foo': 'bar', + 'x-backend-storage-policy-index': '1', + })) + + def test_sticky_headers_overwritten_by_response_header(self): + sticky_headers = HeaderKeyDict({ + 'x-container-meta-foo': 'bar', + 'x-backend-storage-policy-index': '0', + }) + self.swift.update_sticky_response_headers(self.path, sticky_headers) + # register a response with a different value for a sticky header + self.swift.register('HEAD', self.path, HTTPOk, { + 'x-container-meta-foo': 'different', + }, None) + self._check_headers('HEAD', self.path, HeaderKeyDict({ + 'x-container-meta-foo': 'different', + 'x-backend-storage-policy-index': '0', + })) + + +if __name__ == '__main__': + unittest.main()