From 46e7da97c6e620ba2998872901ed44055714e699 Mon Sep 17 00:00:00 2001 From: indianwhocodes Date: Mon, 11 Sep 2023 14:03:01 -0700 Subject: [PATCH] s3api: Support GET/HEAD request with ?partNumber Co-Authored-By: Alistair Coles Co-Authored-By: Clay Gerrard Closes-Bug: #1735284 Change-Id: Ib396309c706fbc6bc419377fe23fcf5603a89f45 --- swift/common/middleware/s3api/acl_handlers.py | 9 + .../s3api/controllers/multi_upload.py | 10 +- .../middleware/s3api/controllers/obj.py | 21 +- swift/common/middleware/s3api/s3request.py | 66 +- swift/common/middleware/s3api/s3response.py | 18 + swift/common/middleware/s3api/utils.py | 1 + test/s3api/__init__.py | 20 +- test/s3api/test_mpu.py | 481 ++++++++++++- test/unit/common/middleware/s3api/__init__.py | 27 +- .../common/middleware/s3api/test_multi_get.py | 661 ++++++++++++++++++ .../middleware/s3api/test_multi_upload.py | 23 + test/unit/common/middleware/s3api/test_obj.py | 141 +++- .../common/middleware/s3api/test_s3request.py | 138 +++- .../common/middleware/s3api/test_utils.py | 1 + 14 files changed, 1569 insertions(+), 48 deletions(-) create mode 100644 test/unit/common/middleware/s3api/test_multi_get.py diff --git a/swift/common/middleware/s3api/acl_handlers.py b/swift/common/middleware/s3api/acl_handlers.py index 699b387713..ebb7acdee1 100644 --- a/swift/common/middleware/s3api/acl_handlers.py +++ b/swift/common/middleware/s3api/acl_handlers.py @@ -133,6 +133,15 @@ class BaseAclHandler(object): query = {} else: query = {'version-id': version_id} + if self.req.method == 'HEAD': + # This HEAD for ACL is going to also be the definitive response + # to the client so we need to include client params. We don't + # do this for other client request methods because they may + # have invalid combinations of params and headers for a swift + # HEAD request. + part_number = self.req.params.get('partNumber') + if part_number is not None: + query['part-number'] = part_number resp = self.req.get_acl_response(app, 'HEAD', container, obj, headers, query=query) diff --git a/swift/common/middleware/s3api/controllers/multi_upload.py b/swift/common/middleware/s3api/controllers/multi_upload.py index 6bfa6eed51..735d2be7cd 100644 --- a/swift/common/middleware/s3api/controllers/multi_upload.py +++ b/swift/common/middleware/s3api/controllers/multi_upload.py @@ -196,15 +196,7 @@ class PartController(Controller): raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') - try: - part_number = int(get_param(req, 'partNumber')) - if part_number < 1 or self.conf.max_upload_part_num < part_number: - raise Exception() - except Exception: - err_msg = 'Part number must be an integer between 1 and %d,' \ - ' inclusive' % self.conf.max_upload_part_num - raise InvalidArgument('partNumber', get_param(req, 'partNumber'), - err_msg) + part_number = req.validate_part_number() upload_id = get_param(req, 'uploadId') _get_upload_info(req, self.app, upload_id) diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py index b1f310e6e6..c116711593 100644 --- a/swift/common/middleware/s3api/controllers/obj.py +++ b/swift/common/middleware/s3api/controllers/obj.py @@ -90,8 +90,14 @@ class ObjectController(Controller): if version_id not in ('null', None) and \ 'object_versioning' not in get_swift_info(): raise S3NotImplemented() + part_number = req.validate_part_number(check_max=False) + + query = {} + if version_id is not None: + query['version-id'] = version_id + if part_number is not None: + query['part-number'] = part_number - query = {} if version_id is None else {'version-id': version_id} if version_id not in ('null', None): container_info = req.get_container_info(self.app) if not container_info.get( @@ -101,6 +107,19 @@ class ObjectController(Controller): resp = req.get_response(self.app, query=query) + if not resp.is_slo: + # SLO ignores part_number for non-slo objects, but s3api only + # allows the query param for non-MPU if it's exactly 1. + part_number = req.validate_part_number(parts_count=1) + if part_number == 1: + # When the query param *is* exactly 1 the response status code + # and headers are updated. + resp.status = HTTP_PARTIAL_CONTENT + resp.headers['Content-Range'] = \ + 'bytes 0-%d/%s' % (int(resp.headers['Content-Length']) - 1, + resp.headers['Content-Length']) + # else: part_number is None + if req.method == 'HEAD': resp.app_iter = None diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index da5220046d..dcc0fb361b 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -56,7 +56,8 @@ from swift.common.middleware.s3api.s3response import AccessDenied, \ MissingContentLength, InvalidStorageClass, S3NotImplemented, InvalidURI, \ MalformedXML, InvalidRequest, RequestTimeout, InvalidBucketName, \ BadDigest, AuthorizationHeaderMalformed, SlowDown, \ - AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU + AuthorizationQueryParametersError, ServiceUnavailable, BrokenMPU, \ + InvalidPartNumber, InvalidPartArgument from swift.common.middleware.s3api.exception import NotS3Request from swift.common.middleware.s3api.utils import utf8encode, \ S3Timestamp, mktime, MULTIUPLOAD_SUFFIX @@ -558,6 +559,57 @@ class S3Request(swob.Request): # by full URL when absolute path given. See swift.swob for more detail. self.environ['swift.leave_relative_location'] = True + def validate_part_number(self, parts_count=None, check_max=True): + """ + Get the partNumber param, if it exists, and check it is valid. + + To be valid, a partNumber must satisfy two criteria. First, it must be + an integer between 1 and the maximum allowed parts, inclusive. The + maximum allowed parts is the maximum of the configured + ``max_upload_part_num`` and, if given, ``parts_count``. Second, the + partNumber must be less than or equal to the ``parts_count``, if it is + given. + + :param parts_count: if given, this is the number of parts in an + existing object. + :raises InvalidPartArgument: if the partNumber param is invalid i.e. + less than 1 or greater than the maximum allowed parts. + :raises InvalidPartNumber: if the partNumber param is valid but greater + than ``num_parts``. + :return: an integer part number if the partNumber param exists, + otherwise ``None``. + """ + part_number = self.params.get('partNumber') + if part_number is None: + return None + + if self.range: + raise InvalidRequest('Cannot specify both Range header and ' + 'partNumber query parameter') + + try: + parts_count = int(parts_count) + except (TypeError, ValueError): + # an invalid/empty param is treated like parts_count=max_parts + parts_count = self.conf.max_upload_part_num + # max_parts may be raised to the number of existing parts + max_parts = max(self.conf.max_upload_part_num, parts_count) + + try: + part_number = int(part_number) + if part_number < 1: + raise ValueError + except ValueError: + raise InvalidPartArgument(max_parts, part_number) # 400 + + if check_max: + if part_number > max_parts: + raise InvalidPartArgument(max_parts, part_number) # 400 + if part_number > parts_count: + raise InvalidPartNumber() # 416 + + return part_number + def check_signature(self, secret): secret = utf8encode(secret) user_signature = self.signature @@ -1044,7 +1096,10 @@ class S3Request(swob.Request): if 'logging' in self.params: return LoggingStatusController if 'partNumber' in self.params: - return PartController + if self.method == 'PUT': + return PartController + else: + return ObjectController if 'uploadId' in self.params: return UploadController if 'uploads' in self.params: @@ -1315,7 +1370,6 @@ class S3Request(swob.Request): 'GET': { HTTP_NOT_FOUND: not_found_handler, HTTP_PRECONDITION_FAILED: PreconditionFailed, - HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: InvalidRange, }, 'PUT': { HTTP_NOT_FOUND: (NoSuchBucket, container), @@ -1414,7 +1468,7 @@ class S3Request(swob.Request): raise InvalidArgument('X-Delete-At', self.headers['X-Delete-At'], err_str) - if 'X-Delete-After' in err_msg.decode('utf8'): + if 'X-Delete-After' in err_str: raise InvalidArgument('X-Delete-After', self.headers['X-Delete-After'], err_str) @@ -1425,6 +1479,10 @@ class S3Request(swob.Request): **self.signature_does_not_match_kwargs()) if status == HTTP_FORBIDDEN: raise AccessDenied(reason='forbidden') + if status == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: + self.validate_part_number( + parts_count=resp.headers.get('x-amz-mp-parts-count')) + raise InvalidRange() if status == HTTP_SERVICE_UNAVAILABLE: raise ServiceUnavailable() if status in (HTTP_RATE_LIMITED, HTTP_TOO_MANY_REQUESTS): diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py index bc1aff7c0a..bcd2240c33 100644 --- a/swift/common/middleware/s3api/s3response.py +++ b/swift/common/middleware/s3api/s3response.py @@ -72,6 +72,8 @@ def translate_swift_to_s3(key, val): return key, val elif _key == 'x-object-version-id': return 'x-amz-version-id', val + elif _key == 'x-parts-count': + return 'x-amz-mp-parts-count', val elif _key == 'x-copied-from-version-id': return 'x-amz-copy-source-version-id', val elif _key == 'x-backend-content-type' and \ @@ -449,6 +451,17 @@ class InvalidObjectState(ErrorResponse): _msg = 'The operation is not valid for the current state of the object.' +class InvalidPartArgument(InvalidArgument): + _code = 'InvalidArgument' + + def __init__(self, max_parts, value): + err_msg = ('Part number must be an integer between ' + '1 and %s, inclusive' % max_parts) + super(InvalidArgument, self).__init__(err_msg, + argument_name='partNumber', + argument_value=value) + + class InvalidPart(ErrorResponse): _status = '400 Bad Request' _msg = 'One or more of the specified parts could not be found. The part ' \ @@ -478,6 +491,11 @@ class InvalidRange(ErrorResponse): _msg = 'The requested range cannot be satisfied.' +class InvalidPartNumber(ErrorResponse): + _status = '416 Requested Range Not Satisfiable' + _msg = 'The requested partnumber is not satisfiable' + + class InvalidRequest(ErrorResponse): _status = '400 Bad Request' _msg = 'Invalid Request.' diff --git a/swift/common/middleware/s3api/utils.py b/swift/common/middleware/s3api/utils.py index 40ff9388f6..2d535cd008 100644 --- a/swift/common/middleware/s3api/utils.py +++ b/swift/common/middleware/s3api/utils.py @@ -172,6 +172,7 @@ class Config(dict): 'allow_no_owner': False, 'allowable_clock_skew': 900, 'ratelimit_as_client_error': False, + 'max_upload_part_num': 1000, } def __init__(self, base=None): diff --git a/test/s3api/__init__.py b/test/s3api/__init__.py index 45a2a84a12..ad5bf0fd6d 100644 --- a/test/s3api/__init__.py +++ b/test/s3api/__init__.py @@ -151,12 +151,10 @@ class BaseS3TestCase(unittest.TestCase): # Default to v4 signatures (as aws-cli does), but subclasses can override signature_version = 's3v4' - @classmethod - def get_s3_client(cls, user): - return get_s3_client(user, cls.signature_version) + def get_s3_client(self, user): + return get_s3_client(user, self.signature_version) - @classmethod - def _remove_all_object_versions_from_bucket(cls, client, bucket_name): + def _remove_all_object_versions_from_bucket(self, client, bucket_name): resp = client.list_object_versions(Bucket=bucket_name) objs_to_delete = (resp.get('Versions', []) + resp.get('DeleteMarkers', [])) @@ -182,11 +180,10 @@ class BaseS3TestCase(unittest.TestCase): objs_to_delete = (resp.get('Versions', []) + resp.get('DeleteMarkers', [])) - @classmethod - def clear_bucket(cls, client, bucket_name): + def clear_bucket(self, client, bucket_name): timeout = time.time() + 10 backoff = 0.1 - cls._remove_all_object_versions_from_bucket(client, bucket_name) + self._remove_all_object_versions_from_bucket(client, bucket_name) try: client.delete_bucket(Bucket=bucket_name) except ClientError as e: @@ -199,7 +196,7 @@ class BaseS3TestCase(unittest.TestCase): Bucket=bucket_name, VersioningConfiguration={'Status': 'Suspended'}) while True: - cls._remove_all_object_versions_from_bucket( + self._remove_all_object_versions_from_bucket( client, bucket_name) # also try some version-unaware operations... for key in client.list_objects(Bucket=bucket_name).get( @@ -224,13 +221,12 @@ class BaseS3TestCase(unittest.TestCase): def create_name(self, slug): return '%s%s-%s' % (TEST_PREFIX, slug, uuid.uuid4().hex) - @classmethod - def clear_account(cls, client): + def clear_account(self, client): for bucket in client.list_buckets()['Buckets']: if not bucket['Name'].startswith(TEST_PREFIX): # these tests run against real s3 accounts continue - cls.clear_bucket(client, bucket['Name']) + self.clear_bucket(client, bucket['Name']) def tearDown(self): client = self.get_s3_client(1) diff --git a/test/s3api/test_mpu.py b/test/s3api/test_mpu.py index 26e550354e..28e7f0313a 100644 --- a/test/s3api/test_mpu.py +++ b/test/s3api/test_mpu.py @@ -17,7 +17,7 @@ from test.s3api import BaseS3TestCase from botocore.exceptions import ClientError -class TestMultiPartUploads(BaseS3TestCase): +class BaseMultiPartUploadTestCase(BaseS3TestCase): maxDiff = None @@ -26,10 +26,157 @@ class TestMultiPartUploads(BaseS3TestCase): self.bucket_name = self.create_name('test-mpu') resp = self.client.create_bucket(Bucket=self.bucket_name) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.num_parts = 3 + self.part_size = 5 * (2 ** 20) # 5 MB def tearDown(self): self.clear_bucket(self.client, self.bucket_name) - super(TestMultiPartUploads, self).tearDown() + super(BaseMultiPartUploadTestCase, self).tearDown() + + def _make_part_bodies(self): + return [ + ('%d' % i) * self.part_size + for i in range(self.num_parts) + ] + + def _iter_part_num_ranges(self): + for i in range(self.num_parts): + start = self.part_size * i + end = start + self.part_size + # part_num is 1 indexed + yield i + 1, start, end + + def _upload_mpu(self, key_name): + create_mpu_resp = self.client.create_multipart_upload( + Bucket=self.bucket_name, Key=key_name) + self.assertEqual(200, create_mpu_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + upload_id = create_mpu_resp['UploadId'] + + part_bodies = self._make_part_bodies() + parts = [] + for i, body in enumerate(part_bodies, 1): + part_resp = self.client.upload_part( + Body=body, Bucket=self.bucket_name, Key=key_name, + PartNumber=i, UploadId=upload_id) + self.assertEqual(200, part_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + parts.append({ + 'ETag': part_resp['ETag'], + 'PartNumber': i, + }) + # this helper doesn't bother calling list-parts, it's not required + # and we know what we uploaded + complete_mpu_resp = self.client.complete_multipart_upload( + Bucket=self.bucket_name, Key=key_name, + MultipartUpload={ + 'Parts': parts, + }, + UploadId=upload_id, + ) + self.assertEqual(200, complete_mpu_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + return complete_mpu_resp + + def upload_mpu_version(self, key_name): + complete_mpu_resp = self._upload_mpu(key_name) + # AWS returns the version_id *in* the MPU-complete response but s3api + # does NOT (see https://bugs.launchpad.net/swift/+bug/2043619), so we + # do an extra HEAD to get the version + head_object_resp = self.client.head_object( + Bucket=self.bucket_name, Key=key_name) + + self.assertEqual(200, head_object_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + + return complete_mpu_resp['ETag'], head_object_resp.get('VersionId') + + def upload_mpu(self, key_name): + complete_mpu_resp = self._upload_mpu(key_name) + return complete_mpu_resp['ETag'] + + def _verify_part_num_response(self, method, key_name, mpu_etag, + version=None): + part_bodies = self._make_part_bodies() + total_size = self.num_parts * self.part_size + + for part_num, start, end in self._iter_part_num_ranges(): + extra_kwargs = {} + if version is not None: + extra_kwargs['VersionId'] = version + resp = method(Bucket=self.bucket_name, Key=key_name, + PartNumber=part_num, **extra_kwargs) + self.assertEqual(206, resp['ResponseMetadata'][ + 'HTTPStatusCode']) + self.assertEqual(self.part_size, resp['ContentLength']) + if method == self.client.get_object: + resp_body = b''.join(resp['Body']).decode() + # our part_bodies are zero indexed + self.assertEqual(resp_body, part_bodies[part_num - 1]) + expected_range = 'bytes %s-%s/%s' % ( + start, end - 1, total_size) + self.assertEqual(expected_range, resp['ContentRange']) + # ETag and PartsCount are from the MPU + self.assertEqual(mpu_etag, resp['ETag'], mpu_etag) + self.assertEqual(self.num_parts, resp['PartsCount']) + self.assertEqual('bytes', resp['AcceptRanges']) + if version is None: + self.assertNotIn('VersionId', resp) + else: + self.assertEqual(version, resp['VersionId']) + + def _verify_copy_parts(self, key_src, key_dest, upload_id): + parts = [] + for part_num, start, end in self._iter_part_num_ranges(): + copy_range = 'bytes=%d-%d' % (start, end - 1) + copy_resp = self.client.\ + upload_part_copy(Bucket=self.bucket_name, + Key=key_dest, PartNumber=part_num, + CopySource={ + 'Bucket': self.bucket_name, + 'Key': key_src, + }, CopySourceRange=copy_range, + UploadId=upload_id) + self.assertEqual(200, copy_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + self.assertTrue(copy_resp['CopyPartResult']['ETag']) + self.assertTrue(copy_resp['CopyPartResult']['LastModified']) + parts.append({ + 'ETag': copy_resp['CopyPartResult']['ETag'], + 'PartNumber': part_num, + }) + + complete_mpu_resp = self.client.complete_multipart_upload( + Bucket=self.bucket_name, Key=key_dest, + MultipartUpload={ + 'Parts': parts, + }, + UploadId=upload_id, + ) + self.assertEqual(200, complete_mpu_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + + return complete_mpu_resp['ETag'] + + +class TestMultiPartUpload(BaseMultiPartUploadTestCase): + + def setUp(self): + super(TestMultiPartUpload, self).setUp() + + def _discover_max_part_num(self): + key_name = self.create_name('discover-max-part-num') + self.upload_mpu(key_name) + with self.assertRaises(ClientError) as cm: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, PartNumber=0) + err_resp = cm.exception.response + self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('InvalidArgument', err_resp['Error']['Code']) + err_msg = err_resp['Error']['Message'] + preamble = 'Part number must be an integer between 1 and ' + self.assertIn(preamble, err_msg) + return int(err_msg[len(preamble):].split(',')[0]) def test_basic_upload(self): key_name = self.create_name('key') @@ -68,6 +215,303 @@ class TestMultiPartUploads(BaseS3TestCase): self.assertEqual(200, complete_mpu_resp[ 'ResponseMetadata']['HTTPStatusCode']) + def _check_part_num_invalid_exc(self, exc, val, max_part_num, + is_head=False): + err_resp = exc.response + self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode']) + if is_head: + err_code = '400' + err_msg = 'Bad Request' + else: + err_code = 'InvalidArgument' + err_msg = 'Part number must be an integer between ' \ + '1 and %d, inclusive' % max_part_num + self.assertEqual(err_code, err_resp['Error']['Code'], err_resp) + self.assertEqual(err_msg, err_resp['Error']['Message']) + if is_head: + self.assertNotIn('ArgumentName', err_resp['Error']) + self.assertNotIn('ArgumentValue', err_resp['Error']) + else: + self.assertEqual('partNumber', err_resp['Error']['ArgumentName']) + self.assertEqual(str(val), err_resp['Error']['ArgumentValue']) + + def _check_part_num_out_of_range_exc(self, exc, is_head=False): + err_resp = exc.response + self.assertEqual(416, err_resp['ResponseMetadata']['HTTPStatusCode']) + if is_head: + err_code = '416' + err_msg = 'Requested Range Not Satisfiable' + else: + err_code = 'InvalidPartNumber' + err_msg = 'The requested partnumber is not satisfiable' + self.assertEqual(err_code, err_resp['Error']['Code'], err_resp) + self.assertEqual(err_msg, err_resp['Error']['Message'], err_resp) + + def test_get_object_partNumber_errors(self): + max_part_num = self._discover_max_part_num() + key_name = self.create_name('invalid-part-num-test') + mpu_etag = self.upload_mpu(key_name) + + # partNumber argument is 1 indexed + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, PartNumber=0) + self._check_part_num_invalid_exc(caught.exception, 0, max_part_num) + + # all other partNumber args are valid + self._verify_part_num_response( + self.client.get_object, key_name, mpu_etag) + + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, PartNumber=self.num_parts + 1) + self._check_part_num_out_of_range_exc(caught.exception) + + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, PartNumber=max_part_num) + self._check_part_num_out_of_range_exc(caught.exception) + + # because of ParamValidationError we can't test 'foo' + val = -1 + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, PartNumber=val) + self._check_part_num_invalid_exc(caught.exception, val, max_part_num) + val = max_part_num + 1 + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, PartNumber=val) + self._check_part_num_invalid_exc(caught.exception, val, max_part_num) + + def test_head_object_partNumber_errors(self): + max_part_num = self._discover_max_part_num() + key_name = self.create_name('invalid-part-num-head') + mpu_etag = self.upload_mpu(key_name) + + # partNumber argument is 1 indexed + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, + Key=key_name, PartNumber=0) + self._check_part_num_invalid_exc(caught.exception, 0, max_part_num, + is_head=True) + + # all other partNumber args are valid + self._verify_part_num_response( + self.client.head_object, key_name, mpu_etag) + + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, Key=key_name, + PartNumber=self.num_parts + 1) + self._check_part_num_out_of_range_exc(caught.exception, is_head=True) + + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, Key=key_name, + PartNumber=max_part_num) + self._check_part_num_out_of_range_exc(caught.exception, is_head=True) + + # because of ParamValidationError we can't test 'foo' + val = -1 + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, Key=key_name, + PartNumber=val) + self._check_part_num_invalid_exc(caught.exception, val, max_part_num, + is_head=True) + val = max_part_num + 1 + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, Key=key_name, + PartNumber=val) + self._check_part_num_invalid_exc(caught.exception, val, max_part_num, + is_head=True) + + def test_part_number_non_mpu(self): + max_part_num = self._discover_max_part_num() + key_name = self.create_name('part-num-non-mpu') + self.client.put_object(Bucket=self.bucket_name, + Key=key_name, + Body=b'non-mpu-object') + head_resp = self.client.head_object(Bucket=self.bucket_name, + Key=key_name) + # sanity check + self.assertEqual(200, + head_resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(head_resp['AcceptRanges'], 'bytes') + self.assertEqual(head_resp['ContentLength'], 14) + + head_resp = self.client.head_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=1) + self.assertEqual(206, + head_resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(head_resp['ContentLength'], 14) + + get_resp = self.client.get_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=1) + self.assertEqual(206, + get_resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(get_resp['ContentLength'], 14) + self.assertEqual(get_resp['ContentRange'], 'bytes 0-13/14') + self.assertEqual(b'non-mpu-object', b''.join(get_resp['Body'])) + + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=4) + self._check_part_num_out_of_range_exc(caught.exception) + + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=4) + self._check_part_num_out_of_range_exc(caught.exception, is_head=True) + + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=0) + self._check_part_num_invalid_exc(caught.exception, 0, max_part_num) + + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=0) + self._check_part_num_invalid_exc(caught.exception, 0, max_part_num, + is_head=True) + + invalid_part_num = 10001 + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=invalid_part_num) + self._check_part_num_invalid_exc(caught.exception, invalid_part_num, + max_part_num) + + with self.assertRaises(ClientError) as caught: + self.client.head_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=invalid_part_num) + self._check_part_num_invalid_exc(caught.exception, invalid_part_num, + max_part_num, is_head=True) + + def test_get_object_partNumber_and_range(self): + # partNumber not allowed with Range even for non-mpu object + key_name = self.create_name('part-num-mpu') + self._upload_mpu(key_name) + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=1, + Range='bytes=1-2') + err_resp = caught.exception.response + self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp) + self.assertEqual('Cannot specify both Range header and partNumber ' + 'query parameter', err_resp['Error']['Message']) + + key_name = self.create_name('part-num-non-mpu') + self.client.put_object(Bucket=self.bucket_name, + Key=key_name, + Body=b'non-mpu-object') + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=1, + Range='bytes=1-2') + err_resp = caught.exception.response + self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp) + self.assertEqual('Cannot specify both Range header and partNumber ' + 'query parameter', err_resp['Error']['Message']) + + # partNumber + Range error trumps bad partNumber + with self.assertRaises(ClientError) as caught: + self.client.get_object(Bucket=self.bucket_name, + Key=key_name, + PartNumber=0, + Range='bytes=1-2') + err_resp = caught.exception.response + self.assertEqual(400, err_resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('InvalidRequest', err_resp['Error']['Code'], err_resp) + self.assertEqual('Cannot specify both Range header and partNumber ' + 'query parameter', err_resp['Error']['Message']) + + def test_upload_part_copy(self): + self.num_parts = 4 + key_src = self.create_name('part-copy-src') + key_dest = self.create_name('part-copy-dest') + mpu_etag_src = self.upload_mpu(key_src) + self._verify_part_num_response( + self.client.get_object, key_src, mpu_etag_src) + self._verify_part_num_response( + self.client.head_object, key_src, mpu_etag_src) + + create_mpu_dest = self.client.create_multipart_upload( + Bucket=self.bucket_name, Key=key_dest) + self.assertEqual(200, create_mpu_dest[ + 'ResponseMetadata']['HTTPStatusCode']) + + upload_id = create_mpu_dest['UploadId'] + mpu_etag_dst = self._verify_copy_parts(key_src, key_dest, upload_id) + self._verify_part_num_response( + self.client.get_object, key_dest, mpu_etag_dst) + self._verify_part_num_response( + self.client.head_object, key_dest, mpu_etag_dst) + + def test_copy_mpu_from_parts(self): + key_src = self.create_name('copy-from-from-src') + mpu_etag_src = self.upload_mpu(key_src) + + # client wanting to copy object would first HEAD + head_object_resp = self.client.head_object( + Bucket=self.bucket_name, Key=key_src) + # the client will know it's an mpu and how many parts + self.assertEqual(mpu_etag_src, head_object_resp['ETag']) + self.assertIn('-', mpu_etag_src) + num_parts = int(mpu_etag_src.strip('"').rsplit('-')[-1]) + + # create new mpu + key_dest = self.create_name('copy-from-from-dest') + create_mpu_dest = self.client.create_multipart_upload( + Bucket=self.bucket_name, Key=key_dest) + self.assertEqual(200, create_mpu_dest[ + 'ResponseMetadata']['HTTPStatusCode']) + upload_id = create_mpu_dest['UploadId'] + + parts = [] + start = 0 + # do HEAD?partNumber to get copy range + for part_num in range(1, num_parts + 1): + part_head_resp = self.client.head_object( + Bucket=self.bucket_name, Key=key_src, PartNumber=part_num) + end = start + part_head_resp['ContentLength'] + copy_range = 'bytes=%s-%s' % (start, end - 1) + copy_resp = self.client.upload_part_copy( + Bucket=self.bucket_name, Key=key_dest, PartNumber=part_num, + CopySource={ + 'Bucket': self.bucket_name, + 'Key': key_src, + }, + CopySourceRange=copy_range, UploadId=upload_id) + self.assertEqual(200, copy_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + parts.append({ + 'ETag': copy_resp['CopyPartResult']['ETag'], + 'PartNumber': part_num, + }) + start = end + + complete_mpu_resp = self.client.complete_multipart_upload( + Bucket=self.bucket_name, Key=key_dest, + MultipartUpload={ + 'Parts': parts, + }, + UploadId=upload_id, + ) + self.assertEqual(200, complete_mpu_resp[ + 'ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(complete_mpu_resp['ETag'], mpu_etag_src) + def test_create_list_abort_multipart_uploads(self): key_name = self.create_name('key') create_mpu_resp = self.client.create_multipart_upload( @@ -138,3 +582,36 @@ class TestMultiPartUploads(BaseS3TestCase): self.assertEqual(complete_mpu_resp['Error']['UploadId'], upload_id) self.assertIn(complete_mpu_resp['Error']['PartNumber'], ('1', '2')) self.assertEqual(complete_mpu_resp['Error']['ETag'], None) + + +class TestVersionedMultiPartUpload(BaseMultiPartUploadTestCase): + + def setUp(self): + super(TestVersionedMultiPartUpload, self).setUp() + resp = self.client.put_bucket_versioning( + Bucket=self.bucket_name, + VersioningConfiguration={'Status': 'Enabled'}) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + + def tearDown(self): + resp = self.client.put_bucket_versioning( + Bucket=self.bucket_name, + VersioningConfiguration={'Status': 'Suspended'}) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + super(TestVersionedMultiPartUpload, self).tearDown() + + def test_get_by_part_number_with_versioning(self): + # create 3 version with progressively larger sizes + parts_counts = [2, 3, 4] + key_name = self.create_name('part-num-versions') + version_vars = [] + for num_parts in parts_counts: + self.num_parts = num_parts + etag, version_id = self.upload_mpu_version(key_name) + version_vars.append((num_parts, etag, version_id)) + for num_parts, mpu_etag, version in version_vars: + self.num_parts = num_parts + self._verify_part_num_response( + self.client.get_object, key_name, mpu_etag, version) + self._verify_part_num_response( + self.client.head_object, key_name, mpu_etag, version) diff --git a/test/unit/common/middleware/s3api/__init__.py b/test/unit/common/middleware/s3api/__init__.py index 4ec20255c2..83304c3987 100644 --- a/test/unit/common/middleware/s3api/__init__.py +++ b/test/unit/common/middleware/s3api/__init__.py @@ -28,19 +28,16 @@ from swift.common.middleware.s3api.etree import fromstring from swift.common.middleware.s3api.subresource import Owner, encode_acl, \ Grant, User, ACL, PERMISSIONS, AllUsers, AuthenticatedUsers -from test.debug_logger import debug_logger from test.unit.common.middleware.helpers import FakeSwift -class FakeApp(object): +class FakeAuthApp(object): container_existence_skip_cache = 0.0 account_existence_skip_cache = 0.0 - def __init__(self): + def __init__(self, app): self.remote_user = 'authorized' - self._pipeline_final_app = self - self.swift = FakeSwift() - self.logger = debug_logger() + self.app = app def _update_s3_path_info(self, env): """ @@ -82,7 +79,7 @@ class FakeApp(object): def __call__(self, env, start_response): self.handle(env) - return self.swift(env, start_response) + return self.app(env, start_response) class S3ApiTestCase(unittest.TestCase): @@ -90,6 +87,9 @@ class S3ApiTestCase(unittest.TestCase): def __init__(self, name): unittest.TestCase.__init__(self, name) + def _wrap_app(self, app): + return FakeAuthApp(app) + def setUp(self): # setup default config dict self.conf = { @@ -110,12 +110,13 @@ class S3ApiTestCase(unittest.TestCase): 'log_level': 'debug' } - self.app = FakeApp() - self.swift = self.app.swift # note: self.conf has no __file__ key so check_pipeline will be skipped # when constructing self.s3api + self.swift = FakeSwift() + self.app = self._wrap_app(self.swift) + self.app._pipeline_final_app = self.swift self.s3api = filter_factory({}, **self.conf)(self.app) - self.logger = self.s3api.logger = self.swift.logger = debug_logger() + self.logger = self.s3api.logger = self.swift.logger # if you change the registered acl response for /bucket or # /bucket/object tearDown will complain at you; you can set this to @@ -282,11 +283,11 @@ class S3ApiTestCaseAcl(S3ApiTestCase): self.swift.register('GET', path, swob.HTTPOk, {}, json.dumps([])), # setup sticky ACL headers... - grants = [_gen_grant(perm) for perm in PERMISSIONS] + self.grants = [_gen_grant(perm) for perm in PERMISSIONS] self.default_owner = Owner('test:tester', 'test:tester') - container_headers = _gen_test_headers(self.default_owner, grants) + container_headers = _gen_test_headers(self.default_owner, self.grants) object_headers = _gen_test_headers( - self.default_owner, grants, 'object') + self.default_owner, self.grants, 'object') public_headers = _gen_test_headers( self.default_owner, [Grant(AllUsers(), 'READ')]) authenticated_headers = _gen_test_headers( diff --git a/test/unit/common/middleware/s3api/test_multi_get.py b/test/unit/common/middleware/s3api/test_multi_get.py new file mode 100644 index 0000000000..93a8590194 --- /dev/null +++ b/test/unit/common/middleware/s3api/test_multi_get.py @@ -0,0 +1,661 @@ +# Copyright (c) 2023 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import binascii +import string +import json +import mock + +from swift.common import swob, utils +from swift.common.request_helpers import get_reserved_name +from swift.common.middleware import symlink +from swift.common.middleware.versioned_writes import object_versioning as ov + +from test.unit import make_timestamp_iter +from test.unit.common.middleware.test_slo import slo, md5hex +from test.unit.common.middleware.s3api import ( + S3ApiTestCase, S3ApiTestCaseAcl, _gen_test_headers) + + +def _prepare_mpu(swift, ts_iter, upload_id, num_segments, + segment_bucket='bucket+segments', segment_key='mpu'): + manifest = [] + for i, letter in enumerate(string.ascii_lowercase): + if len(manifest) >= num_segments: + break + size = (i + 1) * 5 + body = letter * size + etag = md5hex(body) + path = '/%s/%s/%s/%s' % (segment_bucket, segment_key, upload_id, i + 1) + swift.register('GET', '/v1/AUTH_test' + path, swob.HTTPOk, { + 'Content-Length': len(body), + 'Etag': etag, + }, body) + manifest.append({ + "name": path, + "bytes": size, + "hash": etag, + "content_type": "application/octet-stream", + "last_modified": next(ts_iter).isoformat, + }) + slo_etag = md5hex(''.join(s['hash'] for s in manifest)) + s3_hash = md5hex(binascii.a2b_hex(''.join( + s['hash'] for s in manifest))) + s3_etag = "%s-%s" % (s3_hash, len(manifest)) + manifest_json = json.dumps(manifest) + json_md5 = md5hex(manifest_json) + manifest_headers = { + 'Content-Length': str(len(manifest_json)), + 'X-Static-Large-Object': 'true', + 'Etag': json_md5, + 'Content-Type': 'application/octet-stream', + 'X-Object-Sysmeta-Slo-Etag': slo_etag, + 'X-Object-Sysmeta-Slo-Size': str(sum( + s['bytes'] for s in manifest)), + 'X-Object-Sysmeta-S3Api-Etag': s3_etag, + 'X-Object-Sysmeta-S3Api-Upload-Id': upload_id, + 'X-Object-Sysmeta-Container-Update-Override-Etag': + '%s; s3_etag=%s; slo_etag=%s' % (json_md5, s3_etag, slo_etag), + } + return manifest_headers, manifest_json + + +class TestMpuGETorHEAD(S3ApiTestCase): + + def _wrap_app(self, app): + self.slo = slo.filter_factory({'rate_limit_under_size': '0'})(app) + return super(TestMpuGETorHEAD, self)._wrap_app(self.slo) + + def setUp(self): + # this will call our _wrap_app + super(TestMpuGETorHEAD, self).setUp() + self.ts = make_timestamp_iter() + manifest_headers, manifest_json = _prepare_mpu( + self.swift, self.ts, 'X', 3) + self.s3_etag = manifest_headers['X-Object-Sysmeta-S3Api-Etag'] + self.swift.register( + 'GET', '/v1/AUTH_test/bucket/mpu', + swob.HTTPOk, manifest_headers, manifest_json.encode('ascii')) + self.s3_acl = False + + def test_mpu_GET(self): + req = swob.Request.blank('/bucket/mpu', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + self.assertEqual(body, b'aaaaabbbbbbbbbbccccccccccccccc') + expected_calls = [ + ('GET', '/v1/AUTH_test/bucket/mpu'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/1' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/2' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/3' + '?multipart-manifest=get'), + ] + if self.s3_acl: + # pre-flight object ACL check + expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu')) + self.assertEqual(self.swift.calls, expected_calls) + self.assertEqual(headers['Content-Length'], '30') + self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag) + self.assertNotIn('X-Amz-Mp-Parts-Count', headers) + + def test_mpu_GET_part_num(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': '2', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '206') + self.assertEqual(body, b'bbbbbbbbbb') + expected_calls = [ + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=2'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X/2' + '?multipart-manifest=get'), + ] + if self.s3_acl: + expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu')) + self.assertEqual(self.swift.calls, expected_calls) + self.assertEqual(headers['Content-Length'], '10') + self.assertEqual(headers['Content-Range'], 'bytes 5-14/30') + self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag) + self.assertEqual(headers['X-Amz-Mp-Parts-Count'], '3') + + def test_mpu_GET_invalid_part_num(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': 'foo', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual(self.swift.calls, []) + + def test_mpu_GET_zero_part_num(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': '0', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual(self.swift.calls, []) + + def _do_test_mpu_GET_out_of_range_part_num(self, part_number): + self.swift.clear_calls() + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': str(part_number), + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '416') + self.assertEqual(self._get_error_code(body), 'InvalidPartNumber') + expected_calls = [ + # s3api.controller.obj doesn't know yet if it's SLO, we delegate + # param validation + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number), + ] + if self.s3_acl: + expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu')) + self.assertEqual(self.swift.calls, expected_calls) + + def test_mpu_GET_out_of_range_part_num(self): + self._do_test_mpu_GET_out_of_range_part_num(4) + self._do_test_mpu_GET_out_of_range_part_num(10000) + + def test_existing_part_number_greater_than_max_parts_allowed(self): + part_number = 3 + max_parts = 2 + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': str(part_number), + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + bad_req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': str(part_number + 1), + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + with mock.patch.object(self.s3api.conf, + 'max_upload_part_num', max_parts): + # num_parts >= part number > max parts + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '206') + # part number > num parts > max parts + status, headers, body = self.call_s3api(bad_req) + self.assertEqual(status.split()[0], '400') + self.assertIn('must be an integer between 1 and 3, inclusive', + self._get_error_message(body)) + + max_parts = part_number + 1 + with mock.patch.object(self.s3api.conf, + 'max_upload_part_num', max_parts): + # max_parts > num_parts >= part number + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '206') + # max_parts >= part number > num parts + status, headers, body = self.call_s3api(bad_req) + self.assertEqual(status.split()[0], '416') + self.assertIn('The requested partnumber is not satisfiable', + self._get_error_message(body)) + # part number > max_parts > num parts + bad_req.params = {'partNumber': str(max_parts + 1)} + status, headers, body = self.call_s3api(bad_req) + self.assertEqual(status.split()[0], '400') + self.assertIn('must be an integer between 1 and 4, inclusive', + self._get_error_message(body)) + + def test_mpu_GET_huge_part_num(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': '10001', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + expected_calls = [ + # XXX is this value configurable? do we need the SLO request? + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=10001'), + ] + if self.s3_acl: + expected_calls.insert(0, ('HEAD', '/v1/AUTH_test/bucket/mpu')) + self.assertEqual(self.swift.calls, expected_calls) + + def test_mpu_HEAD_part_num(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': '1', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }, method='HEAD') + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '206') + self.assertEqual(body, b'') + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=1'), + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=1'), + ]) + self.assertEqual(headers['Content-Length'], '5') + self.assertEqual(headers['Content-Range'], 'bytes 0-4/30') + self.assertEqual(headers['Etag'], '"%s"' % self.s3_etag) + self.assertEqual(headers['X-Amz-Mp-Parts-Count'], '3') + + def test_mpu_HEAD_invalid_part_num(self): + req = swob.Request.blank('/bucket/mpu', method='HEAD', params={ + 'partNumber': 'foo', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, _ = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self.swift.calls, []) + + def test_mpu_HEAD_zero_part_num(self): + req = swob.Request.blank('/bucket/mpu', method='HEAD', params={ + 'partNumber': '0', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, _ = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self.swift.calls, []) + + def _do_test_mpu_HEAD_out_of_range_part_num(self, part_number): + self.swift.clear_calls() + req = swob.Request.blank('/bucket/mpu', method='HEAD', params={ + 'partNumber': str(part_number), + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, _ = self.call_s3api(req) + self.assertEqual(status.split()[0], '416') + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number), + # SLO has to refetch to *see* if it's out-of-bounds + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=%s' % part_number), + ]) + + def test_mpu_HEAD_out_of_range_part_num(self): + self._do_test_mpu_HEAD_out_of_range_part_num(4) + self._do_test_mpu_HEAD_out_of_range_part_num(10000) + + def test_mpu_HEAD_huge_part_num(self): + req = swob.Request.blank('/bucket/mpu', method='HEAD', params={ + 'partNumber': '10001', + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=10001'), + # XXX were two requests worth it to 400? + # how big can you configure SLO? + # do such manifests *exist*? + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=10001'), + ]) + + +class TestMpuGETorHEADAcl(TestMpuGETorHEAD, S3ApiTestCaseAcl): + + def setUp(self): + super(TestMpuGETorHEADAcl, self).setUp() + object_headers = _gen_test_headers( + self.default_owner, self.grants, 'object') + self.swift.update_sticky_response_headers( + '/v1/AUTH_test/bucket/mpu', object_headers) + # this is used to flag insertion of expected HEAD pre-flight request of + # object ACLs + self.s3_acl = True + + +class TestVersionedMpuGETorHEAD(S3ApiTestCase): + + def _wrap_app(self, app): + self.sym = symlink.filter_factory({})(app) + self.sym.logger = self.swift.logger + self.ov = ov.ObjectVersioningMiddleware(self.sym, {}) + self.ov.logger = self.swift.logger + self.slo = slo.filter_factory({'rate_limit_under_size': '0'})(self.ov) + self.slo.logger = self.swift.logger + return super(TestVersionedMpuGETorHEAD, self)._wrap_app(self.slo) + + def setUp(self): + # this will call our _wrap_app + super(TestVersionedMpuGETorHEAD, self).setUp() + self.ts = make_timestamp_iter() + self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments', + swob.HTTPNoContent, {}, None) + versions_container = get_reserved_name('versions', 'bucket') + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, { + ov.SYSMETA_VERSIONS_CONT: versions_container, + ov.SYSMETA_VERSIONS_ENABLED: True, + }, None) + self.swift.register('HEAD', '/v1/AUTH_test/%s' % versions_container, + swob.HTTPNoContent, {}, None) + num_versions = 3 + self.version_ids = [] + for v in range(num_versions): + upload_id = 'X%s' % v + num_segments = 3 + v + manifest_headers, manifest_json = _prepare_mpu( + self.swift, self.ts, upload_id, num_segments) + version_ts = next(self.ts) + # add in a little user-meta to keep versions stright + manifest_version_headers = dict(manifest_headers, **{ + 'x-object-meta-user-notes': 'version%s' % v, + 'x-backend-timestamp': version_ts.internal, + }) + self.version_ids.append(version_ts.normal) + obj_version_path = get_reserved_name('mpu', (~version_ts).normal) + self.swift.register( + 'GET', '/v1/AUTH_test/%s/%s' % ( + versions_container, obj_version_path), + swob.HTTPOk, manifest_version_headers, + manifest_json.encode('ascii')) + # TODO: make a current version symlink + symlink_target = '%s/%s' % (versions_container, obj_version_path) + slo_etag = manifest_headers['X-Object-Sysmeta-Slo-Etag'] + s3_etag = manifest_headers['X-Object-Sysmeta-S3Api-Etag'] + symlink_target_etag = json_md5 = manifest_headers['Etag'] + symlink_target_bytes = manifest_headers['X-Object-Sysmeta-Slo-Size'] + manifest_symlink_headers = dict(manifest_headers, **{ + 'X-Object-Sysmeta-Container-Update-Override-Etag': + '%s; s3_etag=%s; slo_etag=%s; symlink_target=%s; ' + 'symlink_target_etag=%s; symlink_target_bytes=%s' % ( + json_md5, s3_etag, slo_etag, symlink_target, + symlink_target_etag, symlink_target_bytes), + 'X-Object-Sysmeta-Allow-Reserved-Names': 'true', + 'X-Object-Sysmeta-Symlink-Target': symlink_target, + 'X-Object-Sysmeta-Symlink-Target-Bytes': str(symlink_target_bytes), + 'X-Object-Sysmeta-Symlink-Target-Etag': symlink_target_etag, + 'X-Object-Sysmeta-Symloop-Extend': 'true', + 'X-Object-Sysmeta-Versions-Symlink': 'true', + }) + self.swift.register( + 'GET', '/v1/AUTH_test/bucket/mpu', swob.HTTPOk, + manifest_symlink_headers, '') + self.s3_acl = False + + def test_mpu_GET_version(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'versionId': self.version_ids[0], + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '200 OK') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version0') + self.assertEqual(headers['Content-Length'], '30') + self.assertEqual(body, b'aaaaabbbbbbbbbbccccccccccccccc') + expected_calls = [ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?version-id=%s' % ( + (~utils.Timestamp(self.version_ids[0])).normal, + self.version_ids[0])), + ('HEAD', '/v1/AUTH_test/bucket+segments'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/1' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/2' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X0/3' + '?multipart-manifest=get') + ] + if self.s3_acl: + expected_calls.insert(3, ( + 'HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?version-id=%s' % ( + (~utils.Timestamp(self.version_ids[0])).normal, + self.version_ids[0]) + )) + self.assertEqual(self.swift.calls, expected_calls) + + def test_mpu_GET_last_version(self): + req = swob.Request.blank('/bucket/mpu', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '200 OK') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version2') + self.assertEqual(headers['Content-Length'], '75') + expected_calls = [ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('GET', '/v1/AUTH_test/bucket/mpu'), + ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' % ( + ~utils.Timestamp(self.version_ids[2])).normal), + ('HEAD', '/v1/AUTH_test/bucket+segments'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/1' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/2' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/3' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/4' + '?multipart-manifest=get'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/5' + '?multipart-manifest=get'), + ] + if self.s3_acl: + # the pre-flight head on version marker get's symlinked; but I + # think maybe symlink makes metadata addative? + expected_calls = expected_calls[:3] + [ + ('HEAD', '/v1/AUTH_test/bucket/mpu'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00' + '%s' % (~utils.Timestamp(self.version_ids[2])).normal), + ] + expected_calls[3:] + self.assertEqual(expected_calls, self.swift.calls) + + def test_mpu_HEAD_last_version(self): + req = swob.Request.blank('/bucket/mpu', method='HEAD', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '200 OK') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version2') + self.assertEqual(headers['Content-Length'], '75') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('HEAD', '/v1/AUTH_test/bucket/mpu'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' % ( + ~utils.Timestamp(self.version_ids[2])).normal), + ], self.swift.calls) + + def test_mpu_HEAD_version(self): + req = swob.Request.blank('/bucket/mpu', method='HEAD', params={ + 'versionId': self.version_ids[1], + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '200 OK') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version1') + self.assertEqual(headers['Content-Length'], '50') + self.assertEqual(body, b'') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?version-id=%s' % ( + (~utils.Timestamp(self.version_ids[1])).normal, + self.version_ids[1])), + ], self.swift.calls) + + def test_mpu_GET_version_part_num(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'versionId': self.version_ids[2], + 'partNumber': 5, + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version2') + self.assertEqual(headers['Content-Length'], '25') + self.assertEqual(body, b'e' * 25) + expected_calls = [ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?part-number=5&version-id=%s' % ( + (~utils.Timestamp(self.version_ids[2])).normal, + self.version_ids[2])), + ('HEAD', '/v1/AUTH_test/bucket+segments'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/5' + '?multipart-manifest=get'), + ] + if self.s3_acl: + expected_calls.insert(3, ( + 'HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?version-id=%s' % ( + (~utils.Timestamp(self.version_ids[2])).normal, + self.version_ids[2]) + )) + self.assertEqual(expected_calls, self.swift.calls) + + def test_mpu_HEAD_version_part_num(self): + req = swob.Request.blank('/bucket/mpu', method='HEAD', params={ + 'versionId': self.version_ids[2], + 'partNumber': 3, + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version2') + self.assertEqual(headers['Content-Length'], '15') + self.assertEqual(body, b'') + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?part-number=3&version-id=%s' % ( + (~utils.Timestamp(self.version_ids[2])).normal, + self.version_ids[2])), + ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?part-number=3&version-id=%s' % ( + (~utils.Timestamp(self.version_ids[2])).normal, + self.version_ids[2])), + ]) + + def test_mpu_GET_last_version_part_num(self): + req = swob.Request.blank('/bucket/mpu', params={ + 'partNumber': 4, + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version2') + self.assertEqual(headers['Content-Length'], '20') + self.assertEqual(body, b'd' * 20) + expected_calls = [ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=4'), + ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?part-number=4' % ( + ~utils.Timestamp(self.version_ids[2])).normal), + ('HEAD', '/v1/AUTH_test/bucket+segments'), + ('GET', '/v1/AUTH_test/bucket+segments/mpu/X2/4' + '?multipart-manifest=get'), + ] + if self.s3_acl: + expected_calls = expected_calls[:3] + [ + ('HEAD', '/v1/AUTH_test/bucket/mpu'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00' + '%s' % (~utils.Timestamp(self.version_ids[2])).normal), + ] + expected_calls[3:] + self.assertEqual(expected_calls, self.swift.calls) + + def test_mpu_HEAD_last_version_part_num(self): + req = swob.Request.blank('/bucket/mpu', method='HEAD', params={ + 'partNumber': 5, + }, headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header() + }) + status, headers, body = self.call_s3api(req) + self.assertEqual(status, '206 Partial Content') + self.assertEqual(headers['x-amz-meta-user-notes'], 'version2') + self.assertEqual(headers['Content-Length'], '25') + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket'), + ('HEAD', '/v1/AUTH_test/bucket/mpu?part-number=5'), + ('HEAD', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?part-number=5' % ( + ~utils.Timestamp(self.version_ids[2])).normal), + ('GET', '/v1/AUTH_test/bucket/mpu?part-number=5'), + ('GET', '/v1/AUTH_test/\x00versions\x00bucket/\x00mpu\x00%s' + '?part-number=5' % ( + ~utils.Timestamp(self.version_ids[2])).normal), + ]) + + +class TestVersionedMpuGETorHEADAcl(TestVersionedMpuGETorHEAD, + S3ApiTestCaseAcl): + + def setUp(self): + super(TestVersionedMpuGETorHEADAcl, self).setUp() + object_headers = _gen_test_headers( + self.default_owner, self.grants, 'object') + for version_id in self.version_ids: + # s3acl would add the default object ACL on PUT to each version + version_path = '/v1/AUTH_test/\x00versions\x00bucket/' \ + '\x00mpu\x00%s' % (~utils.Timestamp(version_id)).normal + self.swift.update_sticky_response_headers( + version_path, object_headers) + # this is used to flag insertion of expected HEAD pre-flight request of + # object ACLs + self.s3_acl = True diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py index 7a91011848..e2e121ace7 100644 --- a/test/unit/common/middleware/s3api/test_multi_upload.py +++ b/test/unit/common/middleware/s3api/test_multi_upload.py @@ -685,6 +685,9 @@ class BaseS3ApiMultiUpload(object): body='part object') status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual(self._get_error_message(body), + 'Part number must be an integer between 1 and 10000, ' + 'inclusive') # part number must be > 0 req = Request.blank('/bucket/object?partNumber=0&uploadId=X', @@ -694,6 +697,9 @@ class BaseS3ApiMultiUpload(object): body='part object') status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual(self._get_error_message(body), + 'Part number must be an integer between 1 and 10000, ' + 'inclusive') # part number must be < 10001 req = Request.blank('/bucket/object?partNumber=10001&uploadId=X', @@ -703,6 +709,23 @@ class BaseS3ApiMultiUpload(object): body='part object') status, headers, body = self.call_s3api(req) self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual(self._get_error_message(body), + 'Part number must be an integer between 1 and 10000, ' + 'inclusive') + + with patch.object(self.s3api.conf, 'max_upload_part_num', 1000): + # part number must be < 1001 + req = Request.blank( + '/bucket/object?partNumber=1001&uploadId=X', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body='part object') + status, headers, body = self.call_s3api(req) + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual(self._get_error_message(body), + 'Part number must be an integer between 1 and ' + '1000, inclusive') # without target bucket req = Request.blank('/nobucket/object?partNumber=1&uploadId=X', diff --git a/test/unit/common/middleware/s3api/test_obj.py b/test/unit/common/middleware/s3api/test_obj.py index 180de31a88..617322d4c4 100644 --- a/test/unit/common/middleware/s3api/test_obj.py +++ b/test/unit/common/middleware/s3api/test_obj.py @@ -119,6 +119,12 @@ class BaseS3ApiObj(object): if method == 'GET': self.assertEqual(body, self.object_body) + def test_object_GET(self): + self._test_object_GETorHEAD('GET') + + def test_object_HEAD(self): + self._test_object_GETorHEAD('HEAD') + def test_object_HEAD_error(self): # HEAD does not return the body even an error response in the # specifications of the REST API. @@ -331,8 +337,136 @@ class BaseS3ApiObj(object): expected_status='429 Slow Down') self.assertEqual(code, 'SlowDown') - def test_object_GET(self): - self._test_object_GETorHEAD('GET') + def _test_non_slo_object_GETorHEAD_part_num(self, method, part_number): + req = Request.blank('/bucket/object?partNumber=%s' % part_number, + environ={'REQUEST_METHOD': method}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '206') + self.assertEqual(headers['content-length'], '5') + self.assertTrue('content-range' in headers) + self.assertEqual(headers['content-range'], 'bytes 0-4/5') + self.assertEqual(headers['content-type'], 'text/html') + # we'll want this for logging + self._assert_policy_index(req.headers, headers, + self.bucket_policy_index) + self.assertEqual(headers['etag'], + '"%s"' % self.response_headers['etag']) + + if method == 'GET': + self.assertEqual(body, self.object_body) + + def test_non_slo_object_GET_part_num(self): + self._test_non_slo_object_GETorHEAD_part_num('GET', 1) + + def test_non_slo_object_HEAD_part_num(self): + self._test_non_slo_object_GETorHEAD_part_num('HEAD', 1) + + def _do_test_non_slo_object_part_num_not_satisfiable(self, method, + part_number): + req = Request.blank('/bucket/object', + params={'partNumber': part_number}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req.method = method + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '416') + return body + + def test_non_slo_object_GET_part_num_not_satisfiable(self): + body = self._do_test_non_slo_object_part_num_not_satisfiable( + 'GET', '2') + self.assertEqual(self._get_error_code(body), 'InvalidPartNumber') + body = self._do_test_non_slo_object_part_num_not_satisfiable( + 'GET', '10000') + self.assertEqual(self._get_error_code(body), 'InvalidPartNumber') + + def test_non_slo_object_HEAD_part_num_not_satisfiable(self): + body = self._do_test_non_slo_object_part_num_not_satisfiable( + 'HEAD', '2') + self.assertEqual(body, b'') + body = self._do_test_non_slo_object_part_num_not_satisfiable( + 'HEAD', '10000') + self.assertEqual(body, b'') + + def _do_test_non_slo_object_part_num_invalid(self, method, part_number): + req = Request.blank('/bucket/object', + params={'partNumber': part_number}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req.method = method + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + return body + + def test_non_slo_object_GET_part_num_invalid(self): + body = self._do_test_non_slo_object_part_num_invalid('GET', '0') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + body = self._do_test_non_slo_object_part_num_invalid('GET', '-1') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + body = self._do_test_non_slo_object_part_num_invalid('GET', '10001') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + with patch.object(self.s3api.conf, 'max_upload_part_num', 1000): + body = self._do_test_non_slo_object_part_num_invalid('GET', '1001') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual( + self._get_error_message(body), + 'Part number must be an integer between 1 and 1000, inclusive') + + body = self._do_test_non_slo_object_part_num_invalid('GET', 'foo') + self.assertEqual(self._get_error_code(body), 'InvalidArgument') + self.assertEqual( + self._get_error_message(body), + 'Part number must be an integer between 1 and 10000, inclusive') + + def test_non_slo_object_HEAD_part_num_invalid(self): + body = self._do_test_non_slo_object_part_num_invalid('HEAD', '0') + self.assertEqual(body, b'') + body = self._do_test_non_slo_object_part_num_invalid('HEAD', '-1') + self.assertEqual(body, b'') + body = self._do_test_non_slo_object_part_num_invalid('HEAD', '10001') + self.assertEqual(body, b'') + body = self._do_test_non_slo_object_part_num_invalid('HEAD', 'foo') + self.assertEqual(body, b'') + + def test_non_slo_object_GET_part_num_and_range(self): + req = Request.blank('/bucket/object', + params={'partNumber': '1'}, + headers={'Range': 'bytes=1-2', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req.method = 'GET' + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self._get_error_code(body), 'InvalidRequest') + self.assertEqual( + self._get_error_message(body), + 'Cannot specify both Range header and partNumber query parameter') + + # partNumber + Range error trumps bad partNumber + req = Request.blank('/bucket/object', + params={'partNumber': '0'}, + headers={'Range': 'bytes=1-2', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req.method = 'GET' + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') + self.assertEqual(self._get_error_code(body), 'InvalidRequest') + self.assertEqual( + self._get_error_message(body), + 'Cannot specify both Range header and partNumber query parameter') + + def test_non_slo_object_HEAD_part_num_and_range(self): + req = Request.blank('/bucket/object', + params={'partNumber': '1'}, + headers={'Range': 'bytes=1-2', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req.method = 'HEAD' + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '400') def test_object_GET_Range(self): req = Request.blank('/bucket/object', @@ -1105,9 +1239,6 @@ class TestS3ApiObj(BaseS3ApiObj, S3ApiTestCase): swob.HTTPRequestedRangeNotSatisfiable) self.assertEqual(code, 'InvalidRange') - def test_object_HEAD(self): - self._test_object_GETorHEAD('HEAD') - @patch_policies([ StoragePolicy(0, 'gold', is_default=True), StoragePolicy(1, 'silver')]) diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py index 7391b35f82..0b2216e741 100644 --- a/test/unit/common/middleware/s3api/test_s3request.py +++ b/test/unit/common/middleware/s3api/test_s3request.py @@ -32,7 +32,8 @@ from swift.common.middleware.s3api.s3request import S3Request, \ S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT, HashingInput from swift.common.middleware.s3api.s3response import InvalidArgument, \ NoSuchBucket, InternalError, ServiceUnavailable, \ - AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed, BadDigest + AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed, BadDigest, \ + InvalidPartArgument, InvalidPartNumber, InvalidRequest from swift.common.utils import md5 from test.debug_logger import debug_logger @@ -998,6 +999,104 @@ class TestRequest(S3ApiTestCase): sigv4_req._canonical_request().endswith(sha256_of_nothing.upper())) self.assertTrue(sigv4_req.check_signature('secret')) + def test_validate_part_number(self): + sw_req = Request.blank('/nojunk', + environ={'REQUEST_METHOD': 'GET'}, + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req = S3Request(sw_req.environ) + self.assertIsNone(req.validate_part_number()) + + # ok + sw_req = Request.blank('/nojunk?partNumber=102', + environ={'REQUEST_METHOD': 'GET'}, + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req = S3Request(sw_req.environ) + self.assertEqual(102, req.validate_part_number()) + req = S3Request(sw_req.environ, + conf=Config({'max_upload_part_num': 100})) + self.assertEqual(102, req.validate_part_number(102)) + req = S3Request(sw_req.environ, + conf=Config({'max_upload_part_num': 102})) + self.assertEqual(102, req.validate_part_number(102)) + + def test_validate_part_number_invalid_argument(self): + def check_invalid_argument(part_num, max_parts, parts_count, exp_max): + sw_req = Request.blank('/nojunk?partNumber=%s' % part_num, + environ={'REQUEST_METHOD': 'GET'}, + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req = S3Request(sw_req.environ, + conf=Config({'max_upload_part_num': max_parts})) + with self.assertRaises(InvalidPartArgument) as cm: + req.validate_part_number(parts_count=parts_count) + self.assertEqual('400 Bad Request', str(cm.exception)) + self.assertIn( + b'Part number must be an integer between 1 and %d' % exp_max, + cm.exception.body) + + check_invalid_argument(102, 99, None, 99) + check_invalid_argument(102, 100, 99, 100) + check_invalid_argument(102, 100, 101, 101) + check_invalid_argument(102, 101, 100, 101) + check_invalid_argument(102, 101, 101, 101) + check_invalid_argument('banana', 1000, None, 1000) + check_invalid_argument(0, 10000, None, 10000) + + def test_validate_part_number_invalid_part_number(self): + def check_invalid_part_num(part_num, max_parts, parts_count): + sw_req = Request.blank('/nojunk?partNumber=%s' % part_num, + environ={'REQUEST_METHOD': 'GET'}, + headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req = S3Request(sw_req.environ, + conf=Config({'max_upload_part_num': max_parts})) + with self.assertRaises(InvalidPartNumber) as cm: + req.validate_part_number(parts_count=parts_count) + self.assertEqual('416 Requested Range Not Satisfiable', + str(cm.exception)) + self.assertIn(b'The requested partnumber is not satisfiable', + cm.exception.body) + + check_invalid_part_num(102, 10000, 1) + check_invalid_part_num(102, 102, 101) + check_invalid_part_num(102, 10000, 101) + + def test_validate_part_number_with_range_header(self): + sw_req = Request.blank('/nojunk?partNumber=1', + environ={'REQUEST_METHOD': 'GET'}, + headers={ + 'Range': 'bytes=1-2', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req = S3Request(sw_req.environ) + with self.assertRaises(InvalidRequest) as cm: + req.validate_part_number() + self.assertEqual('400 Bad Request', + str(cm.exception)) + self.assertIn(b'Cannot specify both Range header and partNumber query ' + b'parameter', cm.exception.body) + + # bad part number AND Range header + sw_req = Request.blank('/nojunk?partNumber=0', + environ={'REQUEST_METHOD': 'GET'}, + headers={ + 'Range': 'bytes=1-2', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + req = S3Request(sw_req.environ) + with self.assertRaises(InvalidRequest) as cm: + req.validate_part_number() + self.assertEqual('400 Bad Request', + str(cm.exception)) + self.assertIn(b'Cannot specify both Range header and partNumber query ' + b'parameter', cm.exception.body) + class TestSigV4Request(S3ApiTestCase): def setUp(self): @@ -1204,7 +1303,7 @@ class TestSigV4Request(S3ApiTestCase): return SigV4Request(req.environ, None, config) s3req = make_s3req(Config(), '/bkt', {'partNumber': '3'}) - self.assertEqual(controllers.multi_upload.PartController, + self.assertEqual(controllers.ObjectController, s3req.controller) s3req = make_s3req(Config(), '/bkt', {'uploadId': '4'}) @@ -1235,6 +1334,41 @@ class TestSigV4Request(S3ApiTestCase): self.assertEqual(controllers.ServiceController, s3req.controller) + def test_controller_for_multipart_upload_requests(self): + environ = { + 'HTTP_HOST': 'bucket.s3.test.com', + 'REQUEST_METHOD': 'PUT'} + x_amz_date = self.get_v4_amz_date_header() + auth = ('AWS4-HMAC-SHA256 ' + 'Credential=test/%s/us-east-1/s3/aws4_request,' + 'SignedHeaders=host;x-amz-content-sha256;x-amz-date,' + 'Signature=X' % self.get_v4_amz_date_header().split('T', 1)[0]) + headers = { + 'Authorization': auth, + 'X-Amz-Content-SHA256': '0123456789', + 'Date': self.get_date_header(), + 'X-Amz-Date': x_amz_date} + + def make_s3req(config, path, params): + req = Request.blank(path, environ=environ, headers=headers, + params=params) + return SigV4Request(req.environ, None, config) + + s3req = make_s3req(Config(), '/bkt', {'partNumber': '3', + 'uploadId': '4'}) + self.assertEqual(controllers.multi_upload.PartController, + s3req.controller) + + s3req = make_s3req(Config(), '/bkt', {'partNumber': '3'}) + self.assertEqual(controllers.multi_upload.PartController, + s3req.controller) + + s3req = make_s3req(Config(), '/bkt', {'uploadId': '4', + 'partNumber': '3', + 'copySource': 'bkt2/obj2'}) + self.assertEqual(controllers.multi_upload.PartController, + s3req.controller) + class TestHashingInput(S3ApiTestCase): def test_good(self): diff --git a/test/unit/common/middleware/s3api/test_utils.py b/test/unit/common/middleware/s3api/test_utils.py index 9fc0854f52..01cc07117f 100644 --- a/test/unit/common/middleware/s3api/test_utils.py +++ b/test/unit/common/middleware/s3api/test_utils.py @@ -181,6 +181,7 @@ class TestConfig(unittest.TestCase): del conf.allow_no_owner del conf.allowable_clock_skew del conf.ratelimit_as_client_error + del conf.max_upload_part_num self.assertEqual({}, conf) def test_update(self):