From 6097660f0c350e12c026d62fd2fbd437140b917b Mon Sep 17 00:00:00 2001 From: karen chan Date: Fri, 27 Jan 2017 07:43:37 -0800 Subject: [PATCH] s3api: Implement object versioning API Translate AWS S3 Object Versioning API requests to native Swift Object Versioning API, speficially: * bucket versioning status * bucket versioned objects listing params * object GETorHEAD & DELETE versionId * multi_delete versionId Change-Id: I8296681b61996e073b3ba12ad46f99042dc15c37 Co-Authored-By: Tim Burke Co-Authored-By: Clay Gerrard --- .../conf/ceph-known-failures-keystone.yaml | 10 - .../conf/ceph-known-failures-tempauth.yaml | 24 +- doc/source/s3_compat.rst | 2 +- swift/common/middleware/s3api/acl_handlers.py | 12 +- .../middleware/s3api/controllers/bucket.py | 318 +++++--- .../s3api/controllers/multi_delete.py | 39 +- .../s3api/controllers/multi_upload.py | 9 + .../middleware/s3api/controllers/obj.py | 71 +- .../s3api/controllers/versioning.py | 42 +- swift/common/middleware/s3api/s3request.py | 72 +- swift/common/middleware/s3api/s3response.py | 13 +- swift/common/middleware/s3api/utils.py | 4 +- test/functional/__init__.py | 13 +- test/functional/s3api/s3_test_client.py | 12 +- test/functional/s3api/test_multi_upload.py | 137 +++- test/functional/s3api/test_object.py | 4 +- test/functional/s3api/test_versioning.py | 166 ++++ test/s3api/__init__.py | 89 +- test/s3api/test_service.py | 5 +- test/s3api/test_versioning.py | 758 ++++++++++++++++++ test/unit/common/middleware/s3api/__init__.py | 18 +- test/unit/common/middleware/s3api/helpers.py | 100 +-- .../common/middleware/s3api/test_bucket.py | 529 +++++++++++- .../middleware/s3api/test_multi_delete.py | 138 +++- .../middleware/s3api/test_multi_upload.py | 8 +- test/unit/common/middleware/s3api/test_obj.py | 385 ++++++++- .../middleware/s3api/test_versioning.py | 174 +++- 27 files changed, 2737 insertions(+), 415 deletions(-) create mode 100644 test/functional/s3api/test_versioning.py create mode 100644 test/s3api/test_versioning.py diff --git a/doc/s3api/conf/ceph-known-failures-keystone.yaml b/doc/s3api/conf/ceph-known-failures-keystone.yaml index 3a50627bbf..7c5ce7f184 100644 --- a/doc/s3api/conf/ceph-known-failures-keystone.yaml +++ b/doc/s3api/conf/ceph-known-failures-keystone.yaml @@ -91,23 +91,16 @@ ceph_s3: s3tests.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed: {status: KNOWN} s3tests.functional.test_s3.test_set_cors: {status: KNOWN} s3tests.functional.test_s3.test_stress_bucket_acls_changes: {status: KNOWN} - s3tests.functional.test_s3.test_versioned_concurrent_object_create_and_remove: {status: KNOWN} s3tests.functional.test_s3.test_versioned_concurrent_object_create_concurrent_remove: {status: KNOWN} s3tests.functional.test_s3.test_versioned_object_acl: {status: KNOWN} - s3tests.functional.test_s3.test_versioning_bucket_create_suspend: {status: KNOWN} s3tests.functional.test_s3.test_versioning_copy_obj_version: {status: KNOWN} s3tests.functional.test_s3.test_versioning_multi_object_delete: {status: KNOWN} s3tests.functional.test_s3.test_versioning_multi_object_delete_with_marker: {status: KNOWN} s3tests.functional.test_s3.test_versioning_multi_object_delete_with_marker_create: {status: KNOWN} s3tests.functional.test_s3.test_versioning_obj_create_overwrite_multipart: {status: KNOWN} - s3tests.functional.test_s3.test_versioning_obj_create_read_remove: {status: KNOWN} s3tests.functional.test_s3.test_versioning_obj_create_read_remove_head: {status: KNOWN} s3tests.functional.test_s3.test_versioning_obj_create_versions_remove_all: {status: KNOWN} s3tests.functional.test_s3.test_versioning_obj_create_versions_remove_special_names: {status: KNOWN} - s3tests.functional.test_s3.test_versioning_obj_list_marker: {status: KNOWN} - s3tests.functional.test_s3.test_versioning_obj_plain_null_version_overwrite: {status: KNOWN} - s3tests.functional.test_s3.test_versioning_obj_plain_null_version_overwrite_suspended: {status: KNOWN} - s3tests.functional.test_s3.test_versioning_obj_plain_null_version_removal: {status: KNOWN} s3tests.functional.test_s3.test_versioning_obj_suspend_versions: {status: KNOWN} s3tests.functional.test_s3.test_versioning_obj_suspend_versions_simple: {status: KNOWN} s3tests.functional.test_s3_website.check_can_test_website: {status: KNOWN} @@ -177,9 +170,6 @@ ceph_s3: s3tests.functional.test_s3.test_lifecycle_set_multipart: {status: KNOWN} s3tests.functional.test_s3.test_lifecycle_set_noncurrent: {status: KNOWN} s3tests.functional.test_s3.test_multipart_copy_invalid_range: {status: KNOWN} - s3tests.functional.test_s3.test_multipart_copy_versioned: {status: KNOWN} - s3tests.functional.test_s3.test_object_copy_versioned_bucket: {status: KNOWN} - s3tests.functional.test_s3.test_object_copy_versioning_multipart_upload: {status: KNOWN} s3tests.functional.test_s3.test_post_object_empty_conditions: {status: KNOWN} s3tests.functional.test_s3.test_post_object_tags_anonymous_request: {status: KNOWN} s3tests.functional.test_s3.test_post_object_tags_authenticated_request: {status: KNOWN} diff --git a/doc/s3api/conf/ceph-known-failures-tempauth.yaml b/doc/s3api/conf/ceph-known-failures-tempauth.yaml index 0d9b833acb..99b77ec7ea 100644 --- a/doc/s3api/conf/ceph-known-failures-tempauth.yaml +++ b/doc/s3api/conf/ceph-known-failures-tempauth.yaml @@ -1,5 +1,6 @@ ceph_s3: :teardown: {status: KNOWN} + :teardown: {status: KNOWN} :setup: {status: KNOWN} s3tests.functional.test_headers.test_bucket_create_bad_authorization_invalid_aws2: {status: KNOWN} s3tests.functional.test_headers.test_bucket_create_bad_authorization_none: {status: KNOWN} @@ -45,7 +46,6 @@ ceph_s3: s3tests.functional.test_s3.test_append_object_position_wrong: {status: KNOWN} s3tests.functional.test_s3.test_append_normal_object: {status: KNOWN} s3tests.functional.test_s3.test_append_object: {status: KNOWN} - s3tests.functional.test_s3.test_versioning_obj_read_not_exist_null: {status: KNOWN} s3tests_boto3.functional.test_headers.test_bucket_create_bad_authorization_empty: {status: KNOWN} s3tests_boto3.functional.test_headers.test_bucket_create_bad_authorization_invalid_aws2: {status: KNOWN} s3tests_boto3.functional.test_headers.test_bucket_create_bad_authorization_none: {status: KNOWN} @@ -151,16 +151,12 @@ ceph_s3: s3tests_boto3.functional.test_s3.test_list_buckets_invalid_auth: {status: KNOWN} s3tests_boto3.functional.test_s3.test_logging_toggle: {status: KNOWN} s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_multipart_copy_versioned: {status: KNOWN} s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last: {status: KNOWN} s3tests_boto3.functional.test_s3.test_multipart_upload: {status: KNOWN} s3tests_boto3.functional.test_s3.test_multipart_upload_empty: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_acl_canned_bucketownerread: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_anon_put: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_anon_put_write_access: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_object_copy_versioned_bucket: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_object_copy_versioned_url_encoding: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_object_copy_versioning_multipart_upload: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_header_acl_grants: {status: KNOWN} s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_legal_hold_off: {status: KNOWN} @@ -265,24 +261,6 @@ ceph_s3: s3tests_boto3.functional.test_s3.test_sse_kms_transfer_1MB: {status: KNOWN} s3tests_boto3.functional.test_s3.test_sse_kms_transfer_1b: {status: KNOWN} s3tests_boto3.functional.test_s3.test_sse_kms_transfer_1kb: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioned_concurrent_object_create_and_remove: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioned_concurrent_object_create_concurrent_remove: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioned_object_acl: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioned_object_acl_no_version_specified: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_bucket_atomic_upload_return_version_id: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_bucket_create_suspend: {status: KNOWN} s3tests_boto3.functional.test_s3.test_versioning_bucket_multipart_upload_return_version_id: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_copy_obj_version: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_multi_object_delete: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_multi_object_delete_with_marker: {status: KNOWN} s3tests_boto3.functional.test_s3.test_versioning_multi_object_delete_with_marker_create: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_create_overwrite_multipart: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_create_read_remove: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_create_read_remove_head: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_create_versions_remove_all: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_create_versions_remove_special_names: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_list_marker: {status: KNOWN} s3tests_boto3.functional.test_s3.test_versioning_obj_plain_null_version_overwrite: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_plain_null_version_overwrite_suspended: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_plain_null_version_removal: {status: KNOWN} - s3tests_boto3.functional.test_s3.test_versioning_obj_suspend_versions: {status: KNOWN} diff --git a/doc/source/s3_compat.rst b/doc/source/s3_compat.rst index f0fad2fd6e..7ad655ceea 100644 --- a/doc/source/s3_compat.rst +++ b/doc/source/s3_compat.rst @@ -62,7 +62,7 @@ Amazon S3 operations +------------------------------------------------+------------------+--------------+ | `Object tagging`_ | Core-API | Yes | +------------------------------------------------+------------------+--------------+ -| `Versioning`_ | Versioning | No | +| `Versioning`_ | Versioning | Yes | +------------------------------------------------+------------------+--------------+ | `Bucket notification`_ | Notifications | No | +------------------------------------------------+------------------+--------------+ diff --git a/swift/common/middleware/s3api/acl_handlers.py b/swift/common/middleware/s3api/acl_handlers.py index 03bac56ddd..9064805cbe 100644 --- a/swift/common/middleware/s3api/acl_handlers.py +++ b/swift/common/middleware/s3api/acl_handlers.py @@ -128,9 +128,14 @@ class BaseAclHandler(object): raise Exception('No permission to be checked exists') if resource == 'object': + version_id = self.req.params.get('versionId') + if version_id is None: + query = {} + else: + query = {'version-id': version_id} resp = self.req.get_acl_response(app, 'HEAD', container, obj, - headers) + headers, query=query) acl = resp.object_acl elif resource == 'container': resp = self.req.get_acl_response(app, 'HEAD', @@ -460,4 +465,9 @@ ACL_MAP = { # Initiate Multipart Upload ('POST', 'HEAD', 'container'): {'Permission': 'WRITE'}, + # Versioning + ('PUT', 'POST', 'container'): + {'Permission': 'WRITE'}, + ('DELETE', 'GET', 'container'): + {'Permission': 'WRITE'}, } diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index ae67d8067f..55603cb597 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -21,13 +21,16 @@ from six.moves.urllib.parse import quote from swift.common import swob from swift.common.http import HTTP_OK -from swift.common.utils import json, public, config_true_value +from swift.common.middleware.versioned_writes.object_versioning import \ + DELETE_MARKER_CONTENT_TYPE +from swift.common.utils import json, public, config_true_value, Timestamp, \ + get_swift_info from swift.common.middleware.s3api.controllers.base import Controller -from swift.common.middleware.s3api.etree import Element, SubElement, tostring, \ - fromstring, XMLSyntaxError, DocumentInvalid -from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \ - InvalidArgument, \ +from swift.common.middleware.s3api.etree import Element, SubElement, \ + tostring, fromstring, XMLSyntaxError, DocumentInvalid +from swift.common.middleware.s3api.s3response import \ + HTTPOk, S3NotImplemented, InvalidArgument, \ MalformedXML, InvalidLocationConstraint, NoSuchBucket, \ BucketNotEmpty, InternalError, ServiceUnavailable, NoSuchKey from swift.common.middleware.s3api.utils import MULTIUPLOAD_SUFFIX @@ -94,36 +97,38 @@ class BucketController(Controller): return HTTPOk(headers=resp.headers) - @public - def GET(self, req): - """ - Handle GET Bucket (List Objects) request - """ - - max_keys = req.get_validated_param( - 'max-keys', self.conf.max_bucket_listing) - # TODO: Separate max_bucket_listing and default_bucket_listing - tag_max_keys = max_keys - max_keys = min(max_keys, self.conf.max_bucket_listing) - + def _parse_request_options(self, req, max_keys): encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) + # in order to judge that truncated is valid, check whether + # max_keys + 1 th element exists in swift. query = { - 'format': 'json', 'limit': max_keys + 1, } if 'prefix' in req.params: - query.update({'prefix': req.params['prefix']}) + query['prefix'] = req.params['prefix'] if 'delimiter' in req.params: - query.update({'delimiter': req.params['delimiter']}) + query['delimiter'] = req.params['delimiter'] fetch_owner = False if 'versions' in req.params: + query['versions'] = req.params['versions'] listing_type = 'object-versions' if 'key-marker' in req.params: - query.update({'marker': req.params['key-marker']}) + query['marker'] = req.params['key-marker'] + version_marker = req.params.get('version-id-marker') + if version_marker is not None: + if version_marker != 'null': + try: + Timestamp(version_marker) + except ValueError: + raise InvalidArgument( + 'version-id-marker', + req.params['version-id-marker'], + 'Invalid version id specified') + query['version_marker'] = version_marker elif 'version-id-marker' in req.params: err_msg = ('A version-id marker cannot be specified without ' 'a key marker.') @@ -132,132 +137,190 @@ class BucketController(Controller): elif int(req.params.get('list-type', '1')) == 2: listing_type = 'version-2' if 'start-after' in req.params: - query.update({'marker': req.params['start-after']}) + query['marker'] = req.params['start-after'] # continuation-token overrides start-after if 'continuation-token' in req.params: decoded = b64decode(req.params['continuation-token']) if not six.PY2: decoded = decoded.decode('utf8') - query.update({'marker': decoded}) + query['marker'] = decoded if 'fetch-owner' in req.params: fetch_owner = config_true_value(req.params['fetch-owner']) else: listing_type = 'version-1' if 'marker' in req.params: - query.update({'marker': req.params['marker']}) + query['marker'] = req.params['marker'] + + return encoding_type, query, listing_type, fetch_owner + + def _build_versions_result(self, req, objects, is_truncated): + elem = Element('ListVersionsResult') + SubElement(elem, 'Name').text = req.container_name + SubElement(elem, 'Prefix').text = req.params.get('prefix') + SubElement(elem, 'KeyMarker').text = req.params.get('key-marker') + SubElement(elem, 'VersionIdMarker').text = req.params.get( + 'version-id-marker') + if is_truncated: + if 'name' in objects[-1]: + SubElement(elem, 'NextKeyMarker').text = \ + objects[-1]['name'] + SubElement(elem, 'NextVersionIdMarker').text = \ + objects[-1].get('version') or 'null' + if 'subdir' in objects[-1]: + SubElement(elem, 'NextKeyMarker').text = \ + objects[-1]['subdir'] + SubElement(elem, 'NextVersionIdMarker').text = 'null' + return elem + + def _build_base_listing_element(self, req): + elem = Element('ListBucketResult') + SubElement(elem, 'Name').text = req.container_name + SubElement(elem, 'Prefix').text = req.params.get('prefix') + return elem + + def _build_list_bucket_result_type_one(self, req, objects, encoding_type, + is_truncated): + elem = self._build_base_listing_element(req) + SubElement(elem, 'Marker').text = req.params.get('marker') + if is_truncated and 'delimiter' in req.params: + if 'name' in objects[-1]: + name = objects[-1]['name'] + else: + name = objects[-1]['subdir'] + if encoding_type == 'url': + name = quote(name.encode('utf-8')) + SubElement(elem, 'NextMarker').text = name + # XXX: really? no NextMarker when no delimiter?? + return elem + + def _build_list_bucket_result_type_two(self, req, objects, is_truncated): + elem = self._build_base_listing_element(req) + if is_truncated: + if 'name' in objects[-1]: + SubElement(elem, 'NextContinuationToken').text = \ + b64encode(objects[-1]['name'].encode('utf8')) + if 'subdir' in objects[-1]: + SubElement(elem, 'NextContinuationToken').text = \ + b64encode(objects[-1]['subdir'].encode('utf8')) + if 'continuation-token' in req.params: + SubElement(elem, 'ContinuationToken').text = \ + req.params['continuation-token'] + if 'start-after' in req.params: + SubElement(elem, 'StartAfter').text = \ + req.params['start-after'] + SubElement(elem, 'KeyCount').text = str(len(objects)) + return elem + + def _finish_result(self, req, elem, tag_max_keys, encoding_type, + is_truncated): + SubElement(elem, 'MaxKeys').text = str(tag_max_keys) + if 'delimiter' in req.params: + SubElement(elem, 'Delimiter').text = req.params['delimiter'] + if encoding_type == 'url': + SubElement(elem, 'EncodingType').text = encoding_type + SubElement(elem, 'IsTruncated').text = \ + 'true' if is_truncated else 'false' + + def _add_subdir(self, elem, o, encoding_type): + common_prefixes = SubElement(elem, 'CommonPrefixes') + name = o['subdir'] + if encoding_type == 'url': + name = quote(name.encode('utf-8')) + SubElement(common_prefixes, 'Prefix').text = name + + def _add_object(self, req, elem, o, encoding_type, listing_type, + fetch_owner): + name = o['name'] + if encoding_type == 'url': + name = quote(name.encode('utf-8')) + + if listing_type == 'object-versions': + if o['content_type'] == DELETE_MARKER_CONTENT_TYPE: + contents = SubElement(elem, 'DeleteMarker') + else: + contents = SubElement(elem, 'Version') + SubElement(contents, 'Key').text = name + SubElement(contents, 'VersionId').text = o.get( + 'version_id') or 'null' + if 'object_versioning' in get_swift_info(): + SubElement(contents, 'IsLatest').text = ( + 'true' if o['is_latest'] else 'false') + else: + SubElement(contents, 'IsLatest').text = 'true' + else: + contents = SubElement(elem, 'Contents') + SubElement(contents, 'Key').text = name + SubElement(contents, 'LastModified').text = \ + o['last_modified'][:-3] + 'Z' + if contents.tag != 'DeleteMarker': + if 's3_etag' in o: + # New-enough MUs are already in the right format + etag = o['s3_etag'] + elif 'slo_etag' in o: + # SLOs may be in something *close* to the MU format + etag = '"%s-N"' % o['slo_etag'].strip('"') + else: + # Normal objects just use the MD5 + etag = o['hash'] + if len(etag) < 2 or etag[::len(etag) - 1] != '""': + # Normal objects just use the MD5 + etag = '"%s"' % o['hash'] + # This also catches sufficiently-old SLOs, but we have + # no way to identify those from container listings + # Otherwise, somebody somewhere (proxyfs, maybe?) made this + # look like an RFC-compliant ETag; we don't need to + # quote-wrap. + SubElement(contents, 'ETag').text = etag + SubElement(contents, 'Size').text = str(o['bytes']) + if fetch_owner or listing_type != 'version-2': + owner = SubElement(contents, 'Owner') + SubElement(owner, 'ID').text = req.user_id + SubElement(owner, 'DisplayName').text = req.user_id + if contents.tag != 'DeleteMarker': + SubElement(contents, 'StorageClass').text = 'STANDARD' + + def _add_objects_to_result(self, req, elem, objects, encoding_type, + listing_type, fetch_owner): + for o in objects: + if 'subdir' in o: + self._add_subdir(elem, o, encoding_type) + else: + self._add_object(req, elem, o, encoding_type, listing_type, + fetch_owner) + + @public + def GET(self, req): + """ + Handle GET Bucket (List Objects) request + """ + max_keys = req.get_validated_param( + 'max-keys', self.conf.max_bucket_listing) + tag_max_keys = max_keys + # TODO: Separate max_bucket_listing and default_bucket_listing + max_keys = min(max_keys, self.conf.max_bucket_listing) + + encoding_type, query, listing_type, fetch_owner = \ + self._parse_request_options(req, max_keys) resp = req.get_response(self.app, query=query) objects = json.loads(resp.body) - # in order to judge that truncated is valid, check whether - # max_keys + 1 th element exists in swift. is_truncated = max_keys > 0 and len(objects) > max_keys objects = objects[:max_keys] if listing_type == 'object-versions': - elem = Element('ListVersionsResult') - SubElement(elem, 'Name').text = req.container_name - SubElement(elem, 'Prefix').text = req.params.get('prefix') - SubElement(elem, 'KeyMarker').text = req.params.get('key-marker') - SubElement(elem, 'VersionIdMarker').text = req.params.get( - 'version-id-marker') - if is_truncated: - if 'name' in objects[-1]: - SubElement(elem, 'NextKeyMarker').text = \ - objects[-1]['name'] - if 'subdir' in objects[-1]: - SubElement(elem, 'NextKeyMarker').text = \ - objects[-1]['subdir'] - SubElement(elem, 'NextVersionIdMarker').text = 'null' + elem = self._build_versions_result(req, objects, is_truncated) + elif listing_type == 'version-2': + elem = self._build_list_bucket_result_type_two( + req, objects, is_truncated) else: - elem = Element('ListBucketResult') - SubElement(elem, 'Name').text = req.container_name - SubElement(elem, 'Prefix').text = req.params.get('prefix') - if listing_type == 'version-1': - SubElement(elem, 'Marker').text = req.params.get('marker') - if is_truncated and 'delimiter' in req.params: - if 'name' in objects[-1]: - name = objects[-1]['name'] - else: - name = objects[-1]['subdir'] - if encoding_type == 'url': - name = quote(name.encode('utf-8')) - SubElement(elem, 'NextMarker').text = name - elif listing_type == 'version-2': - if is_truncated: - if 'name' in objects[-1]: - SubElement(elem, 'NextContinuationToken').text = \ - b64encode(objects[-1]['name'].encode('utf-8')) - if 'subdir' in objects[-1]: - SubElement(elem, 'NextContinuationToken').text = \ - b64encode(objects[-1]['subdir'].encode('utf-8')) - if 'continuation-token' in req.params: - SubElement(elem, 'ContinuationToken').text = \ - req.params['continuation-token'] - if 'start-after' in req.params: - SubElement(elem, 'StartAfter').text = \ - req.params['start-after'] - SubElement(elem, 'KeyCount').text = str(len(objects)) - - SubElement(elem, 'MaxKeys').text = str(tag_max_keys) - - if 'delimiter' in req.params: - SubElement(elem, 'Delimiter').text = req.params['delimiter'] - - if encoding_type == 'url': - SubElement(elem, 'EncodingType').text = encoding_type - - SubElement(elem, 'IsTruncated').text = \ - 'true' if is_truncated else 'false' - - for o in objects: - if 'subdir' not in o: - name = o['name'] - if encoding_type == 'url': - name = quote(name.encode('utf-8')) - - if listing_type == 'object-versions': - contents = SubElement(elem, 'Version') - SubElement(contents, 'Key').text = name - SubElement(contents, 'VersionId').text = 'null' - SubElement(contents, 'IsLatest').text = 'true' - else: - contents = SubElement(elem, 'Contents') - SubElement(contents, 'Key').text = name - SubElement(contents, 'LastModified').text = \ - o['last_modified'][:-3] + 'Z' - if 's3_etag' in o: - # New-enough MUs are already in the right format - etag = o['s3_etag'] - elif 'slo_etag' in o: - # SLOs may be in something *close* to the MU format - etag = '"%s-N"' % swob.normalize_etag(o['slo_etag']) - else: - etag = o['hash'] - if len(etag) < 2 or etag[::len(etag) - 1] != '""': - # Normal objects just use the MD5 - etag = '"%s"' % o['hash'] - # This also catches sufficiently-old SLOs, but we have - # no way to identify those from container listings - # Otherwise, somebody somewhere (proxyfs, maybe?) made this - # look like an RFC-compliant ETag; we don't need to - # quote-wrap. - SubElement(contents, 'ETag').text = etag - SubElement(contents, 'Size').text = str(o['bytes']) - if fetch_owner or listing_type != 'version-2': - owner = SubElement(contents, 'Owner') - SubElement(owner, 'ID').text = req.user_id - SubElement(owner, 'DisplayName').text = req.user_id - SubElement(contents, 'StorageClass').text = 'STANDARD' - - for o in objects: - if 'subdir' in o: - common_prefixes = SubElement(elem, 'CommonPrefixes') - name = o['subdir'] - if encoding_type == 'url': - name = quote(name.encode('utf-8')) - SubElement(common_prefixes, 'Prefix').text = name + elem = self._build_list_bucket_result_type_one( + req, objects, encoding_type, is_truncated) + self._finish_result( + req, elem, tag_max_keys, encoding_type, is_truncated) + self._add_objects_to_result( + req, elem, objects, encoding_type, listing_type, fetch_owner) body = tostring(elem) @@ -297,6 +360,7 @@ class BucketController(Controller): """ Handle DELETE Bucket request """ + # NB: object_versioning is responsible for cleaning up its container if self.conf.allow_multipart_uploads: self._delete_segments_bucket(req) resp = req.get_response(self.app) diff --git a/swift/common/middleware/s3api/controllers/multi_delete.py b/swift/common/middleware/s3api/controllers/multi_delete.py index 085cdf3e52..7e7a311217 100644 --- a/swift/common/middleware/s3api/controllers/multi_delete.py +++ b/swift/common/middleware/s3api/controllers/multi_delete.py @@ -17,15 +17,15 @@ import copy import json from swift.common.constraints import MAX_OBJECT_NAME_LENGTH -from swift.common.utils import public, StreamingPile +from swift.common.utils import public, StreamingPile, get_swift_info from swift.common.middleware.s3api.controllers.base import Controller, \ bucket_operation from swift.common.middleware.s3api.etree import Element, SubElement, \ fromstring, tostring, XMLSyntaxError, DocumentInvalid -from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \ - NoSuchKey, ErrorResponse, MalformedXML, UserKeyMustBeSpecified, \ - AccessDenied, MissingRequestBodyError +from swift.common.middleware.s3api.s3response import HTTPOk, \ + S3NotImplemented, NoSuchKey, ErrorResponse, MalformedXML, \ + UserKeyMustBeSpecified, AccessDenied, MissingRequestBodyError class MultiObjectDeleteController(Controller): @@ -35,12 +35,10 @@ class MultiObjectDeleteController(Controller): """ def _gen_error_body(self, error, elem, delete_list): for key, version in delete_list: - if version is not None: - # TODO: delete the specific version of the object - raise S3NotImplemented() - error_elem = SubElement(elem, 'Error') SubElement(error_elem, 'Key').text = key + if version is not None: + SubElement(error_elem, 'VersionId').text = version SubElement(error_elem, 'Code').text = error.__class__.__name__ SubElement(error_elem, 'Message').text = error._msg @@ -105,21 +103,32 @@ class MultiObjectDeleteController(Controller): body = self._gen_error_body(error, elem, delete_list) return HTTPOk(body=body) - if any(version is not None for _key, version in delete_list): - # TODO: support deleting specific versions of objects + if 'object_versioning' not in get_swift_info() and any( + version not in ('null', None) + for _key, version in delete_list): raise S3NotImplemented() def do_delete(base_req, key, version): req = copy.copy(base_req) req.environ = copy.copy(base_req.environ) req.object_name = key + if version: + req.params = {'version-id': version, 'symlink': 'get'} try: - query = req.gen_multipart_manifest_delete_query(self.app) + try: + query = req.gen_multipart_manifest_delete_query( + self.app, version=version) + except NoSuchKey: + query = {} + if version: + query['version-id'] = version + query['symlink'] = 'get' + resp = req.get_response(self.app, method='DELETE', query=query, headers={'Accept': 'application/json'}) # Have to read the response to actually do the SLO delete - if query: + if query.get('multipart-manifest'): try: delete_result = json.loads(resp.body) if delete_result['Errors']: @@ -144,6 +153,12 @@ class MultiObjectDeleteController(Controller): pass except ErrorResponse as e: return key, {'code': e.__class__.__name__, 'message': e._msg} + except Exception: + self.logger.exception( + 'Unexpected Error handling DELETE of %r %r' % ( + req.container_name, key)) + return key, {'code': 'Server Error', 'message': 'Server Error'} + return key, None with StreamingPile(self.conf.multi_delete_concurrency) as pile: diff --git a/swift/common/middleware/s3api/controllers/multi_upload.py b/swift/common/middleware/s3api/controllers/multi_upload.py index 4062c7e08d..0dbbb9934e 100644 --- a/swift/common/middleware/s3api/controllers/multi_upload.py +++ b/swift/common/middleware/s3api/controllers/multi_upload.py @@ -100,10 +100,19 @@ def _get_upload_info(req, app, upload_id): container = req.container_name + MULTIUPLOAD_SUFFIX obj = '%s/%s' % (req.object_name, upload_id) + # XXX: if we leave the copy-source header, somewhere later we might + # drop in a ?version-id=... query string that's utterly inappropriate + # for the upload marker. Until we get around to fixing that, just pop + # it off for now... + copy_source = req.headers.pop('X-Amz-Copy-Source', None) try: return req.get_response(app, 'HEAD', container=container, obj=obj) except NoSuchKey: raise NoSuchUpload(upload_id=upload_id) + finally: + # ...making sure to restore any copy-source before returning + if copy_source is not None: + req.headers['X-Amz-Copy-Source'] = copy_source def _check_upload_info(req, app, upload_id): diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py index fe2b589fcc..293b147029 100644 --- a/swift/common/middleware/s3api/controllers/obj.py +++ b/swift/common/middleware/s3api/controllers/obj.py @@ -13,16 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json + from swift.common.http import HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_NO_CONTENT from swift.common.request_helpers import update_etag_is_at_header from swift.common.swob import Range, content_range_header_value, \ normalize_etag -from swift.common.utils import public, list_from_csv +from swift.common.utils import public, list_from_csv, get_swift_info +from swift.common.middleware.versioned_writes.object_versioning import \ + DELETE_MARKER_CONTENT_TYPE from swift.common.middleware.s3api.utils import S3Timestamp, sysmeta_header from swift.common.middleware.s3api.controllers.base import Controller from swift.common.middleware.s3api.s3response import S3NotImplemented, \ - InvalidRange, NoSuchKey, InvalidArgument, HTTPNoContent + InvalidRange, NoSuchKey, InvalidArgument, HTTPNoContent, \ + PreconditionFailed class ObjectController(Controller): @@ -78,11 +83,20 @@ class ObjectController(Controller): # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) - resp = req.get_response(self.app) + object_name = req.object_name + version_id = req.params.get('versionId') + if version_id not in ('null', None) and \ + 'object_versioning' not in get_swift_info(): + raise S3NotImplemented() + query = {} if version_id is None else {'version-id': version_id} + resp = req.get_response(self.app, query=query) if req.method == 'HEAD': resp.app_iter = None + if 'x-amz-meta-deleted' in resp.headers: + raise NoSuchKey(object_name) + for key in ('content-type', 'content-language', 'expires', 'cache-control', 'content-disposition', 'content-encoding'): @@ -125,12 +139,14 @@ class ObjectController(Controller): req.headers['X-Amz-Copy-Source-Range'], 'Illegal copy header') req.check_copy_source(self.app) + if not req.headers.get('Content-Type'): + # can't setdefault because it can be None for some reason + req.headers['Content-Type'] = 'binary/octet-stream' resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) - # delete object metadata from response for key in list(resp.headers.keys()): if key.lower().startswith('x-amz-meta-'): @@ -143,20 +159,63 @@ class ObjectController(Controller): def POST(self, req): raise S3NotImplemented() + def _restore_on_delete(self, req): + resp = req.get_response(self.app, 'GET', req.container_name, '', + query={'prefix': req.object_name, + 'versions': True}) + if resp.status_int != HTTP_OK: + return resp + old_versions = json.loads(resp.body) + resp = None + for item in old_versions: + if item['content_type'] == DELETE_MARKER_CONTENT_TYPE: + resp = None + break + try: + resp = req.get_response(self.app, 'PUT', query={ + 'version-id': item['version_id']}) + except PreconditionFailed: + self.logger.debug('skipping failed PUT?version-id=%s' % + item['version_id']) + continue + # if that worked, we'll go ahead and fix up the status code + resp.status_int = HTTP_NO_CONTENT + break + return resp + @public def DELETE(self, req): """ Handle DELETE Object request """ + if 'versionId' in req.params and \ + req.params['versionId'] != 'null' and \ + 'object_versioning' not in get_swift_info(): + raise S3NotImplemented() + try: - query = req.gen_multipart_manifest_delete_query(self.app) + try: + query = req.gen_multipart_manifest_delete_query( + self.app, version=req.params.get('versionId')) + except NoSuchKey: + query = {} + req.headers['Content-Type'] = None # Ignore client content-type + + if 'versionId' in req.params: + query['version-id'] = req.params['versionId'] + query['symlink'] = 'get' + resp = req.get_response(self.app, query=query) - if query and resp.status_int == HTTP_OK: + if query.get('multipart-manifest') and resp.status_int == HTTP_OK: for chunk in resp.app_iter: pass # drain the bulk-deleter response resp.status = HTTP_NO_CONTENT resp.body = b'' + if resp.sw_headers.get('X-Object-Current-Version-Id') == 'null': + new_resp = self._restore_on_delete(req) + if new_resp: + resp = new_resp except NoSuchKey: # expect to raise NoSuchBucket when the bucket doesn't exist req.get_container_info(self.app) diff --git a/swift/common/middleware/s3api/controllers/versioning.py b/swift/common/middleware/s3api/controllers/versioning.py index 21d49cc0e5..9c835f3c29 100644 --- a/swift/common/middleware/s3api/controllers/versioning.py +++ b/swift/common/middleware/s3api/controllers/versioning.py @@ -13,12 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.common.utils import public +from swift.common.utils import public, get_swift_info, config_true_value from swift.common.middleware.s3api.controllers.base import Controller, \ bucket_operation -from swift.common.middleware.s3api.etree import Element, tostring -from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented +from swift.common.middleware.s3api.etree import Element, tostring, \ + fromstring, XMLSyntaxError, DocumentInvalid, SubElement +from swift.common.middleware.s3api.s3response import HTTPOk, \ + S3NotImplemented, MalformedXML + +MAX_PUT_VERSIONING_BODY_SIZE = 10240 class VersioningController(Controller): @@ -36,13 +40,16 @@ class VersioningController(Controller): """ Handles GET Bucket versioning. """ - req.get_response(self.app, method='HEAD') + sysmeta = req.get_container_info(self.app).get('sysmeta', {}) - # Just report there is no versioning configured here. elem = Element('VersioningConfiguration') + if sysmeta.get('versions-enabled'): + SubElement(elem, 'Status').text = ( + 'Enabled' if config_true_value(sysmeta['versions-enabled']) + else 'Suspended') body = tostring(elem) - return HTTPOk(body=body, content_type="text/plain") + return HTTPOk(body=body, content_type=None) @public @bucket_operation @@ -50,4 +57,25 @@ class VersioningController(Controller): """ Handles PUT Bucket versioning. """ - raise S3NotImplemented() + if 'object_versioning' not in get_swift_info(): + raise S3NotImplemented() + + xml = req.xml(MAX_PUT_VERSIONING_BODY_SIZE) + try: + elem = fromstring(xml, 'VersioningConfiguration') + status = elem.find('./Status').text + except (XMLSyntaxError, DocumentInvalid): + raise MalformedXML() + except Exception as e: + self.logger.error(e) + raise + + if status not in ['Enabled', 'Suspended']: + raise MalformedXML() + + # Set up versioning + # NB: object_versioning responsible for ensuring its container exists + req.headers['X-Versions-Enabled'] = str(status == 'Enabled').lower() + req.get_response(self.app, 'POST') + + return HTTPOk() diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index 130f54085f..017aabe7e6 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -877,20 +877,16 @@ class S3Request(swob.Request): except KeyError: return None - if '?' in src_path: - src_path, qs = src_path.split('?', 1) - query = parse_qsl(qs, True) - if not query: - pass # ignore it - elif len(query) > 1 or query[0][0] != 'versionId': - raise InvalidArgument('X-Amz-Copy-Source', - self.headers['X-Amz-Copy-Source'], - 'Unsupported copy source parameter.') - elif query[0][1] != 'null': - # TODO: once we support versioning, we'll need to translate - # src_path to the proper location in the versions container - raise S3NotImplemented('Versioning is not yet supported') - self.headers['X-Amz-Copy-Source'] = src_path + src_path, qs = src_path.partition('?')[::2] + parsed = parse_qsl(qs, True) + if not parsed: + query = {} + elif len(parsed) == 1 and parsed[0][0] == 'versionId': + query = {'version-id': parsed[0][1]} + else: + raise InvalidArgument('X-Amz-Copy-Source', + self.headers['X-Amz-Copy-Source'], + 'Unsupported copy source parameter.') src_path = unquote(src_path) src_path = src_path if src_path.startswith('/') else ('/' + src_path) @@ -900,19 +896,15 @@ class S3Request(swob.Request): headers.update(self._copy_source_headers()) src_resp = self.get_response(app, 'HEAD', src_bucket, src_obj, - headers=headers) + headers=headers, query=query) if src_resp.status_int == 304: # pylint: disable-msg=E1101 raise PreconditionFailed() - self.headers['X-Amz-Copy-Source'] = \ - '/' + self.headers['X-Amz-Copy-Source'].lstrip('/') - source_container, source_obj = \ - split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True) - - if (self.container_name == source_container and - self.object_name == source_obj and + if (self.container_name == src_bucket and + self.object_name == src_obj and self.headers.get('x-amz-metadata-directive', - 'COPY') == 'COPY'): + 'COPY') == 'COPY' and + not query): raise InvalidRequest("This copy request is illegal " "because it is trying to copy an " "object to itself without " @@ -920,6 +912,12 @@ class S3Request(swob.Request): "storage class, website redirect " "location or encryption " "attributes.") + # We've done some normalizing; write back so it's ready for + # to_swift_req + self.headers['X-Amz-Copy-Source'] = quote(src_path) + if query: + self.headers['X-Amz-Copy-Source'] += \ + '?versionId=' + query['version-id'] return src_resp def _canonical_uri(self): @@ -1064,6 +1062,7 @@ class S3Request(swob.Request): account = self.account env = self.environ.copy() + env['swift.infocache'] = self.environ.setdefault('swift.infocache', {}) def sanitize(value): if set(value).issubset(string.printable): @@ -1109,8 +1108,10 @@ class S3Request(swob.Request): env['HTTP_X_OBJECT_META_' + key[16:]] = sanitize(env[key]) del env[key] - if 'HTTP_X_AMZ_COPY_SOURCE' in env: - env['HTTP_X_COPY_FROM'] = env['HTTP_X_AMZ_COPY_SOURCE'] + copy_from_version_id = '' + if 'HTTP_X_AMZ_COPY_SOURCE' in env and env['REQUEST_METHOD'] == 'PUT': + env['HTTP_X_COPY_FROM'], copy_from_version_id = env[ + 'HTTP_X_AMZ_COPY_SOURCE'].partition('?versionId=')[::2] del env['HTTP_X_AMZ_COPY_SOURCE'] env['CONTENT_LENGTH'] = '0' if env.pop('HTTP_X_AMZ_METADATA_DIRECTIVE', None) == 'REPLACE': @@ -1143,16 +1144,16 @@ class S3Request(swob.Request): path = '/v1/%s' % (account) env['PATH_INFO'] = path - query_string = '' + params = [] if query is not None: - params = [] for key, value in sorted(query.items()): if value is not None: params.append('%s=%s' % (key, quote(str(value)))) else: params.append(key) - query_string = '&'.join(params) - env['QUERY_STRING'] = query_string + if copy_from_version_id and not (query and query.get('version-id')): + params.append('version-id=' + copy_from_version_id) + env['QUERY_STRING'] = '&'.join(params) return swob.Request.blank(quote(path), environ=env, body=body, headers=headers) @@ -1292,6 +1293,7 @@ class S3Request(swob.Request): HTTP_REQUEST_ENTITY_TOO_LARGE: EntityTooLarge, HTTP_LENGTH_REQUIRED: MissingContentLength, HTTP_REQUEST_TIMEOUT: RequestTimeout, + HTTP_PRECONDITION_FAILED: PreconditionFailed, }, 'POST': { HTTP_NOT_FOUND: not_found_handler, @@ -1445,14 +1447,16 @@ class S3Request(swob.Request): return headers_to_container_info( headers, resp.status_int) # pylint: disable-msg=E1101 - def gen_multipart_manifest_delete_query(self, app, obj=None): + def gen_multipart_manifest_delete_query(self, app, obj=None, version=None): if not self.allow_multipart_uploads: - return None - query = {'multipart-manifest': 'delete'} + return {} if not obj: obj = self.object_name - resp = self.get_response(app, 'HEAD', obj=obj) - return query if resp.is_slo else None + query = {'symlink': 'get'} + if version is not None: + query['version-id'] = version + resp = self.get_response(app, 'HEAD', obj=obj, query=query) + return {'multipart-manifest': 'delete'} if resp.is_slo else {} def set_acl_handler(self, handler): pass diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py index 365bb047c5..fc4b9078d7 100644 --- a/swift/common/middleware/s3api/s3response.py +++ b/swift/common/middleware/s3api/s3response.py @@ -25,6 +25,8 @@ from swift.common.request_helpers import is_sys_meta from swift.common.middleware.s3api.utils import snake_to_camel, \ sysmeta_prefix, sysmeta_header from swift.common.middleware.s3api.etree import Element, SubElement, tostring +from swift.common.middleware.versioned_writes.object_versioning import \ + DELETE_MARKER_CONTENT_TYPE class HeaderKeyDict(header_key_dict.HeaderKeyDict): @@ -109,9 +111,16 @@ class S3Response(S3ResponseBase, swob.Response): 'etag', 'last-modified', 'x-robots-tag', 'cache-control', 'expires'): headers[key] = val + elif _key == 'x-object-version-id': + headers['x-amz-version-id'] = val + elif _key == 'x-copied-from-version-id': + headers['x-amz-copy-source-version-id'] = val elif _key == 'x-static-large-object': # for delete slo self.is_slo = config_true_value(val) + elif _key == 'x-backend-content-type' and \ + val == DELETE_MARKER_CONTENT_TYPE: + headers['x-amz-delete-marker'] = 'true' # Check whether we stored the AWS-style etag on upload override_etag = s3_sysmeta_headers.get( @@ -217,7 +226,7 @@ class ErrorResponse(S3ResponseBase, swob.HTTPException): def _dict_to_etree(self, parent, d): for key, value in d.items(): - tag = re.sub('\W', '', snake_to_camel(key)) + tag = re.sub(r'\W', '', snake_to_camel(key)) elem = SubElement(parent, tag) if isinstance(value, (dict, MutableMapping)): @@ -481,7 +490,7 @@ class MalformedPOSTRequest(ErrorResponse): class MalformedXML(ErrorResponse): _status = '400 Bad Request' _msg = 'The XML you provided was not well-formed or did not validate ' \ - 'against our published schema.' + 'against our published schema' class MaxMessageLengthExceeded(ErrorResponse): diff --git a/swift/common/middleware/s3api/utils.py b/swift/common/middleware/s3api/utils.py index 4a3e04776b..5cf501e4e3 100644 --- a/swift/common/middleware/s3api/utils.py +++ b/swift/common/middleware/s3api/utils.py @@ -95,8 +95,8 @@ def validate_bucket_name(name, dns_compliant_bucket_names): elif name.endswith('.'): # Bucket names must not end with dot return False - elif re.match("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)" - "{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", + elif re.match(r"^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.)" + r"{3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", name): # Bucket names cannot be formatted as an IP Address return False diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 660f73078e..4dda5f50de 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -741,7 +741,7 @@ def get_cluster_info(): conn = Connection(config) conn.authenticate() cluster_info.update(conn.cluster_info()) - except (ResponseError, socket.error): + except (ResponseError, socket.error, SkipTest): # Failed to get cluster_information via /info API, so fall back on # test.conf data pass @@ -1039,10 +1039,13 @@ def teardown_package(): global config if config: - conn = Connection(config) - conn.authenticate() - account = Account(conn, config.get('account', config['username'])) - account.delete_containers() + try: + conn = Connection(config) + conn.authenticate() + account = Account(conn, config.get('account', config['username'])) + account.delete_containers() + except (SkipTest): + pass global in_process global _test_socks diff --git a/test/functional/s3api/s3_test_client.py b/test/functional/s3api/s3_test_client.py index a6727047d0..0eadebd060 100644 --- a/test/functional/s3api/s3_test_client.py +++ b/test/functional/s3api/s3_test_client.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os import test.functional as tf import boto3 @@ -27,6 +28,12 @@ import traceback RETRY_COUNT = 3 +if os.environ.get('SWIFT_TEST_QUIET_BOTO_LOGS'): + logging.getLogger('boto').setLevel(logging.INFO) + logging.getLogger('botocore').setLevel(logging.INFO) + logging.getLogger('boto3').setLevel(logging.INFO) + + def setUpModule(): tf.setup_package() @@ -88,8 +95,9 @@ class Connection(object): for upload in bucket.list_multipart_uploads(): upload.cancel_upload() - for obj in bucket.list(): - bucket.delete_key(obj.name) + for obj in bucket.list_versions(): + bucket.delete_key( + obj.name, version_id=obj.version_id) self.conn.delete_bucket(bucket.name) except S3ResponseError as e: diff --git a/test/functional/s3api/test_multi_upload.py b/test/functional/s3api/test_multi_upload.py index ae01e0977a..f214617d87 100644 --- a/test/functional/s3api/test_multi_upload.py +++ b/test/functional/s3api/test_multi_upload.py @@ -84,9 +84,12 @@ class TestS3ApiMultiUpload(S3ApiBase): return status, headers, body def _upload_part_copy(self, src_bucket, src_obj, dst_bucket, dst_key, - upload_id, part_num=1, src_range=None): + upload_id, part_num=1, src_range=None, + src_version_id=None): src_path = '%s/%s' % (src_bucket, src_obj) + if src_version_id: + src_path += '?versionId=%s' % src_version_id query = 'partNumber=%s&uploadId=%s' % (part_num, upload_id) req_headers = {'X-Amz-Copy-Source': src_path} if src_range: @@ -877,6 +880,133 @@ class TestS3ApiMultiUpload(S3ApiBase): self.assertTrue('content-length' in headers) self.assertEqual(headers['content-length'], '0') + def test_object_multi_upload_part_copy_version(self): + bucket = 'bucket' + keys = ['obj1'] + uploads = [] + + results_generator = self._initiate_multi_uploads_result_generator( + bucket, keys) + + # Initiate Multipart Upload + for expected_key, (status, headers, body) in \ + zip(keys, results_generator): + self.assertEqual(status, 200) + self.assertCommonResponseHeaders(headers) + self.assertTrue('content-type' in headers) + self.assertEqual(headers['content-type'], 'application/xml') + self.assertTrue('content-length' in headers) + self.assertEqual(headers['content-length'], str(len(body))) + elem = fromstring(body, 'InitiateMultipartUploadResult') + self.assertEqual(elem.find('Bucket').text, bucket) + key = elem.find('Key').text + self.assertEqual(expected_key, key) + upload_id = elem.find('UploadId').text + self.assertTrue(upload_id is not None) + self.assertTrue((key, upload_id) not in uploads) + uploads.append((key, upload_id)) + + self.assertEqual(len(uploads), len(keys)) # sanity + + key, upload_id = uploads[0] + src_bucket = 'bucket2' + src_obj = 'obj4' + src_content = b'y' * (self.min_segment_size // 2) + b'z' * \ + self.min_segment_size + etags = [md5(src_content).hexdigest()] + + # prepare null-version src obj + self.conn.make_request('PUT', src_bucket) + self.conn.make_request('PUT', src_bucket, src_obj, body=src_content) + _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj) + self.assertCommonResponseHeaders(headers) + + # Turn on versioning + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', src_bucket, body=xml, query='versioning') + self.assertEqual(status, 200) + + src_obj2 = 'obj5' + src_content2 = b'stub' + etags.append(md5(src_content2).hexdigest()) + + # prepare src obj w/ real version + self.conn.make_request('PUT', src_bucket, src_obj2, body=src_content2) + _, headers, _ = self.conn.make_request('HEAD', src_bucket, src_obj2) + self.assertCommonResponseHeaders(headers) + version_id2 = headers['x-amz-version-id'] + + status, headers, body, resp_etag = \ + self._upload_part_copy(src_bucket, src_obj, bucket, + key, upload_id, 1, + src_version_id='null') + self.assertEqual(status, 200) + self.assertCommonResponseHeaders(headers) + self.assertTrue('content-type' in headers) + self.assertEqual(headers['content-type'], 'application/xml') + self.assertTrue('content-length' in headers) + self.assertEqual(headers['content-length'], str(len(body))) + self.assertTrue('etag' not in headers) + elem = fromstring(body, 'CopyPartResult') + + last_modifieds = [elem.find('LastModified').text] + self.assertTrue(last_modifieds[0] is not None) + + self.assertEqual(resp_etag, etags[0]) + + status, headers, body, resp_etag = \ + self._upload_part_copy(src_bucket, src_obj2, bucket, + key, upload_id, 2, + src_version_id=version_id2) + self.assertEqual(status, 200) + self.assertCommonResponseHeaders(headers) + self.assertTrue('content-type' in headers) + self.assertEqual(headers['content-type'], 'application/xml') + self.assertTrue('content-length' in headers) + self.assertEqual(headers['content-length'], str(len(body))) + self.assertTrue('etag' not in headers) + elem = fromstring(body, 'CopyPartResult') + + last_modifieds.append(elem.find('LastModified').text) + self.assertTrue(last_modifieds[1] is not None) + + self.assertEqual(resp_etag, etags[1]) + + # Check last-modified timestamp + key, upload_id = uploads[0] + query = 'uploadId=%s' % upload_id + status, headers, body = \ + self.conn.make_request('GET', bucket, key, query=query) + + elem = fromstring(body, 'ListPartsResult') + + # FIXME: COPY result drops milli/microseconds but GET doesn't + last_modified_gets = [p.find('LastModified').text + for p in elem.iterfind('Part')] + self.assertEqual( + [lm.rsplit('.', 1)[0] for lm in last_modified_gets], + [lm.rsplit('.', 1)[0] for lm in last_modifieds]) + + # There should be *exactly* two parts in the result + self.assertEqual(2, len(last_modified_gets)) + + # Abort Multipart Upload + key, upload_id = uploads[0] + query = 'uploadId=%s' % upload_id + status, headers, body = \ + self.conn.make_request('DELETE', bucket, key, query=query) + + # sanity checks + self.assertEqual(status, 204) + self.assertCommonResponseHeaders(headers) + self.assertTrue('content-type' in headers) + self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8') + self.assertTrue('content-length' in headers) + self.assertEqual(headers['content-length'], '0') + class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload): @classmethod @@ -892,6 +1022,11 @@ class TestS3ApiMultiUploadSigV4(TestS3ApiMultiUpload): def test_object_multi_upload_part_copy_range(self): if StrictVersion(boto.__version__) < StrictVersion('3.0'): + # boto 2 doesn't sort headers properly; see + # https://github.com/boto/boto/pull/3032 + # or https://github.com/boto/boto/pull/3176 + # or https://github.com/boto/boto/pull/3751 + # or https://github.com/boto/boto/pull/3824 self.skipTest('This stuff got the issue of boto<=2.x') def test_delete_bucket_multi_upload_object_exisiting(self): diff --git a/test/functional/s3api/test_object.py b/test/functional/s3api/test_object.py index ac2eef3505..5b518eaa8c 100644 --- a/test/functional/s3api/test_object.py +++ b/test/functional/s3api/test_object.py @@ -650,7 +650,8 @@ class TestS3ApiObject(S3ApiBase): def test_get_object_range(self): obj = 'object' content = b'abcdefghij' - headers = {'x-amz-meta-test': 'swift'} + headers = {'x-amz-meta-test': 'swift', + 'content-type': 'application/octet-stream'} self.conn.make_request( 'PUT', self.bucket, obj, headers=headers, body=content) @@ -664,6 +665,7 @@ class TestS3ApiObject(S3ApiBase): self.assertTrue('x-amz-meta-test' in headers) self.assertEqual('swift', headers['x-amz-meta-test']) self.assertEqual(body, b'bcdef') + self.assertEqual('application/octet-stream', headers['content-type']) headers = {'Range': 'bytes=5-'} status, headers, body = \ diff --git a/test/functional/s3api/test_versioning.py b/test/functional/s3api/test_versioning.py new file mode 100644 index 0000000000..bbbb72a70b --- /dev/null +++ b/test/functional/s3api/test_versioning.py @@ -0,0 +1,166 @@ +# Copyright (c) 2017 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.common.middleware.s3api.etree import fromstring, tostring, \ + Element, SubElement +import test.functional as tf +from test.functional.s3api import S3ApiBase +from test.functional.s3api.utils import get_error_code + + +def setUpModule(): + tf.setup_package() + + +def tearDownModule(): + tf.teardown_package() + + +class TestS3ApiVersioning(S3ApiBase): + def setUp(self): + super(TestS3ApiVersioning, self).setUp() + if 'object_versioning' not in tf.cluster_info: + # Alternatively, maybe we should assert we get 501s... + raise tf.SkipTest('S3 versioning requires that Swift object ' + 'versioning be enabled') + status, headers, body = self.conn.make_request('PUT', 'bucket') + self.assertEqual(status, 200) + + def tearDown(self): + # TODO: is this necessary on AWS? or can you delete buckets while + # versioning is enabled? + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Suspended' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 200) + + status, headers, body = self.conn.make_request('DELETE', 'bucket') + self.assertEqual(status, 204) + super(TestS3ApiVersioning, self).tearDown() + + def test_versioning_put(self): + # Versioning not configured + status, headers, body = self.conn.make_request( + 'GET', 'bucket', query='versioning') + self.assertEqual(status, 200) + elem = fromstring(body) + self.assertEqual(elem.getchildren(), []) + + # Enable versioning + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 200) + + status, headers, body = self.conn.make_request( + 'GET', 'bucket', query='versioning') + self.assertEqual(status, 200) + elem = fromstring(body) + self.assertEqual(elem.find('./Status').text, 'Enabled') + + # Suspend versioning + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Suspended' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 200) + + status, headers, body = self.conn.make_request( + 'GET', 'bucket', query='versioning') + self.assertEqual(status, 200) + elem = fromstring(body) + self.assertEqual(elem.find('./Status').text, 'Suspended') + + # Resume versioning + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 200) + + status, headers, body = self.conn.make_request( + 'GET', 'bucket', query='versioning') + self.assertEqual(status, 200) + elem = fromstring(body) + self.assertEqual(elem.find('./Status').text, 'Enabled') + + def test_versioning_immediately_suspend(self): + # Versioning not configured + status, headers, body = self.conn.make_request( + 'GET', 'bucket', query='versioning') + self.assertEqual(status, 200) + elem = fromstring(body) + self.assertEqual(elem.getchildren(), []) + + # Suspend versioning + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Suspended' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 200) + + status, headers, body = self.conn.make_request( + 'GET', 'bucket', query='versioning') + self.assertEqual(status, 200) + elem = fromstring(body) + self.assertEqual(elem.find('./Status').text, 'Suspended') + + # Enable versioning + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 200) + + status, headers, body = self.conn.make_request( + 'GET', 'bucket', query='versioning') + self.assertEqual(status, 200) + elem = fromstring(body) + self.assertEqual(elem.find('./Status').text, 'Enabled') + + def test_versioning_put_error(self): + # Root tag is not VersioningConfiguration + elem = Element('foo') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 400) + self.assertEqual(get_error_code(body), 'MalformedXML') + + # Status is not "Enabled" or "Suspended" + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = '...' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 400) + self.assertEqual(get_error_code(body), 'MalformedXML') + + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = '' + xml = tostring(elem) + status, headers, body = self.conn.make_request( + 'PUT', 'bucket', body=xml, query='versioning') + self.assertEqual(status, 400) + self.assertEqual(get_error_code(body), 'MalformedXML') diff --git a/test/s3api/__init__.py b/test/s3api/__init__.py index 43e3fc5d2b..9c722721b4 100644 --- a/test/s3api/__init__.py +++ b/test/s3api/__init__.py @@ -16,8 +16,11 @@ import logging import os import unittest +import uuid +import time import boto3 +from botocore.exceptions import ClientError from six.moves import urllib from swift.common.utils import config_true_value @@ -80,11 +83,14 @@ def get_s3_client(user=1, signature_version='s3v4', addressing_style='path'): path -- produces URLs like ``http(s)://host.domain/bucket/key`` virtual -- produces URLs like ``http(s)://bucket.host.domain/key`` ''' - endpoint = get_opt_or_error('endpoint') - scheme = urllib.parse.urlsplit(endpoint).scheme - if scheme not in ('http', 'https'): - raise ConfigError('unexpected scheme in endpoint: %r; ' - 'expected http or https' % scheme) + endpoint = get_opt('endpoint', None) + if endpoint: + scheme = urllib.parse.urlsplit(endpoint).scheme + if scheme not in ('http', 'https'): + raise ConfigError('unexpected scheme in endpoint: %r; ' + 'expected http or https' % scheme) + else: + scheme = None region = get_opt('region', 'us-east-1') access_key = get_opt_or_error('access_key%d' % user) secret_key = get_opt_or_error('secret_key%d' % user) @@ -112,6 +118,9 @@ def get_s3_client(user=1, signature_version='s3v4', addressing_style='path'): ) +TEST_PREFIX = 's3api-test-' + + class BaseS3TestCase(unittest.TestCase): # Default to v4 signatures (as aws-cli does), but subclasses can override signature_version = 's3v4' @@ -121,15 +130,77 @@ class BaseS3TestCase(unittest.TestCase): return get_s3_client(user, cls.signature_version) @classmethod - def clear_bucket(cls, client, bucket): - for key in client.list_objects(Bucket=bucket).get('Contents', []): - client.delete_key(Bucket=bucket, Key=key['Name']) + def _remove_all_object_versions_from_bucket(cls, client, bucket_name): + resp = client.list_object_versions(Bucket=bucket_name) + objs_to_delete = (resp.get('Versions', []) + + resp.get('DeleteMarkers', [])) + while objs_to_delete: + multi_delete_body = { + 'Objects': [ + {'Key': obj['Key'], 'VersionId': obj['VersionId']} + for obj in objs_to_delete + ], + 'Quiet': False, + } + del_resp = client.delete_objects(Bucket=bucket_name, + Delete=multi_delete_body) + if any(del_resp.get('Errors', [])): + raise Exception('Unable to delete %r' % del_resp['Errors']) + if not resp['IsTruncated']: + break + key_marker = resp['NextKeyMarker'] + version_id_marker = resp['NextVersionIdMarker'] + resp = client.list_object_versions( + Bucket=bucket_name, KeyMarker=key_marker, + VersionIdMarker=version_id_marker) + objs_to_delete = (resp.get('Versions', []) + + resp.get('DeleteMarkers', [])) + + @classmethod + def clear_bucket(cls, client, bucket_name): + timeout = time.time() + 10 + backoff = 0.1 + cls._remove_all_object_versions_from_bucket(client, bucket_name) + try: + client.delete_bucket(Bucket=bucket_name) + except ClientError as e: + if 'BucketNotEmpty' not in str(e): + raise + # Something's gone sideways. Try harder + client.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration={'Status': 'Suspended'}) + while True: + cls._remove_all_object_versions_from_bucket( + client, bucket_name) + # also try some version-unaware operations... + for key in client.list_objects(Bucket=bucket_name).get( + 'Contents', []): + client.delete_object(Bucket=bucket_name, Key=key['Key']) + + # *then* try again + try: + client.delete_bucket(Bucket=bucket_name) + except ClientError as e: + if 'BucketNotEmpty' not in str(e): + raise + if time.time() > timeout: + raise Exception('Timeout clearing %r' % bucket_name) + time.sleep(backoff) + backoff *= 2 + else: + break + + def create_name(self, slug): + return '%s%s-%s' % (TEST_PREFIX, slug, uuid.uuid4().hex) @classmethod def clear_account(cls, client): for bucket in client.list_buckets()['Buckets']: + if not bucket['Name'].startswith(TEST_PREFIX): + # these tests run against real s3 accounts + continue cls.clear_bucket(client, bucket['Name']) - client.delete_bucket(Bucket=bucket['Name']) def tearDown(self): client = self.get_s3_client(1) diff --git a/test/s3api/test_service.py b/test/s3api/test_service.py index 2d7cd7b2e0..2ec4dc610a 100644 --- a/test/s3api/test_service.py +++ b/test/s3api/test_service.py @@ -14,7 +14,6 @@ # limitations under the License. import unittest -import uuid from test.s3api import BaseS3TestCase, ConfigError @@ -43,7 +42,7 @@ class TestGetServiceSigV4(BaseS3TestCase): def test_service_with_buckets(self): c = self.get_s3_client(1) - buckets = [str(uuid.uuid4()) for _ in range(5)] + buckets = [self.create_name('bucket%s' % i) for i in range(5)] for bucket in buckets: c.create_bucket(Bucket=bucket) @@ -65,7 +64,7 @@ class TestGetServiceSigV4(BaseS3TestCase): c2 = self.get_s3_client(2) except ConfigError as err: raise unittest.SkipTest(str(err)) - buckets2 = [str(uuid.uuid4()) for _ in range(2)] + buckets2 = [self.create_name('bucket%s' % i) for i in range(2)] for bucket in buckets2: c2.create_bucket(Bucket=bucket) self.assertEqual(sorted(buckets2), [ diff --git a/test/s3api/test_versioning.py b/test/s3api/test_versioning.py new file mode 100644 index 0000000000..7f2364dd5b --- /dev/null +++ b/test/s3api/test_versioning.py @@ -0,0 +1,758 @@ +# Copyright (c) 2019 SwiftStack, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import hashlib +from collections import defaultdict + +from botocore.exceptions import ClientError +import six + +from swift.common.header_key_dict import HeaderKeyDict +from test.s3api import BaseS3TestCase + + +def retry(f, timeout=10): + timelimit = time.time() + timeout + while True: + try: + f() + except (ClientError, AssertionError): + if time.time() > timelimit: + raise + continue + else: + break + + +class TestObjectVersioning(BaseS3TestCase): + + maxDiff = None + + def setUp(self): + self.client = self.get_s3_client(1) + self.bucket_name = self.create_name('versioning') + resp = self.client.create_bucket(Bucket=self.bucket_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + + def enable_versioning(): + resp = self.client.put_bucket_versioning( + Bucket=self.bucket_name, + VersioningConfiguration={'Status': 'Enabled'}) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + retry(enable_versioning) + + def tearDown(self): + resp = self.client.put_bucket_versioning( + Bucket=self.bucket_name, + VersioningConfiguration={'Status': 'Suspended'}) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.clear_bucket(self.client, self.bucket_name) + super(TestObjectVersioning, self).tearDown() + + def test_setup(self): + bucket_name = self.create_name('new-bucket') + resp = self.client.create_bucket(Bucket=bucket_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + expected_location = '/%s' % bucket_name + self.assertEqual(expected_location, resp['Location']) + headers = HeaderKeyDict(resp['ResponseMetadata']['HTTPHeaders']) + self.assertEqual('0', headers['content-length']) + self.assertEqual(expected_location, headers['location']) + + # get versioning + resp = self.client.get_bucket_versioning(Bucket=bucket_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertNotIn('Status', resp) + + # put versioning + versioning_config = { + 'Status': 'Enabled', + } + resp = self.client.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration=versioning_config) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + + # ... now it's enabled + def check_status(): + resp = self.client.get_bucket_versioning(Bucket=bucket_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + try: + self.assertEqual('Enabled', resp['Status']) + except KeyError: + self.fail('Status was not in %r' % resp) + retry(check_status) + + # send over some bogus junk + versioning_config['Status'] = 'Disabled' + with self.assertRaises(ClientError) as ctx: + self.client.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration=versioning_config) + expected_err = 'An error occurred (MalformedXML) when calling the ' \ + 'PutBucketVersioning operation: The XML you provided was ' \ + 'not well-formed or did not validate against our published schema' + self.assertEqual(expected_err, str(ctx.exception)) + + # disable it + versioning_config['Status'] = 'Suspended' + resp = self.client.put_bucket_versioning( + Bucket=bucket_name, + VersioningConfiguration=versioning_config) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + + # ... now it's disabled again + def check_status(): + resp = self.client.get_bucket_versioning(Bucket=bucket_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('Suspended', resp['Status']) + retry(check_status) + + def test_upload_fileobj_versioned(self): + obj_data = self.create_name('some-data').encode('ascii') + obj_etag = hashlib.md5(obj_data).hexdigest() + obj_name = self.create_name('versioned-obj') + self.client.upload_fileobj(six.BytesIO(obj_data), + self.bucket_name, obj_name) + + # object is in the listing + resp = self.client.list_objects_v2(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + for obj in objs: + obj.pop('LastModified') + self.assertEqual([{ + 'ETag': '"%s"' % obj_etag, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # object version listing + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + obj.pop('VersionId') + self.assertEqual([{ + 'ETag': '"%s"' % obj_etag, + 'IsLatest': True, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # overwrite the object + new_obj_data = self.create_name('some-new-data').encode('ascii') + new_obj_etag = hashlib.md5(new_obj_data).hexdigest() + self.client.upload_fileobj(six.BytesIO(new_obj_data), + self.bucket_name, obj_name) + + # new object is in the listing + resp = self.client.list_objects_v2(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + for obj in objs: + obj.pop('LastModified') + self.assertEqual([{ + 'ETag': '"%s"' % new_obj_etag, + 'Key': obj_name, + 'Size': len(new_obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # both object versions in the versions listing + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + obj.pop('VersionId') + self.assertEqual([{ + 'ETag': '"%s"' % new_obj_etag, + 'IsLatest': True, + 'Key': obj_name, + 'Size': len(new_obj_data), + 'StorageClass': 'STANDARD', + }, { + 'ETag': '"%s"' % obj_etag, + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + def test_delete_versioned_objects(self): + etags = [] + obj_name = self.create_name('versioned-obj') + for i in range(3): + obj_data = self.create_name('some-data-%s' % i).encode('ascii') + etags.insert(0, hashlib.md5(obj_data).hexdigest()) + self.client.upload_fileobj(six.BytesIO(obj_data), + self.bucket_name, obj_name) + + # only one object appears in the listing + resp = self.client.list_objects_v2(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + for obj in objs: + obj.pop('LastModified') + self.assertEqual([{ + 'ETag': '"%s"' % etags[0], + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # but everything is layed out in the object versions listing + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + versions = [] + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + versions.append(obj.pop('VersionId')) + self.assertEqual([{ + 'ETag': '"%s"' % etags[0], + 'IsLatest': True, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }, { + 'ETag': '"%s"' % etags[1], + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }, { + 'ETag': '"%s"' % etags[2], + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # we can delete a specific version + resp = self.client.delete_object(Bucket=self.bucket_name, + Key=obj_name, + VersionId=versions[1]) + + # and that just pulls it out of the versions listing + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + obj.pop('VersionId') + self.assertEqual([{ + 'ETag': '"%s"' % etags[0], + 'IsLatest': True, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }, { + 'ETag': '"%s"' % etags[2], + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # ... but the current listing is unaffected + resp = self.client.list_objects_v2(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + for obj in objs: + obj.pop('LastModified') + self.assertEqual([{ + 'ETag': '"%s"' % etags[0], + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # OTOH, if you delete specifically the latest version + # we can delete a specific version + resp = self.client.delete_object(Bucket=self.bucket_name, + Key=obj_name, + VersionId=versions[0]) + + # the versions listing has a new IsLatest + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + obj.pop('VersionId') + self.assertEqual([{ + 'ETag': '"%s"' % etags[2], + 'IsLatest': True, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # and the stack pops + resp = self.client.list_objects_v2(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + for obj in objs: + obj.pop('LastModified') + self.assertEqual([{ + 'ETag': '"%s"' % etags[2], + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + def test_delete_versioned_deletes(self): + etags = [] + obj_name = self.create_name('versioned-obj') + for i in range(3): + obj_data = self.create_name('some-data-%s' % i).encode('ascii') + etags.insert(0, hashlib.md5(obj_data).hexdigest()) + self.client.upload_fileobj(six.BytesIO(obj_data), + self.bucket_name, obj_name) + # and make a delete marker + self.client.delete_object(Bucket=self.bucket_name, Key=obj_name) + + # current listing is empty + resp = self.client.list_objects_v2(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + self.assertEqual([], objs) + + # but everything is in layed out in the versions listing + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + versions = [] + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + versions.append(obj.pop('VersionId')) + self.assertEqual([{ + 'ETag': '"%s"' % etag, + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + } for etag in etags], objs) + # ... plus the delete markers + delete_markers = resp.get('DeleteMarkers', []) + marker_versions = [] + for marker in delete_markers: + marker.pop('LastModified') + marker.pop('Owner') + marker_versions.append(marker.pop('VersionId')) + self.assertEqual([{ + 'Key': obj_name, + 'IsLatest': is_latest, + } for is_latest in (True, False, False)], delete_markers) + + # delete an old delete markers + resp = self.client.delete_object(Bucket=self.bucket_name, + Key=obj_name, + VersionId=marker_versions[2]) + + # since IsLatest is still marker we'll raise NoSuchKey + with self.assertRaises(ClientError) as caught: + resp = self.client.get_object(Bucket=self.bucket_name, + Key=obj_name) + expected_err = 'An error occurred (NoSuchKey) when calling the ' \ + 'GetObject operation: The specified key does not exist.' + self.assertEqual(expected_err, str(caught.exception)) + + # now delete the delete marker (IsLatest) + resp = self.client.delete_object(Bucket=self.bucket_name, + Key=obj_name, + VersionId=marker_versions[0]) + + # most recent version is now latest + resp = self.client.get_object(Bucket=self.bucket_name, + Key=obj_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('"%s"' % etags[0], resp['ETag']) + + # now delete the IsLatest object version + resp = self.client.delete_object(Bucket=self.bucket_name, + Key=obj_name, + VersionId=versions[0]) + + # and object is deleted again + with self.assertRaises(ClientError) as caught: + resp = self.client.get_object(Bucket=self.bucket_name, + Key=obj_name) + expected_err = 'An error occurred (NoSuchKey) when calling the ' \ + 'GetObject operation: The specified key does not exist.' + self.assertEqual(expected_err, str(caught.exception)) + + # delete marker IsLatest + resp = self.client.list_object_versions(Bucket=self.bucket_name) + delete_markers = resp.get('DeleteMarkers', []) + for marker in delete_markers: + marker.pop('LastModified') + marker.pop('Owner') + self.assertEqual([{ + 'Key': obj_name, + 'IsLatest': True, + 'VersionId': marker_versions[1], + }], delete_markers) + + def test_multipart_upload(self): + obj_name = self.create_name('versioned-obj') + obj_data = b'data' + + mu = self.client.create_multipart_upload( + Bucket=self.bucket_name, + Key=obj_name) + part_md5 = self.client.upload_part( + Bucket=self.bucket_name, + Key=obj_name, + UploadId=mu['UploadId'], + PartNumber=1, + Body=obj_data)['ETag'] + complete_response = self.client.complete_multipart_upload( + Bucket=self.bucket_name, + Key=obj_name, + UploadId=mu['UploadId'], + MultipartUpload={'Parts': [ + {'PartNumber': 1, 'ETag': part_md5}, + ]}) + obj_etag = complete_response['ETag'] + + delete_response = self.client.delete_object( + Bucket=self.bucket_name, + Key=obj_name) + marker_version_id = delete_response['VersionId'] + + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + versions = [] + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + versions.append(obj.pop('VersionId')) + self.assertEqual([{ + 'ETag': obj_etag, + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + markers = resp.get('DeleteMarkers', []) + for marker in markers: + marker.pop('LastModified') + marker.pop('Owner') + self.assertEqual([{ + 'IsLatest': True, + 'Key': obj_name, + 'VersionId': marker_version_id, + }], markers) + + # Can still get the old version + resp = self.client.get_object( + Bucket=self.bucket_name, + Key=obj_name, + VersionId=versions[0]) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(obj_etag, resp['ETag']) + + delete_response = self.client.delete_object( + Bucket=self.bucket_name, + Key=obj_name, + VersionId=versions[0]) + + resp = self.client.list_object_versions(Bucket=self.bucket_name) + self.assertEqual([], resp.get('Versions', [])) + + markers = resp.get('DeleteMarkers', []) + for marker in markers: + marker.pop('LastModified') + marker.pop('Owner') + self.assertEqual([{ + 'IsLatest': True, + 'Key': obj_name, + 'VersionId': marker_version_id, + }], markers) + + def test_get_versioned_object(self): + etags = [] + obj_name = self.create_name('versioned-obj') + for i in range(3): + obj_data = self.create_name('some-data-%s' % i).encode('ascii') + # TODO: pull etag from response instead + etags.insert(0, hashlib.md5(obj_data).hexdigest()) + self.client.upload_fileobj( + six.BytesIO(obj_data), self.bucket_name, obj_name) + + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + versions = [] + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + versions.append(obj.pop('VersionId')) + self.assertEqual([{ + 'ETag': '"%s"' % etags[0], + 'IsLatest': True, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }, { + 'ETag': '"%s"' % etags[1], + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }, { + 'ETag': '"%s"' % etags[2], + 'IsLatest': False, + 'Key': obj_name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }], objs) + + # un-versioned get_object returns IsLatest + resp = self.client.get_object(Bucket=self.bucket_name, Key=obj_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('"%s"' % etags[0], resp['ETag']) + + # but you can get any object by version + for i, version in enumerate(versions): + resp = self.client.get_object( + Bucket=self.bucket_name, Key=obj_name, VersionId=version) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('"%s"' % etags[i], resp['ETag']) + + # and head_object works about the same + resp = self.client.head_object(Bucket=self.bucket_name, Key=obj_name) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('"%s"' % etags[0], resp['ETag']) + self.assertEqual(versions[0], resp['VersionId']) + for version, etag in zip(versions, etags): + resp = self.client.head_object( + Bucket=self.bucket_name, Key=obj_name, VersionId=version) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual(version, resp['VersionId']) + self.assertEqual('"%s"' % etag, resp['ETag']) + + def test_get_versioned_object_invalid_params(self): + with self.assertRaises(ClientError) as ctx: + self.client.list_object_versions(Bucket=self.bucket_name, + KeyMarker='', + VersionIdMarker='bogus') + expected_err = 'An error occurred (InvalidArgument) when calling ' \ + 'the ListObjectVersions operation: Invalid version id specified' + self.assertEqual(expected_err, str(ctx.exception)) + + with self.assertRaises(ClientError) as ctx: + self.client.list_object_versions( + Bucket=self.bucket_name, + VersionIdMarker='a' * 32) + expected_err = 'An error occurred (InvalidArgument) when calling ' \ + 'the ListObjectVersions operation: A version-id marker cannot ' \ + 'be specified without a key marker.' + self.assertEqual(expected_err, str(ctx.exception)) + + def test_get_versioned_object_key_marker(self): + obj00_name = self.create_name('00-versioned-obj') + obj01_name = self.create_name('01-versioned-obj') + names = [obj00_name] * 3 + [obj01_name] * 3 + latest = [True, False, False, True, False, False] + etags = [] + for i in range(3): + obj_data = self.create_name('some-data-%s' % i).encode('ascii') + etags.insert(0, '"%s"' % hashlib.md5(obj_data).hexdigest()) + self.client.upload_fileobj( + six.BytesIO(obj_data), self.bucket_name, obj01_name) + for i in range(3): + obj_data = self.create_name('some-data-%s' % i).encode('ascii') + etags.insert(0, '"%s"' % hashlib.md5(obj_data).hexdigest()) + self.client.upload_fileobj( + six.BytesIO(obj_data), self.bucket_name, obj00_name) + resp = self.client.list_object_versions(Bucket=self.bucket_name) + versions = [] + objs = [] + for o in resp.get('Versions', []): + versions.append(o['VersionId']) + objs.append({ + 'Key': o['Key'], + 'VersionId': o['VersionId'], + 'IsLatest': o['IsLatest'], + 'ETag': o['ETag'], + }) + expected = [{ + 'Key': name, + 'VersionId': version, + 'IsLatest': is_latest, + 'ETag': etag, + } for name, etag, version, is_latest in zip( + names, etags, versions, latest)] + self.assertEqual(expected, objs) + + # on s3 this makes expected[0]['IsLatest'] magicaly change to False? + # resp = self.client.list_object_versions(Bucket=self.bucket_name, + # KeyMarker='', + # VersionIdMarker=versions[0]) + # objs = [{ + # 'Key': o['Key'], + # 'VersionId': o['VersionId'], + # 'IsLatest': o['IsLatest'], + # 'ETag': o['ETag'], + # } for o in resp.get('Versions', [])] + # self.assertEqual(expected, objs) + + # KeyMarker skips past that key + resp = self.client.list_object_versions(Bucket=self.bucket_name, + KeyMarker=obj00_name) + objs = [{ + 'Key': o['Key'], + 'VersionId': o['VersionId'], + 'IsLatest': o['IsLatest'], + 'ETag': o['ETag'], + } for o in resp.get('Versions', [])] + self.assertEqual(expected[3:], objs) + + # KeyMarker with VersionIdMarker skips past that version + resp = self.client.list_object_versions(Bucket=self.bucket_name, + KeyMarker=obj00_name, + VersionIdMarker=versions[0]) + objs = [{ + 'Key': o['Key'], + 'VersionId': o['VersionId'], + 'IsLatest': o['IsLatest'], + 'ETag': o['ETag'], + } for o in resp.get('Versions', [])] + self.assertEqual(expected[1:], objs) + + # KeyMarker with bogus version skips past that key + resp = self.client.list_object_versions( + Bucket=self.bucket_name, + KeyMarker=obj00_name, + VersionIdMarker=versions[4]) + objs = [{ + 'Key': o['Key'], + 'VersionId': o['VersionId'], + 'IsLatest': o['IsLatest'], + 'ETag': o['ETag'], + } for o in resp.get('Versions', [])] + self.assertEqual(expected[3:], objs) + + def test_list_objects(self): + etags = defaultdict(list) + for i in range(3): + obj_name = self.create_name('versioned-obj') + for i in range(3): + obj_data = self.create_name('some-data-%s' % i).encode('ascii') + etags[obj_name].insert(0, hashlib.md5(obj_data).hexdigest()) + self.client.upload_fileobj( + six.BytesIO(obj_data), self.bucket_name, obj_name) + + # both unversioned list_objects responses are similar + expected = [] + for name, obj_etags in sorted(etags.items()): + expected.append({ + 'ETag': '"%s"' % obj_etags[0], + 'Key': name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }) + resp = self.client.list_objects(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + for obj in objs: + obj.pop('LastModified') + # one difference seems to be the Owner key + self.assertEqual({'DisplayName', 'ID'}, + set(obj.pop('Owner').keys())) + self.assertEqual(expected, objs) + resp = self.client.list_objects_v2(Bucket=self.bucket_name) + objs = resp.get('Contents', []) + for obj in objs: + obj.pop('LastModified') + self.assertEqual(expected, objs) + + # versioned listings has something for everyone + expected = [] + for name, obj_etags in sorted(etags.items()): + is_latest = True + for etag in obj_etags: + expected.append({ + 'ETag': '"%s"' % etag, + 'IsLatest': is_latest, + 'Key': name, + 'Size': len(obj_data), + 'StorageClass': 'STANDARD', + }) + is_latest = False + + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + versions = [] + for obj in objs: + obj.pop('LastModified') + obj.pop('Owner') + versions.append(obj.pop('VersionId')) + self.assertEqual(expected, objs) + + def test_copy_object(self): + etags = [] + obj_name = self.create_name('versioned-obj') + for i in range(3): + obj_data = self.create_name('some-data-%s' % i).encode('ascii') + etags.insert(0, hashlib.md5(obj_data).hexdigest()) + self.client.upload_fileobj( + six.BytesIO(obj_data), self.bucket_name, obj_name) + + resp = self.client.list_object_versions(Bucket=self.bucket_name) + objs = resp.get('Versions', []) + versions = [] + for obj in objs: + versions.append(obj.pop('VersionId')) + + # CopySource can just be Bucket/Key string + first_target = self.create_name('target-obj1') + copy_resp = self.client.copy_object( + Bucket=self.bucket_name, Key=first_target, + CopySource='%s/%s' % (self.bucket_name, obj_name)) + self.assertEqual(versions[0], copy_resp['CopySourceVersionId']) + + # and you'll just get the most recent version + resp = self.client.head_object(Bucket=self.bucket_name, + Key=first_target) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('"%s"' % etags[0], resp['ETag']) + + # or you can be more explicit + explicit_target = self.create_name('target-%s' % versions[0]) + copy_source = {'Bucket': self.bucket_name, 'Key': obj_name, + 'VersionId': versions[0]} + copy_resp = self.client.copy_object( + Bucket=self.bucket_name, Key=explicit_target, + CopySource=copy_source) + self.assertEqual(versions[0], copy_resp['CopySourceVersionId']) + # and you still get the same thing + resp = self.client.head_object(Bucket=self.bucket_name, + Key=explicit_target) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('"%s"' % etags[0], resp['ETag']) + + # but you can also copy from a specific version + version_target = self.create_name('target-%s' % versions[2]) + copy_source['VersionId'] = versions[2] + copy_resp = self.client.copy_object( + Bucket=self.bucket_name, Key=version_target, + CopySource=copy_source) + self.assertEqual(versions[2], copy_resp['CopySourceVersionId']) + resp = self.client.head_object(Bucket=self.bucket_name, + Key=version_target) + self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) + self.assertEqual('"%s"' % etags[2], resp['ETag']) diff --git a/test/unit/common/middleware/s3api/__init__.py b/test/unit/common/middleware/s3api/__init__.py index 2a180b2da8..e16a40c9ce 100644 --- a/test/unit/common/middleware/s3api/__init__.py +++ b/test/unit/common/middleware/s3api/__init__.py @@ -16,6 +16,7 @@ import unittest from datetime import datetime import email +import mock import time from swift.common import swob @@ -24,8 +25,8 @@ from swift.common.middleware.s3api.s3api import filter_factory from swift.common.middleware.s3api.etree import fromstring from swift.common.middleware.s3api.utils import Config -from test.unit.common.middleware.s3api.helpers import FakeSwift from test.unit import debug_logger +from test.unit.common.middleware.s3api.helpers import FakeSwift class FakeApp(object): @@ -80,7 +81,7 @@ class S3ApiTestCase(unittest.TestCase): self.app = FakeApp() self.swift = self.app.swift self.s3api = filter_factory({}, **self.conf)(self.app) - self.s3api.logger = debug_logger() + self.s3api.logger = self.swift.logger = debug_logger() self.swift.register('HEAD', '/v1/AUTH_test', swob.HTTPOk, {}, None) @@ -100,6 +101,19 @@ class S3ApiTestCase(unittest.TestCase): self.swift.register('DELETE', '/v1/AUTH_test/bucket/object', swob.HTTPNoContent, {}, None) + self.mock_get_swift_info_result = {'object_versioning': {}} + for s3api_path in ( + 'controllers.obj', + 'controllers.bucket', + 'controllers.multi_delete', + 'controllers.versioning', + ): + patcher = mock.patch( + 'swift.common.middleware.s3api.%s.get_swift_info' % s3api_path, + return_value=self.mock_get_swift_info_result) + patcher.start() + self.addCleanup(patcher.stop) + def _get_error_code(self, body): elem = fromstring(body, 'Error') return elem.find('./Code').text diff --git a/test/unit/common/middleware/s3api/helpers.py b/test/unit/common/middleware/s3api/helpers.py index 525dc35111..5f816f5b2b 100644 --- a/test/unit/common/middleware/s3api/helpers.py +++ b/test/unit/common/middleware/s3api/helpers.py @@ -15,25 +15,21 @@ # This stuff can't live in test/unit/__init__.py due to its swob dependency. -from copy import deepcopy -from hashlib import md5 from swift.common import swob from swift.common.utils import split_path from swift.common.request_helpers import is_sys_meta +from test.unit.common.middleware.helpers import FakeSwift as BaseFakeSwift -class FakeSwift(object): + +class FakeSwift(BaseFakeSwift): """ A good-enough fake Swift proxy server to use in testing middleware. """ + ALLOWED_METHODS = BaseFakeSwift.ALLOWED_METHODS + ['TEST'] def __init__(self, s3_acl=False): - self._calls = [] - self.req_method_paths = [] - self.swift_sources = [] - self.uploaded = {} - # mapping of (method, path) --> (response class, headers, body) - self._responses = {} + super(FakeSwift, self).__init__() self.s3_acl = s3_acl self.remote_user = 'authorized' @@ -69,88 +65,7 @@ class FakeSwift(object): def __call__(self, env, start_response): if self.s3_acl: self._fake_auth_middleware(env) - - req = swob.Request(env) - method = env['REQUEST_METHOD'] - path = env['PATH_INFO'] - _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, - rest_with_last=True) - if env.get('QUERY_STRING'): - path += '?' + env['QUERY_STRING'] - - if 'swift.authorize' in env: - resp = env['swift.authorize'](req) - if resp: - return resp(env, start_response) - - headers = req.headers - self._calls.append((method, path, headers)) - self.swift_sources.append(env.get('swift.source')) - - try: - resp_class, raw_headers, body = self._responses[(method, path)] - headers = swob.HeaderKeyDict(raw_headers) - except KeyError: - # FIXME: suppress print state error for python3 compatibility. - # pylint: disable-msg=E1601 - if (env.get('QUERY_STRING') - and (method, env['PATH_INFO']) in self._responses): - resp_class, raw_headers, body = self._responses[ - (method, env['PATH_INFO'])] - headers = swob.HeaderKeyDict(raw_headers) - elif method == 'HEAD' and ('GET', path) in self._responses: - resp_class, raw_headers, _ = self._responses[('GET', path)] - body = None - headers = swob.HeaderKeyDict(raw_headers) - elif method == 'GET' and obj and path in self.uploaded: - resp_class = swob.HTTPOk - headers, body = self.uploaded[path] - else: - print("Didn't find %r in allowed responses" % - ((method, path),)) - raise - - # simulate object PUT - if method == 'PUT' and obj: - input = env['wsgi.input'].read() - etag = md5(input).hexdigest() - if env.get('HTTP_ETAG', etag) != etag: - raise Exception('Client sent a bad ETag! Got %r, but ' - 'md5(body) = %r' % (env['HTTP_ETAG'], etag)) - headers.setdefault('Etag', etag) - headers.setdefault('Content-Length', len(input)) - - # keep it for subsequent GET requests later - self.uploaded[path] = (deepcopy(headers), input) - if "CONTENT_TYPE" in env: - self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] - - # range requests ought to work, but copies are special - support_range_and_conditional = not ( - method == 'PUT' and - 'X-Copy-From' in req.headers and - 'Range' in req.headers) - if isinstance(body, list): - app_iter = body - body = None - else: - app_iter = None - resp = resp_class( - req=req, headers=headers, body=body, app_iter=app_iter, - conditional_response=support_range_and_conditional) - return resp(env, start_response) - - @property - def calls(self): - return [(method, path) for method, path, headers in self._calls] - - @property - def calls_with_headers(self): - return self._calls - - @property - def call_count(self): - return len(self._calls) + return super(FakeSwift, self).__call__(env, start_response) def register(self, method, path, response_class, headers, body): # assuming the path format like /v1/account/container/object @@ -167,7 +82,8 @@ class FakeSwift(object): if body is not None and not isinstance(body, (bytes, list)): body = body.encode('utf8') - self._responses[(method, path)] = (response_class, headers, body) + return super(FakeSwift, self).register( + method, path, response_class, headers, body) def register_unconditionally(self, method, path, response_class, headers, body): diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index f021368740..8353d58d42 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -21,6 +21,8 @@ import six from six.moves.urllib.parse import quote from swift.common import swob +from swift.common.middleware.versioned_writes.object_versioning import \ + DELETE_MARKER_CONTENT_TYPE from swift.common.swob import Request from swift.common.utils import json @@ -30,6 +32,7 @@ from swift.common.middleware.s3api.subresource import Owner, encode_acl, \ ACLPublicRead from swift.common.middleware.s3api.s3request import MAX_32BIT_INT +from test.unit.common.middleware.helpers import normalize_path from test.unit.common.middleware.s3api import S3ApiTestCase from test.unit.common.middleware.s3api.test_s3_acl import s3acl from test.unit.common.middleware.s3api.helpers import UnreadableInput @@ -41,25 +44,43 @@ PFS_ETAG = '"pfsv2/AUTH_test/01234567/89abcdef-32"' class TestS3ApiBucket(S3ApiTestCase): def setup_objects(self): self.objects = (('lily', '2011-01-05T02:19:14.275290', '0', '3909'), - ('rose', '2011-01-05T02:19:14.275290', 0, 303), - ('viola', '2011-01-05T02:19:14.275290', '0', 3909), (u'lily-\u062a', '2011-01-05T02:19:14.275290', 0, 390), ('mu', '2011-01-05T02:19:14.275290', 'md5-of-the-manifest; s3_etag=0', '3909'), ('pfs-obj', '2011-01-05T02:19:14.275290', PFS_ETAG, '3909'), + ('rose', '2011-01-05T02:19:14.275290', 0, 303), ('slo', '2011-01-05T02:19:14.275290', 'md5-of-the-manifest', '3909'), + ('viola', '2011-01-05T02:19:14.275290', '0', 3909), ('with space', '2011-01-05T02:19:14.275290', 0, 390), ('with%20space', '2011-01-05T02:19:14.275290', 0, 390)) - objects = [ + self.objects_list = [ {'name': item[0], 'last_modified': str(item[1]), + 'content_type': 'application/octet-stream', 'hash': str(item[2]), 'bytes': str(item[3])} for item in self.objects] - objects[6]['slo_etag'] = '"0"' - object_list = json.dumps(objects) + self.objects_list[5]['slo_etag'] = '"0"' + self.versioned_objects = [{ + 'name': 'rose', + 'version_id': '2', + 'hash': '0', + 'bytes': '0', + 'last_modified': '2010-03-01T17:09:51.510928', + 'content_type': DELETE_MARKER_CONTENT_TYPE, + 'is_latest': False, + }, { + 'name': 'rose', + 'version_id': '1', + 'hash': '1234', + 'bytes': '6', + 'last_modified': '2010-03-01T17:09:50.510928', + 'content_type': 'application/octet-stream', + 'is_latest': False, + }] + listing_body = json.dumps(self.objects_list) self.prefixes = ['rose', 'viola', 'lily'] object_list_subdir = [{"subdir": p} for p in self.prefixes] @@ -79,7 +100,7 @@ class TestS3ApiBucket(S3ApiTestCase): json.dumps([])) self.swift.register( 'GET', '/v1/AUTH_test/bucket+segments?format=json&marker=', - swob.HTTPOk, {'Content-Type': 'application/json'}, object_list) + swob.HTTPOk, {'Content-Type': 'application/json'}, listing_body) self.swift.register( 'HEAD', '/v1/AUTH_test/junk', swob.HTTPNoContent, {}, None) self.swift.register( @@ -89,20 +110,14 @@ class TestS3ApiBucket(S3ApiTestCase): {}, None) self.swift.register( 'GET', '/v1/AUTH_test/junk', swob.HTTPOk, - {'Content-Type': 'application/json'}, object_list) - self.swift.register( - 'GET', - '/v1/AUTH_test/junk?delimiter=a&format=json&limit=3&marker=viola', - swob.HTTPOk, - {'Content-Type': 'application/json; charset=utf-8'}, - json.dumps(objects[2:])) + {'Content-Type': 'application/json'}, listing_body) self.swift.register( 'GET', '/v1/AUTH_test/junk-subdir', swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'}, json.dumps(object_list_subdir)) self.swift.register( 'GET', - '/v1/AUTH_test/subdirs?delimiter=/&format=json&limit=3', + '/v1/AUTH_test/subdirs?delimiter=/&limit=3', swob.HTTPOk, {}, json.dumps([ {'subdir': 'nothing/'}, {'subdir': u'but-\u062a/'}, @@ -189,11 +204,13 @@ class TestS3ApiBucket(S3ApiTestCase): items.append((o.find('./Key').text, o.find('./ETag').text)) self.assertEqual('2011-01-05T02:19:14.275Z', o.find('./LastModified').text) - self.assertEqual(items, [ + expected = [ (i[0].encode('utf-8') if six.PY2 else i[0], PFS_ETAG if i[0] == 'pfs-obj' else '"0-N"' if i[0] == 'slo' else '"0"') - for i in self.objects]) + for i in self.objects + ] + self.assertEqual(items, expected) def test_bucket_GET_url_encoded(self): bucket_name = 'junk' @@ -483,15 +500,16 @@ class TestS3ApiBucket(S3ApiTestCase): def test_bucket_GET_with_delimiter_max_keys(self): bucket_name = 'junk' - req = Request.blank('/%s?delimiter=a&max-keys=2' % bucket_name, + req = Request.blank('/%s?delimiter=a&max-keys=4' % bucket_name, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('./NextMarker').text, 'rose') - self.assertEqual(elem.find('./MaxKeys').text, '2') + self.assertEqual(elem.find('./NextMarker').text, + self.objects_list[3]['name']) + self.assertEqual(elem.find('./MaxKeys').text, '4') self.assertEqual(elem.find('./IsTruncated').text, 'true') def test_bucket_GET_v2_with_delimiter_max_keys(self): @@ -567,6 +585,14 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertIsNotNone(o.find('./Owner')) def test_bucket_GET_with_versions_versioning_not_configured(self): + for obj in self.objects: + self.swift.register( + 'HEAD', '/v1/AUTH_test/junk/%s' % quote(obj[0].encode('utf8')), + swob.HTTPOk, {}, None) + # self.swift.register('HEAD', '/v1/AUTH_test/junk/viola', + # swob.HTTPOk, {}, None) + + self._add_versions_request(versioned_objects=[]) req = Request.blank('/junk?versions', environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', @@ -585,11 +611,10 @@ class TestS3ApiBucket(S3ApiTestCase): versions = elem.findall('./Version') objects = list(self.objects) if six.PY2: - self.assertEqual([v.find('./Key').text for v in versions], - [v[0].encode('utf-8') for v in objects]) + expected = [v[0].encode('utf-8') for v in objects] else: - self.assertEqual([v.find('./Key').text for v in versions], - [v[0] for v in objects]) + expected = [v[0] for v in objects] + self.assertEqual([v.find('./Key').text for v in versions], expected) self.assertEqual([v.find('./IsLatest').text for v in versions], ['true' for v in objects]) self.assertEqual([v.find('./VersionId').text for v in versions], @@ -612,6 +637,446 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual([v.find('./StorageClass').text for v in versions], ['STANDARD' for v in objects]) + def _add_versions_request(self, orig_objects=None, versioned_objects=None, + bucket='junk'): + if orig_objects is None: + orig_objects = self.objects_list + if versioned_objects is None: + versioned_objects = self.versioned_objects + all_versions = versioned_objects + [ + dict(i, version_id='null', is_latest=True) + for i in orig_objects] + all_versions.sort(key=lambda o: ( + o['name'], '' if o['version_id'] == 'null' else o['version_id'])) + self.swift.register( + 'GET', '/v1/AUTH_test/%s' % bucket, swob.HTTPOk, + {'Content-Type': 'application/json'}, json.dumps(all_versions)) + + def _assert_delete_markers(self, elem): + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(len(delete_markers), 1) + self.assertEqual(delete_markers[0].find('./IsLatest').text, 'false') + self.assertEqual(delete_markers[0].find('./VersionId').text, '2') + self.assertEqual(delete_markers[0].find('./Key').text, 'rose') + + def test_bucket_GET_with_versions(self): + self._add_versions_request() + req = Request.blank('/junk?versions', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./Name').text, 'junk') + self._assert_delete_markers(elem) + versions = elem.findall('./Version') + self.assertEqual(len(versions), len(self.objects) + 1) + + expected = [] + for o in self.objects_list: + name = o['name'] + if six.PY2: + name = name.encode('utf8') + expected.append((name, 'true', 'null')) + if name == 'rose': + expected.append((name, 'false', '1')) + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in versions + ] + self.assertEqual(expected, discovered) + + def test_bucket_GET_with_versions_with_max_keys(self): + self._add_versions_request() + req = Request.blank('/junk?versions&max-keys=7', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./MaxKeys').text, '7') + self.assertEqual(elem.find('./IsTruncated').text, 'true') + self._assert_delete_markers(elem) + versions = elem.findall('./Version') + self.assertEqual(len(versions), 6) + + expected = [] + for o in self.objects_list[:5]: + name = o['name'] + if six.PY2: + name = name.encode('utf8') + expected.append((name, 'true', 'null')) + if name == 'rose': + expected.append((name, 'false', '1')) + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in versions + ] + self.assertEqual(expected, discovered) + + def test_bucket_GET_with_versions_with_max_keys_and_key_marker(self): + self._add_versions_request(orig_objects=self.objects_list[4:]) + req = Request.blank('/junk?versions&max-keys=3&key-marker=ros', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./MaxKeys').text, '3') + self.assertEqual(elem.find('./IsTruncated').text, 'true') + self._assert_delete_markers(elem) + versions = elem.findall('./Version') + self.assertEqual(len(versions), 2) + + expected = [ + ('rose', 'true', 'null'), + ('rose', 'false', '1'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in versions + ] + self.assertEqual(expected, discovered) + + def test_bucket_GET_versions_with_key_marker_and_version_id_marker(self): + container_listing = [{ + "bytes": 8192, + "content_type": "binary/octet-stream", + "hash": "221994040b14294bdf7fbc128e66633c", + "last_modified": "2019-08-16T19:39:53.152780", + "name": "subdir/foo", + }] + versions_listing = [{ + 'bytes': 0, + 'content_type': DELETE_MARKER_CONTENT_TYPE, + 'hash': '0', + "last_modified": "2019-08-19T19:05:33.565940", + 'name': 'subdir/bar', + "version_id": "1565241533.55320", + 'is_latest': True, + }, { + "bytes": 8192, + "content_type": "binary/octet-stream", + "hash": "221994040b14294bdf7fbc128e66633c", + "last_modified": "2019-08-16T19:39:53.508510", + "name": "subdir/bar", + "version_id": "1564984393.68962", + 'is_latest': False, + }, { + "bytes": 8192, + "content_type": "binary/octet-stream", + "hash": "221994040b14294bdf7fbc128e66633c", + "last_modified": "2019-08-16T19:39:42.673260", + "name": "subdir/foo", + "version_id": "1565984382.67326", + 'is_latest': False, + }] + self._add_versions_request(container_listing, versions_listing, + bucket='mybucket') + req = Request.blank( + '/mybucket?versions&key-marker=subdir/bar&' + 'version-id-marker=1566589611.065522', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(['subdir/bar'], [ + o.find('Key').text for o in delete_markers]) + expected = [ + ('subdir/bar', 'false', '1564984393.68962'), + ('subdir/foo', 'true', 'null'), + ('subdir/foo', 'false', '1565984382.67326'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + self._add_versions_request(container_listing, versions_listing[1:], + bucket='mybucket') + req = Request.blank( + '/mybucket?versions&key-marker=subdir/bar&' + 'version-id-marker=1565241533.55320', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(0, len(delete_markers)) + expected = [ + ('subdir/bar', 'false', '1564984393.68962'), + ('subdir/foo', 'true', 'null'), + ('subdir/foo', 'false', '1565984382.67326'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + self._add_versions_request([], versions_listing[-1:], + bucket='mybucket') + req = Request.blank( + '/mybucket?versions&key-marker=subdir/foo&' + 'version-id-marker=null', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(0, len(delete_markers)) + expected = [ + ('subdir/foo', 'false', '1565984382.67326'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + def test_bucket_GET_versions_with_version_id_marker(self): + self._add_versions_request() + req = Request.blank( + '/junk?versions', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + + # sanity + elem = fromstring(body, 'ListVersionsResult') + expected = [('rose', 'false', '2')] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./DeleteMarker') + ] + self.assertEqual(expected, discovered) + expected = [ + ('lily', 'true', 'null'), + (b'lily-\xd8\xaa', 'true', 'null'), + ('mu', 'true', 'null'), + ('pfs-obj', 'true', 'null'), + ('rose', 'true', 'null'), + ('rose', 'false', '1'), + ('slo', 'true', 'null'), + ('viola', 'true', 'null'), + ('with space', 'true', 'null'), + ('with%20space', 'true', 'null'), + ] + if not six.PY2: + item = list(expected[1]) + item[0] = item[0].decode('utf8') + expected[1] = tuple(item) + + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + self._add_versions_request(self.objects_list[5:]) + req = Request.blank( + '/junk?versions&key-marker=rose&version-id-marker=null', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(len(delete_markers), 1) + + expected = [ + ('rose', 'false', '1'), + ('slo', 'true', 'null'), + ('viola', 'true', 'null'), + ('with space', 'true', 'null'), + ('with%20space', 'true', 'null'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + # N.B. versions are sorted most recent to oldest + self._add_versions_request(self.objects_list[5:], + self.versioned_objects[1:]) + req = Request.blank( + '/junk?versions&key-marker=rose&version-id-marker=2', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(len(delete_markers), 0) + + expected = [ + ('rose', 'false', '1'), + ('slo', 'true', 'null'), + ('viola', 'true', 'null'), + ('with space', 'true', 'null'), + ('with%20space', 'true', 'null'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + self._add_versions_request(self.objects_list[5:], + self.versioned_objects[2:]) + req = Request.blank( + '/junk?versions&key-marker=rose&version-id-marker=1', + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(len(delete_markers), 0) + + expected = [ + ('slo', 'true', 'null'), + ('viola', 'true', 'null'), + ('with space', 'true', 'null'), + ('with%20space', 'true', 'null'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + def test_bucket_GET_versions_non_existent_version_id_marker(self): + self._add_versions_request(orig_objects=self.objects_list[5:]) + req = Request.blank( + '/junk?versions&key-marker=rose&' + 'version-id-marker=null', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200', body) + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./Name').text, 'junk') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(len(delete_markers), 1) + + expected = [ + ('rose', 'false', '1'), + ('slo', 'true', 'null'), + ('viola', 'true', 'null'), + ('with space', 'true', 'null'), + ('with%20space', 'true', 'null'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + self.assertEqual(self.swift.calls, [ + ('GET', normalize_path('/v1/AUTH_test/junk?' + 'limit=1001&marker=rose&version_marker=null&versions=')), + ]) + + def test_bucket_GET_versions_prefix(self): + container_listing = [{ + "bytes": 8192, + "content_type": "binary/octet-stream", + "hash": "221994040b14294bdf7fbc128e66633c", + "last_modified": "2019-08-16T19:39:53.152780", + "name": "subdir/foo", + }] + versions_listing = [{ + "bytes": 8192, + "content_type": "binary/octet-stream", + "hash": "221994040b14294bdf7fbc128e66633c", + "last_modified": "2019-08-16T19:39:53.508510", + "name": "subdir/bar", + "version_id": "1565984393.68962", + "is_latest": True, + }, { + 'bytes': 0, + 'content_type': DELETE_MARKER_CONTENT_TYPE, + 'hash': '0', + "last_modified": "2019-08-19T19:05:33.565940", + 'name': 'subdir/bar', + 'version_id': '1566241533.55320', + 'is_latest': False, + }, { + "bytes": 8192, + "content_type": "binary/octet-stream", + "hash": "221994040b14294bdf7fbc128e66633c", + "last_modified": "2019-08-16T19:39:42.673260", + "name": "subdir/foo", + "version_id": "1565984382.67326", + 'is_latest': False, + }] + self._add_versions_request(container_listing, versions_listing) + req = Request.blank( + '/junk?versions&prefix=subdir/', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./Name').text, 'junk') + delete_markers = elem.findall('./DeleteMarker') + self.assertEqual(len(delete_markers), 1) + + expected = [ + ('subdir/bar', 'true', '1565984393.68962'), + ('subdir/foo', 'true', 'null'), + ('subdir/foo', 'false', '1565984382.67326'), + ] + discovered = [ + tuple(e.find('./%s' % key).text for key in ( + 'Key', 'IsLatest', 'VersionId')) + for e in elem.findall('./Version') + ] + self.assertEqual(expected, discovered) + + self.assertEqual(self.swift.calls, [ + ('GET', normalize_path('/v1/AUTH_test/junk' + '?limit=1001&prefix=subdir/&versions=')), + ]) + @s3acl def test_bucket_PUT_error(self): code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated, @@ -847,6 +1312,24 @@ class TestS3ApiBucket(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '204') + @s3acl + def test_bucket_DELETE_with_empty_versioning(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket+versioning', + swob.HTTPNoContent, {}, None) + self.swift.register('DELETE', '/v1/AUTH_test/bucket+versioning', + swob.HTTPNoContent, {}, None) + # overwrite default HEAD to return x-container-object-count + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, + {'X-Container-Object-Count': 0}, None) + + req = Request.blank('/bucket', + environ={'REQUEST_METHOD': 'DELETE'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + @s3acl def test_bucket_DELETE_error_while_segment_bucket_delete(self): # An error occurred while deleting segment objects diff --git a/test/unit/common/middleware/s3api/test_multi_delete.py b/test/unit/common/middleware/s3api/test_multi_delete.py index b5295721c8..3421e4aec0 100644 --- a/test/unit/common/middleware/s3api/test_multi_delete.py +++ b/test/unit/common/middleware/s3api/test_multi_delete.py @@ -23,6 +23,7 @@ import mock from swift.common import swob from swift.common.swob import Request +from test.unit import make_timestamp_iter from test.unit.common.middleware.s3api import S3ApiTestCase from test.unit.common.middleware.s3api.helpers import UnreadableInput from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \ @@ -38,6 +39,7 @@ class TestS3ApiMultiDelete(S3ApiTestCase): swob.HTTPOk, {}, None) self.swift.register('HEAD', '/v1/AUTH_test/bucket/Key2', swob.HTTPNotFound, {}, None) + self.ts = make_timestamp_iter() @s3acl def test_object_multi_DELETE_to_object(self): @@ -98,10 +100,11 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertEqual(len(elem.findall('Deleted')), 3) self.assertEqual(self.swift.calls, [ ('HEAD', '/v1/AUTH_test/bucket'), - ('HEAD', '/v1/AUTH_test/bucket/Key1'), + ('HEAD', '/v1/AUTH_test/bucket/Key1?symlink=get'), ('DELETE', '/v1/AUTH_test/bucket/Key1'), - ('HEAD', '/v1/AUTH_test/bucket/Key2'), - ('HEAD', '/v1/AUTH_test/bucket/Key3'), + ('HEAD', '/v1/AUTH_test/bucket/Key2?symlink=get'), + ('DELETE', '/v1/AUTH_test/bucket/Key2'), + ('HEAD', '/v1/AUTH_test/bucket/Key3?symlink=get'), ('DELETE', '/v1/AUTH_test/bucket/Key3?multipart-manifest=delete'), ]) @@ -161,11 +164,12 @@ class TestS3ApiMultiDelete(S3ApiTestCase): ) self.assertEqual(self.swift.calls, [ ('HEAD', '/v1/AUTH_test/bucket'), - ('HEAD', '/v1/AUTH_test/bucket/Key1'), + ('HEAD', '/v1/AUTH_test/bucket/Key1?symlink=get'), ('DELETE', '/v1/AUTH_test/bucket/Key1'), - ('HEAD', '/v1/AUTH_test/bucket/Key2'), - ('HEAD', '/v1/AUTH_test/bucket/Key3'), - ('HEAD', '/v1/AUTH_test/bucket/Key4'), + ('HEAD', '/v1/AUTH_test/bucket/Key2?symlink=get'), + ('DELETE', '/v1/AUTH_test/bucket/Key2'), + ('HEAD', '/v1/AUTH_test/bucket/Key3?symlink=get'), + ('HEAD', '/v1/AUTH_test/bucket/Key4?symlink=get'), ('DELETE', '/v1/AUTH_test/bucket/Key4?multipart-manifest=delete'), ]) @@ -221,18 +225,42 @@ class TestS3ApiMultiDelete(S3ApiTestCase): self.assertEqual(self._get_error_code(body), 'UserKeyMustBeSpecified') @s3acl - def test_object_multi_DELETE_versioned(self): - self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1', - swob.HTTPNoContent, {}, None) - self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key2', - swob.HTTPNotFound, {}, None) + def test_object_multi_DELETE_versioned_enabled(self): + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, { + 'X-Container-Sysmeta-Versions-Enabled': 'True', + }, None) + t1 = next(self.ts) + key1 = '/v1/AUTH_test/bucket/Key1' \ + '?symlink=get&version-id=%s' % t1.normal + self.swift.register('HEAD', key1, swob.HTTPOk, {}, None) + self.swift.register('DELETE', key1, swob.HTTPNoContent, {}, None) + t2 = next(self.ts) + key2 = '/v1/AUTH_test/bucket/Key2' \ + '?symlink=get&version-id=%s' % t2.normal + # this 404 could just mean it's a delete marker + self.swift.register('HEAD', key2, swob.HTTPNotFound, {}, None) + self.swift.register('DELETE', key2, swob.HTTPNoContent, {}, None) + key3 = '/v1/AUTH_test/bucket/Key3' + self.swift.register('HEAD', key3 + '?symlink=get', + swob.HTTPOk, {}, None) + self.swift.register('DELETE', key3, swob.HTTPNoContent, {}, None) + key4 = '/v1/AUTH_test/bucket/Key4?symlink=get&version-id=null' + self.swift.register('HEAD', key4, swob.HTTPOk, {}, None) + self.swift.register('DELETE', key4, swob.HTTPNoContent, {}, None) elem = Element('Delete') - SubElement(elem, 'Quiet').text = 'true' - for key in ['Key1', 'Key2']: + items = ( + ('Key1', t1.normal), + ('Key2', t2.normal), + ('Key3', None), + ('Key4', 'null'), + ) + for key, version in items: obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = key - SubElement(obj, 'VersionId').text = 'not-supported' + if version: + SubElement(obj, 'VersionId').text = version body = tostring(elem, use_s3ns=False) content_md5 = base64.b64encode(md5(body).digest()).strip() @@ -243,7 +271,80 @@ class TestS3ApiMultiDelete(S3ApiTestCase): 'Content-MD5': content_md5}, body=body) status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NotImplemented') + self.assertEqual(status.split()[0], '200') + + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', key1), + ('DELETE', key1), + ('HEAD', key2), + ('DELETE', key2), + ('HEAD', key3 + '?symlink=get'), + ('DELETE', key3), + ('HEAD', key4), + ('DELETE', key4), + ]) + + elem = fromstring(body) + self.assertEqual({'Key1', 'Key2', 'Key3', 'Key4'}, set( + e.findtext('Key') for e in elem.findall('Deleted'))) + + @s3acl + def test_object_multi_DELETE_versioned_suspended(self): + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent, {}, None) + t1 = next(self.ts) + key1 = '/v1/AUTH_test/bucket/Key1' + \ + '?symlink=get&version-id=%s' % t1.normal + self.swift.register('HEAD', key1, swob.HTTPOk, {}, None) + self.swift.register('DELETE', key1, swob.HTTPNoContent, {}, None) + t2 = next(self.ts) + key2 = '/v1/AUTH_test/bucket/Key2' + \ + '?symlink=get&version-id=%s' % t2.normal + self.swift.register('HEAD', key2, swob.HTTPNotFound, {}, None) + self.swift.register('DELETE', key2, swob.HTTPNotFound, {}, None) + key3 = '/v1/AUTH_test/bucket/Key3' + self.swift.register('HEAD', key3, swob.HTTPOk, {}, None) + self.swift.register('DELETE', key3, swob.HTTPNoContent, {}, None) + + elem = Element('Delete') + items = ( + ('Key1', t1), + ('Key2', t2), + ('Key3', None), + ) + for key, ts in items: + obj = SubElement(elem, 'Object') + SubElement(obj, 'Key').text = key + if ts: + SubElement(obj, 'VersionId').text = ts.normal + body = tostring(elem, use_s3ns=False) + content_md5 = base64.b64encode(md5(body).digest()).strip() + + req = Request.blank('/bucket?delete', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-MD5': content_md5}, + body=body) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + elem = fromstring(body) + self.assertEqual(len(elem.findall('Deleted')), 3) + + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test/bucket'), + ('HEAD', '/v1/AUTH_test/bucket/Key1' + '?symlink=get&version-id=%s' % t1.normal), + ('DELETE', '/v1/AUTH_test/bucket/Key1' + '?symlink=get&version-id=%s' % t1.normal), + ('HEAD', '/v1/AUTH_test/bucket/Key2' + '?symlink=get&version-id=%s' % t2.normal), + ('DELETE', '/v1/AUTH_test/bucket/Key2' + '?symlink=get&version-id=%s' % t2.normal), + ('HEAD', '/v1/AUTH_test/bucket/Key3?symlink=get'), + ('DELETE', '/v1/AUTH_test/bucket/Key3'), + ]) @s3acl def test_object_multi_DELETE_with_invalid_md5(self): @@ -282,9 +383,12 @@ class TestS3ApiMultiDelete(S3ApiTestCase): def test_object_multi_DELETE_lots_of_keys(self): elem = Element('Delete') for i in range(self.conf.max_multi_delete_objects): + status = swob.HTTPOk if i % 2 else swob.HTTPNotFound name = 'x' * 1000 + str(i) self.swift.register('HEAD', '/v1/AUTH_test/bucket/%s' % name, - swob.HTTPNotFound, {}, None) + status, {}, None) + self.swift.register('DELETE', '/v1/AUTH_test/bucket/%s' % name, + swob.HTTPNoContent, {}, None) obj = SubElement(elem, 'Object') SubElement(obj, 'Key').text = name body = tostring(elem, use_s3ns=False) diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py index ef570ddb06..bc6292a2b9 100644 --- a/test/unit/common/middleware/s3api/test_multi_upload.py +++ b/test/unit/common/middleware/s3api/test_multi_upload.py @@ -20,7 +20,7 @@ from mock import patch import os import time import unittest -from six.moves.urllib.parse import quote +from six.moves.urllib.parse import quote, quote_plus from swift.common import swob from swift.common.swob import Request @@ -346,7 +346,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): query[key] = arg self.assertEqual(query['format'], 'json') self.assertEqual(query['limit'], '1001') - self.assertEqual(query['marker'], 'object/Y') + self.assertEqual(query['marker'], quote_plus('object/Y')) @s3acl def test_bucket_multipart_uploads_GET_with_key_marker(self): @@ -380,7 +380,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): query[key] = arg self.assertEqual(query['format'], 'json') self.assertEqual(query['limit'], '1001') - self.assertEqual(query['marker'], quote('object/~')) + self.assertEqual(query['marker'], quote_plus('object/~')) @s3acl def test_bucket_multipart_uploads_GET_with_prefix(self): @@ -550,7 +550,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase): query[key] = arg self.assertEqual(query['format'], 'json') self.assertEqual(query['limit'], '1001') - self.assertEqual(query['prefix'], 'dir/') + self.assertEqual(query['prefix'], quote_plus('dir/')) self.assertTrue(query.get('delimiter') is None) @patch('swift.common.middleware.s3api.controllers.' diff --git a/test/unit/common/middleware/s3api/test_obj.py b/test/unit/common/middleware/s3api/test_obj.py index aab5a7af01..2db4c4f237 100644 --- a/test/unit/common/middleware/s3api/test_obj.py +++ b/test/unit/common/middleware/s3api/test_obj.py @@ -22,6 +22,7 @@ from os.path import join import time from mock import patch import six +import json from swift.common import swob from swift.common.swob import Request @@ -32,6 +33,8 @@ from swift.common.middleware.s3api.subresource import ACL, User, encode_acl, \ Owner, Grant from swift.common.middleware.s3api.etree import fromstring from swift.common.middleware.s3api.utils import mktime, S3Timestamp +from swift.common.middleware.versioned_writes.object_versioning import \ + DELETE_MARKER_CONTENT_TYPE class TestS3ApiObj(S3ApiTestCase): @@ -57,6 +60,9 @@ class TestS3ApiObj(S3ApiTestCase): self.swift.register('GET', '/v1/AUTH_test/bucket/object', swob.HTTPOk, self.response_headers, self.object_body) + self.swift.register('GET', '/v1/AUTH_test/bucket/object?symlink=get', + swob.HTTPOk, self.response_headers, + self.object_body) self.swift.register('PUT', '/v1/AUTH_test/bucket/object', swob.HTTPCreated, {'etag': self.etag, @@ -370,6 +376,82 @@ class TestS3ApiObj(S3ApiTestCase): self.assertTrue('content-encoding' in headers) self.assertEqual(headers['content-encoding'], 'gzip') + @s3acl + def test_object_GET_version_id_not_implemented(self): + # GET version that is not null + req = Request.blank('/bucket/object?versionId=2', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + + with patch('swift.common.middleware.s3api.controllers.obj.' + 'get_swift_info', return_value={}): + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '501', body) + + # GET current version + req = Request.blank('/bucket/object?versionId=null', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with patch('swift.common.middleware.s3api.controllers.obj.' + 'get_swift_info', return_value={}): + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200', body) + self.assertEqual(body, self.object_body) + + @s3acl + def test_object_GET_version_id(self): + # GET current version + req = Request.blank('/bucket/object?versionId=null', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200', body) + self.assertEqual(body, self.object_body) + + # GET current version that is not null + req = Request.blank('/bucket/object?versionId=2', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200', body) + self.assertEqual(body, self.object_body) + + # GET version in archive + headers = self.response_headers.copy() + headers['Content-Length'] = 6 + account = 'test:tester' + grants = [Grant(User(account), 'FULL_CONTROL')] + headers.update( + encode_acl('object', ACL(Owner(account, account), grants))) + self.swift.register( + 'HEAD', '/v1/AUTH_test/bucket/object?version-id=1', swob.HTTPOk, + headers, None) + self.swift.register( + 'GET', '/v1/AUTH_test/bucket/object?version-id=1', swob.HTTPOk, + headers, 'hello1') + req = Request.blank('/bucket/object?versionId=1', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200', body) + self.assertEqual(body, b'hello1') + + # Version not found + self.swift.register( + 'GET', '/v1/AUTH_test/bucket/object?version-id=A', + swob.HTTPNotFound, {}, None) + req = Request.blank('/bucket/object?versionId=A', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '404') + @s3acl def test_object_PUT_error(self): code = self._test_method_error('PUT', '/bucket/object', @@ -393,9 +475,10 @@ class TestS3ApiObj(S3ApiTestCase): code = self._test_method_error('PUT', '/bucket/object', swob.HTTPLengthRequired) self.assertEqual(code, 'MissingContentLength') + # Swift can 412 if the versions container is missing code = self._test_method_error('PUT', '/bucket/object', swob.HTTPPreconditionFailed) - self.assertEqual(code, 'InternalError') + self.assertEqual(code, 'PreconditionFailed') code = self._test_method_error('PUT', '/bucket/object', swob.HTTPServiceUnavailable) self.assertEqual(code, 'ServiceUnavailable') @@ -432,11 +515,6 @@ class TestS3ApiObj(S3ApiTestCase): swob.HTTPCreated, {'X-Amz-Copy-Source': '/bucket/src_obj?bar=baz&versionId=foo'}) self.assertEqual(code, 'InvalidArgument') - code = self._test_method_error( - 'PUT', '/bucket/object', - swob.HTTPCreated, - {'X-Amz-Copy-Source': '/bucket/src_obj?versionId=foo'}) - self.assertEqual(code, 'NotImplemented') code = self._test_method_error( 'PUT', '/bucket/object', swob.HTTPCreated, @@ -447,6 +525,35 @@ class TestS3ApiObj(S3ApiTestCase): swob.HTTPRequestTimeout) self.assertEqual(code, 'RequestTimeout') + def test_object_PUT_with_version(self): + self.swift.register('GET', + '/v1/AUTH_test/bucket/src_obj?version-id=foo', + swob.HTTPOk, self.response_headers, + self.object_body) + self.swift.register('PUT', '/v1/AUTH_test/bucket/object', + swob.HTTPCreated, { + 'etag': self.etag, + 'last-modified': self.last_modified, + }, None) + + req = Request.blank('/bucket/object', method='PUT', body='', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'X-Amz-Copy-Source': '/bucket/src_obj?versionId=foo', + }) + status, headers, body = self.call_s3api(req) + + self.assertEqual('200 OK', status) + elem = fromstring(body, 'CopyObjectResult') + self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) + + self.assertEqual(self.swift.calls, [ + ('HEAD', '/v1/AUTH_test/bucket/src_obj?version-id=foo'), + ('PUT', '/v1/AUTH_test/bucket/object?version-id=foo'), + ]) + _, _, headers = self.swift.calls_with_headers[-1] + self.assertEqual(headers['x-copy-from'], '/bucket/src_obj') + @s3acl def test_object_PUT(self): etag = self.response_headers['etag'] @@ -643,7 +750,7 @@ class TestS3ApiObj(S3ApiTestCase): @s3acl def test_object_PUT_copy(self): - def do_test(src_path=None): + def do_test(src_path): date_header = self.get_date_header() timestamp = mktime(date_header) allowed_last_modified = [S3Timestamp(timestamp).s3xmlformat] @@ -990,6 +1097,257 @@ class TestS3ApiObj(S3ApiTestCase): _, path = self.swift.calls[-1] self.assertEqual(path.count('?'), 0) + def test_object_DELETE_old_version_id(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, self.response_headers, None) + resp_headers = {'X-Object-Current-Version-Id': '1574360804.34906'} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293', + swob.HTTPNoContent, resp_headers, None) + req = Request.blank('/bucket/object?versionId=1574358170.12293', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293') + ], self.swift.calls) + + def test_object_DELETE_current_version_id(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, self.response_headers, None) + resp_headers = {'X-Object-Current-Version-Id': 'null'} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293', + swob.HTTPNoContent, resp_headers, None) + old_versions = [{ + 'name': 'object', + 'version_id': '1574341899.21751', + 'content_type': 'application/found', + }, { + 'name': 'object', + 'version_id': '1574333192.15190', + 'content_type': 'application/older', + }] + self.swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPOk, {}, + json.dumps(old_versions)) + req = Request.blank('/bucket/object?versionId=1574358170.12293', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('GET', '/v1/AUTH_test/bucket' + '?prefix=object&versions=True'), + ('PUT', '/v1/AUTH_test/bucket/object' + '?version-id=1574341899.21751'), + ], self.swift.calls) + + def test_object_DELETE_version_id_not_implemented(self): + req = Request.blank('/bucket/object?versionId=1574358170.12293', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + + with patch('swift.common.middleware.s3api.controllers.obj.' + 'get_swift_info', return_value={}): + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '501', body) + + def test_object_DELETE_current_version_id_is_delete_marker(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, self.response_headers, None) + resp_headers = {'X-Object-Current-Version-Id': 'null'} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293', + swob.HTTPNoContent, resp_headers, None) + old_versions = [{ + 'name': 'object', + 'version_id': '1574341899.21751', + 'content_type': 'application/x-deleted;swift_versions_deleted=1', + }] + self.swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPOk, {}, + json.dumps(old_versions)) + req = Request.blank('/bucket/object?versionId=1574358170.12293', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('GET', '/v1/AUTH_test/bucket' + '?prefix=object&versions=True'), + ], self.swift.calls) + + def test_object_DELETE_current_version_id_is_missing(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, self.response_headers, None) + resp_headers = {'X-Object-Current-Version-Id': 'null'} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293', + swob.HTTPNoContent, resp_headers, None) + old_versions = [{ + 'name': 'object', + 'version_id': '1574341899.21751', + 'content_type': 'application/missing', + }, { + 'name': 'object', + 'version_id': '1574333192.15190', + 'content_type': 'application/found', + }] + self.swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPOk, {}, + json.dumps(old_versions)) + self.swift.register('PUT', '/v1/AUTH_test/bucket/object' + '?version-id=1574341899.21751', + swob.HTTPPreconditionFailed, {}, None) + self.swift.register('PUT', '/v1/AUTH_test/bucket/object' + '?version-id=1574333192.15190', + swob.HTTPCreated, {}, None) + req = Request.blank('/bucket/object?versionId=1574358170.12293', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('GET', '/v1/AUTH_test/bucket' + '?prefix=object&versions=True'), + ('PUT', '/v1/AUTH_test/bucket/object' + '?version-id=1574341899.21751'), + ('PUT', '/v1/AUTH_test/bucket/object' + '?version-id=1574333192.15190'), + ], self.swift.calls) + + def test_object_DELETE_current_version_id_GET_error(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, self.response_headers, None) + resp_headers = {'X-Object-Current-Version-Id': 'null'} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293', + swob.HTTPNoContent, resp_headers, None) + self.swift.register('GET', '/v1/AUTH_test/bucket', + swob.HTTPServerError, {}, '') + req = Request.blank('/bucket/object?versionId=1574358170.12293', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '500') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('GET', '/v1/AUTH_test/bucket' + '?prefix=object&versions=True'), + ], self.swift.calls) + + def test_object_DELETE_current_version_id_PUT_error(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, self.response_headers, None) + resp_headers = {'X-Object-Current-Version-Id': 'null'} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293', + swob.HTTPNoContent, resp_headers, None) + old_versions = [{ + 'name': 'object', + 'version_id': '1574341899.21751', + 'content_type': 'application/foo', + }] + self.swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPOk, {}, + json.dumps(old_versions)) + self.swift.register('PUT', '/v1/AUTH_test/bucket/object' + '?version-id=1574341899.21751', + swob.HTTPServerError, {}, None) + req = Request.blank('/bucket/object?versionId=1574358170.12293', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '500') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574358170.12293'), + ('GET', '/v1/AUTH_test/bucket' + '?prefix=object&versions=True'), + ('PUT', '/v1/AUTH_test/bucket/object' + '?version-id=1574341899.21751'), + ], self.swift.calls) + + def test_object_DELETE_in_versioned_container_without_version(self): + resp_headers = { + 'X-Object-Version-Id': '1574360804.34906', + 'X-Backend-Content-Type': DELETE_MARKER_CONTENT_TYPE} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object', + swob.HTTPNoContent, resp_headers, None) + self.swift.register('HEAD', '/v1/AUTH_test/bucket', + swob.HTTPNoContent, { + 'X-Container-Sysmeta-Versions-Enabled': True}, + None) + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPNotFound, self.response_headers, None) + req = Request.blank('/bucket/object', method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('DELETE', '/v1/AUTH_test/bucket/object'), + ], self.swift.calls) + + self.assertEqual('1574360804.34906', headers.get('x-amz-version-id')) + self.assertEqual('true', headers.get('x-amz-delete-marker')) + + def test_object_DELETE_in_versioned_container_with_version_id(self): + resp_headers = { + 'X-Object-Version-Id': '1574701081.61553'} + self.swift.register('DELETE', '/v1/AUTH_test/bucket/object', + swob.HTTPNoContent, resp_headers, None) + self.swift.register('HEAD', '/v1/AUTH_test/bucket', + swob.HTTPNoContent, { + 'X-Container-Sysmeta-Versions-Enabled': True}, + None) + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPNotFound, self.response_headers, None) + req = Request.blank('/bucket/object?versionId=1574701081.61553', + method='DELETE', headers={ + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '204') + self.assertEqual([ + ('HEAD', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574701081.61553'), + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + ('DELETE', '/v1/AUTH_test/bucket/object' + '?symlink=get&version-id=1574701081.61553'), + ], self.swift.calls) + + self.assertEqual('1574701081.61553', headers.get('x-amz-version-id')) + @s3acl def test_object_DELETE_multipart(self): req = Request.blank('/bucket/object', @@ -999,7 +1357,7 @@ class TestS3ApiObj(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '204') - self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object'), + self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), self.swift.calls) self.assertEqual(('DELETE', '/v1/AUTH_test/bucket/object'), self.swift.calls[-1]) @@ -1017,10 +1375,11 @@ class TestS3ApiObj(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '204') - self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object'), - self.swift.calls) - self.assertNotIn(('DELETE', '/v1/AUTH_test/bucket/object'), - self.swift.calls) + self.assertEqual(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), + self.swift.calls[0]) + # the s3acl retests w/ a get_container_info HEAD @ self.swift.calls[1] + self.assertEqual(('DELETE', '/v1/AUTH_test/bucket/object'), + self.swift.calls[-1]) @s3acl def test_slo_object_DELETE(self): @@ -1039,7 +1398,7 @@ class TestS3ApiObj(S3ApiTestCase): self.assertEqual(status.split()[0], '204') self.assertEqual(body, b'') - self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object'), + self.assertIn(('HEAD', '/v1/AUTH_test/bucket/object?symlink=get'), self.swift.calls) self.assertIn(('DELETE', '/v1/AUTH_test/bucket/object' '?multipart-manifest=delete'), diff --git a/test/unit/common/middleware/s3api/test_versioning.py b/test/unit/common/middleware/s3api/test_versioning.py index edee6e8413..33acd82b3c 100644 --- a/test/unit/common/middleware/s3api/test_versioning.py +++ b/test/unit/common/middleware/s3api/test_versioning.py @@ -15,42 +15,180 @@ import unittest -from swift.common.swob import Request +from mock import patch + +from swift.common.swob import Request, HTTPNoContent +from swift.common.middleware.s3api.etree import fromstring, tostring, \ + Element, SubElement from test.unit.common.middleware.s3api import S3ApiTestCase -from swift.common.middleware.s3api.etree import fromstring class TestS3ApiVersioning(S3ApiTestCase): - def setUp(self): - super(TestS3ApiVersioning, self).setUp() - - def test_object_versioning_GET(self): - req = Request.blank('/bucket/object?versioning', + def _versioning_GET(self, path): + req = Request.blank('%s?versioning' % path, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header()}) status, headers, body = self.call_s3api(req) - self.assertEqual(status.split()[0], '200') - fromstring(body, 'VersioningConfiguration') + return status, headers, body - def test_object_versioning_PUT(self): - req = Request.blank('/bucket/object?versioning', + def _versioning_GET_not_configured(self, path): + self.swift.register('HEAD', '/v1/AUTH_test/bucket', + HTTPNoContent, {}, None) + + status, headers, body = self._versioning_GET(path) + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'VersioningConfiguration') + self.assertEqual(elem.getchildren(), []) + + def _versioning_GET_enabled(self, path): + self.swift.register('HEAD', '/v1/AUTH_test/bucket', HTTPNoContent, { + 'X-Container-Sysmeta-Versions-Enabled': 'True', + }, None) + + status, headers, body = self._versioning_GET(path) + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'VersioningConfiguration') + status = elem.find('./Status').text + self.assertEqual(status, 'Enabled') + + def _versioning_GET_suspended(self, path): + self.swift.register('HEAD', '/v1/AUTH_test/bucket', HTTPNoContent, { + 'X-Container-Sysmeta-Versions-Enabled': 'False', + }, None) + + status, headers, body = self._versioning_GET('/bucket/object') + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'VersioningConfiguration') + status = elem.find('./Status').text + self.assertEqual(status, 'Suspended') + + def _versioning_PUT_error(self, path): + # Root tag is not VersioningConfiguration + elem = Element('foo') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + + req = Request.blank('%s?versioning' % path, environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) + 'Date': self.get_date_header()}, + body=xml) status, headers, body = self.call_s3api(req) - self.assertEqual(self._get_error_code(body), 'NotImplemented') + self.assertEqual(status.split()[0], '400') - def test_bucket_versioning_GET(self): - req = Request.blank('/bucket?versioning', - environ={'REQUEST_METHOD': 'GET'}, + # Status is not "Enabled" or "Suspended" + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'enabled' + xml = tostring(elem) + + req = Request.blank('%s?versioning' % path, + environ={'REQUEST_METHOD': 'PUT'}, headers={'Authorization': 'AWS test:tester:hmac', - 'Date': self.get_date_header()}) + 'Date': self.get_date_header()}, + body=xml) status, headers, body = self.call_s3api(req) - fromstring(body, 'VersioningConfiguration') + self.assertEqual(status.split()[0], '400') + + def _versioning_PUT_enabled(self, path): + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + + self.swift.register('POST', '/v1/AUTH_test/bucket', HTTPNoContent, + {'X-Container-Sysmeta-Versions-Enabled': 'True'}, + None) + + req = Request.blank('%s?versioning' % path, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body=xml) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + + calls = self.swift.calls_with_headers + self.assertEqual(calls[-1][0], 'POST') + self.assertIn(('X-Versions-Enabled', 'true'), + list(calls[-1][2].items())) + + def _versioning_PUT_suspended(self, path): + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Suspended' + xml = tostring(elem) + + self.swift.register('POST', '/v1/AUTH_test/bucket', HTTPNoContent, + {'x-container-sysmeta-versions-enabled': 'False'}, + None) + + req = Request.blank('%s?versioning' % path, + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body=xml) + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '200') + + calls = self.swift.calls_with_headers + self.assertEqual(calls[-1][0], 'POST') + self.assertIn(('X-Versions-Enabled', 'false'), + list(calls[-1][2].items())) + + def test_object_versioning_GET_not_configured(self): + self._versioning_GET_not_configured('/bucket/object') + + def test_object_versioning_GET_enabled(self): + self._versioning_GET_enabled('/bucket/object') + + def test_object_versioning_GET_suspended(self): + self._versioning_GET_suspended('/bucket/object') + + def test_object_versioning_PUT_error(self): + self._versioning_PUT_error('/bucket/object') + + def test_object_versioning_PUT_enabled(self): + self._versioning_PUT_enabled('/bucket/object') + + def test_object_versioning_PUT_suspended(self): + self._versioning_PUT_suspended('/bucket/object') + + def test_bucket_versioning_GET_not_configured(self): + self._versioning_GET_not_configured('/bucket') + + def test_bucket_versioning_GET_enabled(self): + self._versioning_GET_enabled('/bucket') + + def test_bucket_versioning_GET_suspended(self): + self._versioning_GET_suspended('/bucket') + + def test_bucket_versioning_PUT_error(self): + self._versioning_PUT_error('/bucket') + + def test_object_versioning_PUT_not_implemented(self): + elem = Element('VersioningConfiguration') + SubElement(elem, 'Status').text = 'Enabled' + xml = tostring(elem) + + req = Request.blank('/bucket?versioning', + environ={'REQUEST_METHOD': 'PUT'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}, + body=xml) + + with patch('swift.common.middleware.s3api.controllers.versioning.' + 'get_swift_info', return_value={}): + status, headers, body = self.call_s3api(req) + self.assertEqual(status.split()[0], '501', body) + + def test_bucket_versioning_PUT_enabled(self): + self._versioning_PUT_enabled('/bucket') + + def test_bucket_versioning_PUT_suspended(self): + self._versioning_PUT_suspended('/bucket') + if __name__ == '__main__': unittest.main()