From a2fb335e4b43d7fb0b17daf000922be4c3c4db7a Mon Sep 17 00:00:00 2001 From: karen chan Date: Fri, 15 Jun 2018 13:23:40 -0700 Subject: [PATCH] s3api: Add basic support for ?versions bucket listings We still don't have support for toggling S3 bucket versioning, but we can at least support getting the latest versions of all objects. See https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETVersion.html for more information about the API. Note that the returned format is distinct from both "GET Bucket (List Objects) Version 1" and "GET Bucket (List Objects) Version 2" APIs. Change-Id: Ic57c273a3d5d7cdc34ca3a03e35e99b202a0bb01 --- .../middleware/s3api/controllers/bucket.py | 98 ++++++++++++------- .../common/middleware/s3api/test_bucket.py | 46 ++++++++- 2 files changed, 107 insertions(+), 37 deletions(-) diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index 137e9b4639..7b0a28a507 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -110,17 +110,22 @@ class BucketController(Controller): 'format': 'json', 'limit': max_keys + 1, } - if 'marker' in req.params: - query.update({'marker': req.params['marker']}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) if 'delimiter' in req.params: query.update({'delimiter': req.params['delimiter']}) - - # GET Bucket (List Objects) Version 2 parameters - is_v2 = int(req.params.get('list-type', '1')) == 2 fetch_owner = False - if is_v2: + if 'versions' in req.params: + listing_type = 'object-versions' + if 'key-marker' in req.params: + query.update({'marker': req.params['key-marker']}) + elif 'version-id-marker' in req.params: + err_msg = ('A version-id marker cannot be specified without ' + 'a key marker.') + raise InvalidArgument('version-id-marker', + req.params['version-id-marker'], err_msg) + elif int(req.params.get('list-type', '1')) == 2: + listing_type = 'version-2' if 'start-after' in req.params: query.update({'marker': req.params['start-after']}) # continuation-token overrides start-after @@ -129,44 +134,63 @@ class BucketController(Controller): query.update({'marker': decoded}) if 'fetch-owner' in req.params: fetch_owner = config_true_value(req.params['fetch-owner']) + else: + listing_type = 'version-1' + if 'marker' in req.params: + query.update({'marker': req.params['marker']}) resp = req.get_response(self.app, query=query) objects = json.loads(resp.body) - elem = Element('ListBucketResult') - SubElement(elem, 'Name').text = req.container_name - SubElement(elem, 'Prefix').text = req.params.get('prefix') - # in order to judge that truncated is valid, check whether # max_keys + 1 th element exists in swift. is_truncated = max_keys > 0 and len(objects) > max_keys objects = objects[:max_keys] - if not is_v2: - SubElement(elem, 'Marker').text = req.params.get('marker') - if is_truncated and 'delimiter' in req.params: - if 'name' in objects[-1]: - SubElement(elem, 'NextMarker').text = \ - objects[-1]['name'] - if 'subdir' in objects[-1]: - SubElement(elem, 'NextMarker').text = \ - objects[-1]['subdir'] - else: + if listing_type == 'object-versions': + elem = Element('ListVersionsResult') + SubElement(elem, 'Name').text = req.container_name + SubElement(elem, 'Prefix').text = req.params.get('prefix') + SubElement(elem, 'KeyMarker').text = req.params.get('key-marker') + SubElement(elem, 'VersionIdMarker').text = req.params.get( + 'version-id-marker') if is_truncated: if 'name' in objects[-1]: - SubElement(elem, 'NextContinuationToken').text = \ - b64encode(objects[-1]['name']) + SubElement(elem, 'NextKeyMarker').text = \ + objects[-1]['name'] if 'subdir' in objects[-1]: - SubElement(elem, 'NextContinuationToken').text = \ - b64encode(objects[-1]['subdir']) - if 'continuation-token' in req.params: - SubElement(elem, 'ContinuationToken').text = \ - req.params['continuation-token'] - if 'start-after' in req.params: - SubElement(elem, 'StartAfter').text = \ - req.params['start-after'] - SubElement(elem, 'KeyCount').text = str(len(objects)) + SubElement(elem, 'NextKeyMarker').text = \ + objects[-1]['subdir'] + SubElement(elem, 'NextVersionIdMarker').text = 'null' + else: + elem = Element('ListBucketResult') + SubElement(elem, 'Name').text = req.container_name + SubElement(elem, 'Prefix').text = req.params.get('prefix') + if listing_type == 'version-1': + SubElement(elem, 'Marker').text = req.params.get('marker') + if is_truncated and 'delimiter' in req.params: + if 'name' in objects[-1]: + SubElement(elem, 'NextMarker').text = \ + objects[-1]['name'] + if 'subdir' in objects[-1]: + SubElement(elem, 'NextMarker').text = \ + objects[-1]['subdir'] + elif listing_type == 'version-2': + if is_truncated: + if 'name' in objects[-1]: + SubElement(elem, 'NextContinuationToken').text = \ + b64encode(objects[-1]['name']) + if 'subdir' in objects[-1]: + SubElement(elem, 'NextContinuationToken').text = \ + b64encode(objects[-1]['subdir']) + if 'continuation-token' in req.params: + SubElement(elem, 'ContinuationToken').text = \ + req.params['continuation-token'] + if 'start-after' in req.params: + SubElement(elem, 'StartAfter').text = \ + req.params['start-after'] + SubElement(elem, 'KeyCount').text = str(len(objects)) SubElement(elem, 'MaxKeys').text = str(tag_max_keys) @@ -181,8 +205,14 @@ class BucketController(Controller): for o in objects: if 'subdir' not in o: - contents = SubElement(elem, 'Contents') - SubElement(contents, 'Key').text = o['name'] + if listing_type == 'object-versions': + contents = SubElement(elem, 'Version') + SubElement(contents, 'Key').text = o['name'] + SubElement(contents, 'VersionId').text = 'null' + SubElement(contents, 'IsLatest').text = 'true' + else: + contents = SubElement(elem, 'Contents') + SubElement(contents, 'Key').text = o['name'] SubElement(contents, 'LastModified').text = \ o['last_modified'][:-3] + 'Z' if 's3_etag' in o: @@ -192,7 +222,7 @@ class BucketController(Controller): etag = '"%s"' % o['hash'] SubElement(contents, 'ETag').text = etag SubElement(contents, 'Size').text = str(o['bytes']) - if fetch_owner or not is_v2: + if fetch_owner or listing_type != 'version-2': owner = SubElement(contents, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index fb6a1d0a60..bd7222d83c 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -33,9 +33,9 @@ from test.unit.common.middleware.s3api.helpers import UnreadableInput class TestS3ApiBucket(S3ApiTestCase): def setup_objects(self): - self.objects = (('rose', '2011-01-05T02:19:14.275290', 0, 303), + self.objects = (('lily', '2011-01-05T02:19:14.275290', '0', '3909'), + ('rose', '2011-01-05T02:19:14.275290', 0, 303), ('viola', '2011-01-05T02:19:14.275290', '0', 3909), - ('lily', '2011-01-05T02:19:14.275290', '0', '3909'), ('mu', '2011-01-05T02:19:14.275290', 'md5-of-the-manifest; s3_etag=0', '3909'), ('with space', '2011-01-05T02:19:14.275290', 0, 390), @@ -394,7 +394,7 @@ class TestS3ApiBucket(S3ApiTestCase): status, headers, body = self.call_s3api(req) self.assertEqual(status.split()[0], '200') elem = fromstring(body, 'ListBucketResult') - self.assertEqual(elem.find('./NextMarker').text, 'viola') + self.assertEqual(elem.find('./NextMarker').text, 'rose') self.assertEqual(elem.find('./MaxKeys').text, '2') self.assertEqual(elem.find('./IsTruncated').text, 'true') @@ -470,6 +470,46 @@ class TestS3ApiBucket(S3ApiTestCase): for o in objects: self.assertIsNotNone(o.find('./Owner')) + def test_bucket_GET_with_versions_versioning_not_configured(self): + req = Request.blank('/junk?versions', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + status, headers, body = self.call_s3api(req) + + self.assertEqual(status.split()[0], '200') + elem = fromstring(body, 'ListVersionsResult') + self.assertEqual(elem.find('./Name').text, 'junk') + self.assertIsNone(elem.find('./Prefix').text) + self.assertIsNone(elem.find('./KeyMarker').text) + self.assertIsNone(elem.find('./VersionIdMarker').text) + self.assertEqual(elem.find('./MaxKeys').text, '1000') + self.assertEqual(elem.find('./IsTruncated').text, 'false') + self.assertEqual(elem.findall('./DeleteMarker'), []) + versions = elem.findall('./Version') + objects = list(self.objects) + self.assertEqual([v.find('./Key').text for v in versions], + [v[0] for v in objects]) + self.assertEqual([v.find('./IsLatest').text for v in versions], + ['true' for v in objects]) + self.assertEqual([v.find('./VersionId').text for v in versions], + ['null' for v in objects]) + # Last modified in self.objects is 2011-01-05T02:19:14.275290 but + # the returned value is 2011-01-05T02:19:14.275Z + self.assertEqual([v.find('./LastModified').text for v in versions], + [v[1][:-3] + 'Z' for v in objects]) + self.assertEqual([v.find('./ETag').text for v in versions], + ['"0"' for v in objects]) + self.assertEqual([v.find('./Size').text for v in versions], + [str(v[3]) for v in objects]) + self.assertEqual([v.find('./Owner/ID').text for v in versions], + ['test:tester' for v in objects]) + self.assertEqual([v.find('./Owner/DisplayName').text + for v in versions], + ['test:tester' for v in objects]) + self.assertEqual([v.find('./StorageClass').text for v in versions], + ['STANDARD' for v in objects]) + @s3acl def test_bucket_PUT_error(self): code = self._test_method_error('PUT', '/bucket', swob.HTTPCreated,