x/swift3
Code Issues Proposed changes

Merge "Fix max-uploads, key-marker, prefix and upload-id-marker in queries of List Multipart Uploads"

Browse Source
This commit is contained in:
Jenkins
2015-01-13 01:41:13 +00:00
committed by Gerrit Code Review
parent e53b5d41db 578fb137de
commit 315604c31b
2 changed files with 272 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
swift3/controllers/multi_upload.py
View File

@@ -131,27 +131,62 @@ class UploadsController(Controller):
"""
Handles List Multipart Uploads
"""
def filter_max_uploads(o):
name = o.get('name', '')
return name.count('/') == 1
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
# TODO: add support for prefix, key-marker, upload-id-marker, and
# max-uploads queries.
# TODO: add support for delimiter query.
keymarker = req.params.get('key-marker', '')
uploadid = req.params.get('upload-id-marker', '')
maxuploads = DEFAULT_MAX_UPLOADS
if 'max-uploads' in req.params:
try:
maxuploads = int(req.params['max-uploads'])
if maxuploads < 0 or DEFAULT_MAX_UPLOADS < maxuploads:
err_msg = 'Argument max-uploads must be an integer ' \
'between 0 and %d' % DEFAULT_MAX_UPLOADS
raise InvalidArgument('max-uploads', maxuploads, err_msg)
except ValueError:
err_msg = 'Provided max-uploads not an integer or within ' \
'integer range'
raise InvalidArgument('max-uploads', req.params['max-uploads'],
err_msg)
query = {
'format': 'json',
'limit': maxuploads + 1,
}
if uploadid and keymarker:
query.update({'marker': '%s/%s' % (keymarker, uploadid)})
elif keymarker:
query.update({'marker': '%s/~' % (keymarker)})
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, container=container, query=query)
objects = loads(resp.body)
uploads = []
for o in objects:
obj, upid = split_path('/' + o['name'], 1, 2, True)
if '/' in upid:
# This is a part object.
continue
objects = filter(filter_max_uploads, objects)
if len(objects) > maxuploads:
objects = objects[:maxuploads]
truncated = True
else:
truncated = False
uploads = []
prefixes = []
for o in objects:
obj, upid = split_path('/' + o['name'], 1, 2)
uploads.append(
{'key': obj,
'upload_id': upid,
@@ -166,17 +201,17 @@ class UploadsController(Controller):
result_elem = Element('ListMultipartUploadsResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'KeyMarker').text = ''
SubElement(result_elem, 'UploadIdMarker').text = ''
SubElement(result_elem, 'KeyMarker').text = keymarker
SubElement(result_elem, 'UploadIdMarker').text = uploadid
SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker
SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker
SubElement(result_elem, 'MaxUploads').text = str(DEFAULT_MAX_UPLOADS)
if 'prefix' in req.params:
SubElement(result_elem, 'Prefix').text = req.params['prefix']
SubElement(result_elem, 'MaxUploads').text = str(maxuploads)
if encoding_type is not None:
SubElement(result_elem, 'EncodingType').text = encoding_type
SubElement(result_elem, 'IsTruncated').text = 'false'
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
# TODO: don't show uploads which are initiated before this bucket is
# created.
@@ -194,6 +229,10 @@ class UploadsController(Controller):
SubElement(upload_elem, 'Initiated').text = \
u['last_modified'][:-3] + 'Z'
for p in prefixes:
elem = SubElement(result_elem, 'CommonPrefixes')
SubElement(elem, 'Prefix').text = p
body = tostring(result_elem, encoding_type=encoding_type)
return HTTPOk(body=body, content_type='application/xml')

234
swift3/test/unit/test_multi_upload.py
View File

@@ -16,6 +16,7 @@
import unittest
import simplejson as json
from mock import patch
from urllib import quote
from swift.common import swob
from swift.common.swob import Request
@@ -36,6 +37,17 @@ xml = '<CompleteMultipartUpload>' \
'</Part>' \
'</CompleteMultipartUpload>'
multiparts_template = \
(('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1),
('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11),
('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21),
('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2),
('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12),
('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22),
('object/Z', '2014-05-07T19:47:56.592270', 'HASH', 3),
('object/Z/1', '2014-05-07T19:47:57.592270', 'HASH', 13),
('object/Z/2', '2014-05-07T19:47:58.592270', 'HASH', 23))
class TestSwift3MultiUpload(Swift3TestCase):
@@ -44,21 +56,19 @@ class TestSwift3MultiUpload(Swift3TestCase):
segment_bucket = '/v1/AUTH_test/bucket+segments'
objects = \
(('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 100),
('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 200))
objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]},
objects)
object_list = json.dumps(objects)
self.swift.register('PUT',
'/v1/AUTH_test/bucket+segments',
swob.HTTPAccepted, {}, None)
self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
json.dumps([{'name': 'object/X/1',
'last_modified':
'2014-05-07T19:47:54.592270',
'hash': 'HASH',
'bytes': 100},
{'name': 'object/X/2',
'last_modified':
'2014-05-07T19:47:54.592270',
'hash': 'HASH',
'bytes': 100},
]))
object_list)
self.swift.register('HEAD', segment_bucket + '/object/X',
swob.HTTPOk, {}, None)
self.swift.register('PUT', segment_bucket + '/object/X',
@@ -74,6 +84,32 @@ class TestSwift3MultiUpload(Swift3TestCase):
self.swift.register('DELETE', segment_bucket + '/object/X/2',
swob.HTTPNoContent, {}, None)
self.swift.register('HEAD', segment_bucket + '/object/Y',
swob.HTTPOk, {}, None)
self.swift.register('PUT', segment_bucket + '/object/Y',
swob.HTTPCreated, {}, None)
self.swift.register('DELETE', segment_bucket + '/object/Y',
swob.HTTPNoContent, {}, None)
self.swift.register('PUT', segment_bucket + '/object/Y/1',
swob.HTTPCreated, {}, None)
self.swift.register('DELETE', segment_bucket + '/object/Y/1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', segment_bucket + '/object/Y/2',
swob.HTTPNoContent, {}, None)
self.swift.register('HEAD', segment_bucket + '/object2/Z',
swob.HTTPOk, {}, None)
self.swift.register('PUT', segment_bucket + '/object2/Z',
swob.HTTPCreated, {}, None)
self.swift.register('DELETE', segment_bucket + '/object2/Z',
swob.HTTPNoContent, {}, None)
self.swift.register('PUT', segment_bucket + '/object2/Z/1',
swob.HTTPCreated, {}, None)
self.swift.register('DELETE', segment_bucket + '/object2/Z/1',
swob.HTTPNoContent, {}, None)
self.swift.register('DELETE', segment_bucket + '/object2/Z/2',
swob.HTTPNoContent, {}, None)
@s3acl
def test_bucket_upload_part(self):
req = Request.blank('/bucket?partNumber=1&uploadId=x',
@@ -122,15 +158,181 @@ class TestSwift3MultiUpload(Swift3TestCase):
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidRequest')
@s3acl
def test_bucket_multipart_uploads_GET(self):
req = Request.blank('/bucket/?uploads',
def _test_bucket_multipart_uploads_GET(self, query=None,
multiparts=None):
segment_bucket = '/v1/AUTH_test/bucket+segments'
objects = multiparts or multiparts_template
objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]},
objects)
object_list = json.dumps(objects)
self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
object_list)
query = '?uploads&' + query if query else '?uploads'
req = Request.blank('/bucket/%s' % query,
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
fromstring(body, 'ListMultipartUploadsResult')
return self.call_swift3(req)
@s3acl
def test_bucket_multipart_uploads_GET(self):
status, headers, body = self._test_bucket_multipart_uploads_GET()
elem = fromstring(body, 'ListMultipartUploadsResult')
self.assertEquals(elem.find('Bucket').text, 'bucket')
self.assertEquals(elem.find('KeyMarker').text, None)
self.assertEquals(elem.find('UploadIdMarker').text, None)
self.assertEquals(elem.find('NextUploadIdMarker').text, 'Z')
self.assertEquals(elem.find('MaxUploads').text, '1000')
self.assertEquals(elem.find('IsTruncated').text, 'false')
self.assertEquals(len(elem.findall('Upload')), 3)
objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts_template]
for u in elem.findall('Upload'):
name = u.find('Key').text + '/' + u.find('UploadId').text
initiated = u.find('Initiated').text
self.assertTrue((name, initiated) in objects)
self.assertEquals(u.find('Initiator/ID').text, 'test:tester')
self.assertEquals(u.find('Initiator/DisplayName').text,
'test:tester')
self.assertEquals(u.find('Owner/ID').text, 'test:tester')
self.assertEquals(u.find('Owner/DisplayName').text, 'test:tester')
self.assertEquals(u.find('StorageClass').text, 'STANDARD')
self.assertEquals(status.split()[0], '200')
@s3acl
def test_bucket_multipart_uploads_GET_encoding_type_error(self):
query = 'encoding-type=xml'
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
@s3acl
def test_bucket_multipart_uploads_GET_maxuploads(self):
query = 'max-uploads=2'
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query)
elem = fromstring(body, 'ListMultipartUploadsResult')
self.assertEquals(len(elem.findall('Upload/UploadId')), 2)
self.assertEquals(elem.find('NextKeyMarker').text, 'object')
self.assertEquals(elem.find('NextUploadIdMarker').text, 'Y')
self.assertEquals(elem.find('MaxUploads').text, '2')
self.assertEquals(elem.find('IsTruncated').text, 'true')
self.assertEquals(status.split()[0], '200')
@s3acl
def test_bucket_multipart_uploads_GET_str_maxuploads(self):
query = 'max-uploads=invalid'
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
@s3acl
def test_bucket_multipart_uploads_GET_negative_maxuploads(self):
query = 'max-uploads=-1'
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
@s3acl
def test_bucket_multipart_uploads_GET_maxuploads_over_default(self):
query = 'max-uploads=1001'
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
@s3acl
def test_bucket_multipart_uploads_GET_with_id_and_key_marker(self):
query = 'upload-id-marker=Y&key-marker=object'
multiparts = \
(('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2),
('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12),
('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22))
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query, multiparts)
elem = fromstring(body, 'ListMultipartUploadsResult')
self.assertEquals(elem.find('KeyMarker').text, 'object')
self.assertEquals(elem.find('UploadIdMarker').text, 'Y')
self.assertEquals(len(elem.findall('Upload')), 1)
objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts]
for u in elem.findall('Upload'):
name = u.find('Key').text + '/' + u.find('UploadId').text
initiated = u.find('Initiated').text
self.assertTrue((name, initiated) in objects)
self.assertEquals(status.split()[0], '200')
_, path, _ = self.swift.calls_with_headers[-1]
path, query_string = path.split('?', 1)
query = {}
for q in query_string.split('&'):
key, arg = q.split('=')
query[key] = arg
self.assertEquals(query['format'], 'json')
self.assertEquals(query['limit'], '1001')
self.assertEquals(query['marker'], 'object/Y')
@s3acl
def test_bucket_multipart_uploads_GET_with_key_marker(self):
query = 'key-marker=object'
multiparts = \
(('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1),
('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11),
('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21),
('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2),
('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12),
('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22))
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query, multiparts)
elem = fromstring(body, 'ListMultipartUploadsResult')
self.assertEquals(elem.find('KeyMarker').text, 'object')
self.assertEquals(elem.find('NextKeyMarker').text, 'object')
self.assertEquals(elem.find('NextUploadIdMarker').text, 'Y')
self.assertEquals(len(elem.findall('Upload')), 2)
objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts]
for u in elem.findall('Upload'):
name = u.find('Key').text + '/' + u.find('UploadId').text
initiated = u.find('Initiated').text
self.assertTrue((name, initiated) in objects)
self.assertEquals(status.split()[0], '200')
_, path, _ = self.swift.calls_with_headers[-1]
path, query_string = path.split('?', 1)
query = {}
for q in query_string.split('&'):
key, arg = q.split('=')
query[key] = arg
self.assertEquals(query['format'], 'json')
self.assertEquals(query['limit'], '1001')
self.assertEquals(query['marker'], quote('object/~'))
@s3acl
def test_bucket_multipart_uploads_GET_with_prefix(self):
query = 'prefix=X'
multiparts = \
(('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1),
('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11),
('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21))
status, headers, body = \
self._test_bucket_multipart_uploads_GET(query, multiparts)
elem = fromstring(body, 'ListMultipartUploadsResult')
self.assertEquals(len(elem.findall('Upload')), 1)
objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts]
for u in elem.findall('Upload'):
name = u.find('Key').text + '/' + u.find('UploadId').text
initiated = u.find('Initiated').text
self.assertTrue((name, initiated) in objects)
self.assertEquals(status.split()[0], '200')
_, path, _ = self.swift.calls_with_headers[-1]
path, query_string = path.split('?', 1)
query = {}
for q in query_string.split('&'):
key, arg = q.split('=')
query[key] = arg
self.assertEquals(query['format'], 'json')
self.assertEquals(query['limit'], '1001')
self.assertEquals(query['prefix'], 'X')
@s3acl
@patch('swift3.controllers.multi_upload.unique_id', lambda: 'X')
def test_object_multipart_upload_initiate(self):