x/swift3
Code Issues Proposed changes

Fix max-parts, part-number-marker, and encoding-type in queries of List Parts

Browse Source 
I've fixed ToDo in qeuries of  List Part.

encoding-type: Requests Amazon S3 to encode the response and specifies the encoding method to use.
max-parts: Sets the maximum number of parts to return in the response body.
part-number-marker: Specifies the part after which listing should begin. Only parts with higher part numbers will be listed.

For details, refer to the following.
http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadListParts.html

Change-Id: I1a3b3a0aba188fcb928e89e78ad8859dbecca2e8
This commit is contained in:
Masaki Tsukuda
2014-12-18 13:36:27 +09:00
parent 315604c31b
commit 9a95fc22a5
4 changed files with 188 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
etc/proxy-server.conf-sample
View File

@@ -44,6 +44,9 @@ use = egg:swift3#swift3
# response.
# max_bucket_listing = 1000
#
# Set the maximum number of parts returned in the List Parts operation.
# max_parts = 10000
#
# Set the maximum number of objects we can delete with the Multi-Object Delete
# operation.
# max_multi_delete_objects = 1000

1
swift3/cfg.py
View File

@@ -54,6 +54,7 @@ CONF = Config({
'allow_no_owner': False,
'location': 'US',
'max_bucket_listing': 1000,
'max_parts': 10000,
'max_multi_delete_objects': 1000,
's3_acl': False,
'storage_domain': '',

74
swift3/controllers/multi_upload.py
View File

@@ -56,6 +56,7 @@ from swift3.exception import BadSwiftRequest
from swift3.utils import LOGGER, unique_id, MULTIUPLOAD_SUFFIX
from swift3.etree import Element, SubElement, fromstring, tostring, \
XMLSyntaxError, DocumentInvalid
from swift3.cfg import CONF
DEFAULT_MAX_PARTS = 1000
DEFAULT_MAX_UPLOADS = 1000
@@ -280,6 +281,13 @@ class UploadController(Controller):
"""
Handles List Parts.
"""
def filter_part_num_marker(o):
try:
num = int(os.path.basename(o['name']))
return num > part_num_marker
except ValueError:
return False
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
@@ -288,11 +296,43 @@ class UploadController(Controller):
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
maxparts = DEFAULT_MAX_PARTS
part_num_marker = 0
# TODO: add support for max-parts and part-number-marker queries.
if 'max-parts' in req.params:
try:
maxparts = int(req.params['max-parts'])
if maxparts < 0 or CONF.max_parts < maxparts:
err_msg = 'Argument max-parts must be an integer between 0 and' \
' %d' % CONF.max_parts
raise InvalidArgument('max-parts',
req.params['max-parts'],
err_msg)
except ValueError:
err_msg = 'Provided max-parts not an integer or within ' \
'integer range'
raise InvalidArgument('max-parts', req.params['max-parts'],
err_msg)
if 'part-number-marker' in req.params:
try:
part_num_marker = int(req.params['part-number-marker'])
if part_num_marker < 0 or CONF.max_parts < part_num_marker:
err_msg = 'Argument part-number-marker must be an integer ' \
'between 0 and %d' % CONF.max_parts
raise InvalidArgument('part-number-marker',
req.params['part-number-marker'],
err_msg)
except ValueError:
err_msg = 'Provided part-number-marker not an integer or ' \
'within integer range'
raise InvalidArgument('part-number-marker',
req.params['part-number-marker'],
err_msg)
query = {
'format': 'json',
'limit': maxparts + 1,
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
@@ -304,11 +344,24 @@ class UploadController(Controller):
last_part = 0
# pylint: disable-msg=E1103
objects.sort(key=lambda o: int(o['name'].split('/')[-1]))
# If the caller requested a list starting at a specific part number,
# construct a sub-set of the object list.
objList = filter(filter_part_num_marker, objects)
if len(objects) > 0:
o = objects[-1]
# pylint: disable-msg=E1103
objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
if len(objList) > maxparts:
objList = objList[:maxparts]
truncated = True
else:
truncated = False
# TODO: We have to retrieve object list again when truncated is True
# and some objects filtered by invalid name because there could be no
# enough objects for limit defined by maxparts.
if objList:
o = objList[-1]
last_part = os.path.basename(o['name'])
result_elem = Element('ListPartsResult')
@@ -326,11 +379,14 @@ class UploadController(Controller):
SubElement(result_elem, 'StorageClass').text = 'STANDARD'
SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker)
SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part)
SubElement(result_elem, 'MaxParts').text = str(DEFAULT_MAX_PARTS)
# TODO: add support for EncodingType
SubElement(result_elem, 'IsTruncated').text = 'false'
SubElement(result_elem, 'MaxParts').text = str(maxparts)
if 'encoding-type' in req.params:
SubElement(result_elem, 'EncodingType').text = \
req.params['encoding-type']
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
for i in objects:
for i in objList:
part_elem = SubElement(result_elem, 'Part')
SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1]
SubElement(part_elem, 'LastModified').text = \

124
swift3/test/unit/test_multi_upload.py
View File

@@ -25,6 +25,7 @@ from swift3.test.unit import Swift3TestCase
from swift3.etree import fromstring
from swift3.subresource import Owner, Grant, User, ACL, encode_acl
from swift3.test.unit.test_s3_acl import s3acl
from swift3.cfg import CONF
xml = '<CompleteMultipartUpload>' \
'<Part>' \
@@ -37,6 +38,10 @@ xml = '<CompleteMultipartUpload>' \
'</Part>' \
'</CompleteMultipartUpload>'
objects_template = \
(('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 100),
('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 200))
multiparts_template = \
(('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1),
('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11),
@@ -56,12 +61,9 @@ class TestSwift3MultiUpload(Swift3TestCase):
segment_bucket = '/v1/AUTH_test/bucket+segments'
objects = \
(('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 100),
('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 200))
objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
'hash': item[2], 'bytes': item[3]},
objects)
objects_template)
object_list = json.dumps(objects)
self.swift.register('PUT',
@@ -412,7 +414,119 @@ class TestSwift3MultiUpload(Swift3TestCase):
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
fromstring(body, 'ListPartsResult')
elem = fromstring(body, 'ListPartsResult')
self.assertEquals(elem.find('Bucket').text, 'bucket')
self.assertEquals(elem.find('Key').text, 'object')
self.assertEquals(elem.find('UploadId').text, 'X')
self.assertEquals(elem.find('Initiator/ID').text, 'test:tester')
self.assertEquals(elem.find('Initiator/ID').text, 'test:tester')
self.assertEquals(elem.find('Owner/ID').text, 'test:tester')
self.assertEquals(elem.find('Owner/ID').text, 'test:tester')
self.assertEquals(elem.find('StorageClass').text, 'STANDARD')
self.assertEquals(elem.find('PartNumberMarker').text, '0')
self.assertEquals(elem.find('NextPartNumberMarker').text, '2')
self.assertEquals(elem.find('MaxParts').text, '1000')
self.assertEquals(elem.find('IsTruncated').text, 'false')
self.assertEquals(len(elem.findall('Part')), 2)
for p in elem.findall('Part'):
partnum = int(p.find('PartNumber').text)
self.assertEquals(p.find('LastModified').text,
objects_template[partnum - 1][1][:-3]
+ 'Z')
self.assertEquals(p.find('ETag').text,
objects_template[partnum - 1][2])
self.assertEquals(p.find('Size').text,
str(objects_template[partnum - 1][3]))
self.assertEquals(status.split()[0], '200')
def test_object_list_parts_encoding_type(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object@@/X',
swob.HTTPOk, {}, None)
req = Request.blank('/bucket/object@@?uploadId=X&encoding-type=url',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
elem = fromstring(body, 'ListPartsResult')
self.assertEquals(elem.find('Key').text, quote('object@@'))
self.assertEquals(elem.find('EncodingType').text, 'url')
self.assertEquals(status.split()[0], '200')
def test_object_list_parts_without_encoding_type(self):
self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object@@/X',
swob.HTTPOk, {}, None)
req = Request.blank('/bucket/object@@?uploadId=X',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
elem = fromstring(body, 'ListPartsResult')
self.assertEquals(elem.find('Key').text, 'object@@')
self.assertEquals(status.split()[0], '200')
def test_object_list_parts_encoding_type_error(self):
req = Request.blank('/bucket/object?uploadId=X&encoding-type=xml',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
def test_object_list_parts_max_parts(self):
req = Request.blank('/bucket/object?uploadId=X&max-parts=1',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
elem = fromstring(body, 'ListPartsResult')
self.assertEquals(elem.find('IsTruncated').text, 'true')
self.assertEquals(len(elem.findall('Part')), 1)
self.assertEquals(status.split()[0], '200')
def test_object_list_parts_str_max_parts(self):
req = Request.blank('/bucket/object?uploadId=X&max-parts=invalid',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
def test_object_list_parts_negative_max_parts(self):
req = Request.blank('/bucket/object?uploadId=X&max-parts=-1',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
def test_object_list_parts_over_max_parts(self):
req = Request.blank('/bucket/object?uploadId=X&max-parts=%d' %
(CONF.max_parts + 1),
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
def test_object_list_parts_with_part_number_marker(self):
req = Request.blank('/bucket/object?uploadId=X&'
'part-number-marker=1',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
elem = fromstring(body, 'ListPartsResult')
self.assertEquals(len(elem.findall('Part')), 1)
self.assertEquals(elem.find('Part/PartNumber').text, '2')
self.assertEquals(status.split()[0], '200')
def test_object_list_parts_invalid_part_number_marker(self):
req = Request.blank('/bucket/object?uploadId=X&part-number-marker='
'invalid',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
self.assertEquals(self._get_error_code(body), 'InvalidArgument')
def test_object_list_parts_same_max_marts_as_objects_num(self):
req = Request.blank('/bucket/object?uploadId=X&max-parts=2',
environ={'REQUEST_METHOD': 'GET'},
headers={'Authorization': 'AWS test:tester:hmac'})
status, headers, body = self.call_swift3(req)
elem = fromstring(body, 'ListPartsResult')
self.assertEquals(len(elem.findall('Part')), 2)
self.assertEquals(status.split()[0], '200')
def _test_for_s3acl(self, method, query, account, hasObj=True, body=None):