diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 200c8d07..8d19f644 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -44,6 +44,9 @@ use = egg:swift3#swift3 # response. # max_bucket_listing = 1000 # +# Set the maximum number of parts returned in the List Parts operation. +# max_parts = 10000 +# # Set the maximum number of objects we can delete with the Multi-Object Delete # operation. # max_multi_delete_objects = 1000 diff --git a/swift3/cfg.py b/swift3/cfg.py index 811738c8..1520d981 100644 --- a/swift3/cfg.py +++ b/swift3/cfg.py @@ -54,6 +54,7 @@ CONF = Config({ 'allow_no_owner': False, 'location': 'US', 'max_bucket_listing': 1000, + 'max_parts': 10000, 'max_multi_delete_objects': 1000, 's3_acl': False, 'storage_domain': '', diff --git a/swift3/controllers/multi_upload.py b/swift3/controllers/multi_upload.py index 6909f1a9..684cd3ea 100644 --- a/swift3/controllers/multi_upload.py +++ b/swift3/controllers/multi_upload.py @@ -56,6 +56,7 @@ from swift3.exception import BadSwiftRequest from swift3.utils import LOGGER, unique_id, MULTIUPLOAD_SUFFIX from swift3.etree import Element, SubElement, fromstring, tostring, \ XMLSyntaxError, DocumentInvalid +from swift3.cfg import CONF DEFAULT_MAX_PARTS = 1000 DEFAULT_MAX_UPLOADS = 1000 @@ -280,6 +281,13 @@ class UploadController(Controller): """ Handles List Parts. """ + def filter_part_num_marker(o): + try: + num = int(os.path.basename(o['name'])) + return num > part_num_marker + except ValueError: + return False + encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' @@ -288,11 +296,43 @@ class UploadController(Controller): upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) + maxparts = DEFAULT_MAX_PARTS part_num_marker = 0 - # TODO: add support for max-parts and part-number-marker queries. + if 'max-parts' in req.params: + try: + maxparts = int(req.params['max-parts']) + if maxparts < 0 or CONF.max_parts < maxparts: + err_msg = 'Argument max-parts must be an integer between 0 and' \ + ' %d' % CONF.max_parts + raise InvalidArgument('max-parts', + req.params['max-parts'], + err_msg) + except ValueError: + err_msg = 'Provided max-parts not an integer or within ' \ + 'integer range' + raise InvalidArgument('max-parts', req.params['max-parts'], + err_msg) + + if 'part-number-marker' in req.params: + try: + part_num_marker = int(req.params['part-number-marker']) + if part_num_marker < 0 or CONF.max_parts < part_num_marker: + err_msg = 'Argument part-number-marker must be an integer ' \ + 'between 0 and %d' % CONF.max_parts + raise InvalidArgument('part-number-marker', + req.params['part-number-marker'], + err_msg) + except ValueError: + err_msg = 'Provided part-number-marker not an integer or ' \ + 'within integer range' + raise InvalidArgument('part-number-marker', + req.params['part-number-marker'], + err_msg) + query = { 'format': 'json', + 'limit': maxparts + 1, 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } @@ -304,11 +344,24 @@ class UploadController(Controller): last_part = 0 - # pylint: disable-msg=E1103 - objects.sort(key=lambda o: int(o['name'].split('/')[-1])) + # If the caller requested a list starting at a specific part number, + # construct a sub-set of the object list. + objList = filter(filter_part_num_marker, objects) - if len(objects) > 0: - o = objects[-1] + # pylint: disable-msg=E1103 + objList.sort(key=lambda o: int(o['name'].split('/')[-1])) + + if len(objList) > maxparts: + objList = objList[:maxparts] + truncated = True + else: + truncated = False + # TODO: We have to retrieve object list again when truncated is True + # and some objects filtered by invalid name because there could be no + # enough objects for limit defined by maxparts. + + if objList: + o = objList[-1] last_part = os.path.basename(o['name']) result_elem = Element('ListPartsResult') @@ -326,11 +379,14 @@ class UploadController(Controller): SubElement(result_elem, 'StorageClass').text = 'STANDARD' SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker) SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part) - SubElement(result_elem, 'MaxParts').text = str(DEFAULT_MAX_PARTS) - # TODO: add support for EncodingType - SubElement(result_elem, 'IsTruncated').text = 'false' + SubElement(result_elem, 'MaxParts').text = str(maxparts) + if 'encoding-type' in req.params: + SubElement(result_elem, 'EncodingType').text = \ + req.params['encoding-type'] + SubElement(result_elem, 'IsTruncated').text = \ + 'true' if truncated else 'false' - for i in objects: + for i in objList: part_elem = SubElement(result_elem, 'Part') SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1] SubElement(part_elem, 'LastModified').text = \ diff --git a/swift3/test/unit/test_multi_upload.py b/swift3/test/unit/test_multi_upload.py index f28325f1..32fd805a 100644 --- a/swift3/test/unit/test_multi_upload.py +++ b/swift3/test/unit/test_multi_upload.py @@ -25,6 +25,7 @@ from swift3.test.unit import Swift3TestCase from swift3.etree import fromstring from swift3.subresource import Owner, Grant, User, ACL, encode_acl from swift3.test.unit.test_s3_acl import s3acl +from swift3.cfg import CONF xml = '' \ '' \ @@ -37,6 +38,10 @@ xml = '' \ '' \ '' +objects_template = \ + (('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 100), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 200)) + multiparts_template = \ (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), @@ -56,12 +61,9 @@ class TestSwift3MultiUpload(Swift3TestCase): segment_bucket = '/v1/AUTH_test/bucket+segments' - objects = \ - (('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 100), - ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 200)) objects = map(lambda item: {'name': item[0], 'last_modified': item[1], 'hash': item[2], 'bytes': item[3]}, - objects) + objects_template) object_list = json.dumps(objects) self.swift.register('PUT', @@ -412,7 +414,119 @@ class TestSwift3MultiUpload(Swift3TestCase): environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac'}) status, headers, body = self.call_swift3(req) - fromstring(body, 'ListPartsResult') + elem = fromstring(body, 'ListPartsResult') + self.assertEquals(elem.find('Bucket').text, 'bucket') + self.assertEquals(elem.find('Key').text, 'object') + self.assertEquals(elem.find('UploadId').text, 'X') + self.assertEquals(elem.find('Initiator/ID').text, 'test:tester') + self.assertEquals(elem.find('Initiator/ID').text, 'test:tester') + self.assertEquals(elem.find('Owner/ID').text, 'test:tester') + self.assertEquals(elem.find('Owner/ID').text, 'test:tester') + self.assertEquals(elem.find('StorageClass').text, 'STANDARD') + self.assertEquals(elem.find('PartNumberMarker').text, '0') + self.assertEquals(elem.find('NextPartNumberMarker').text, '2') + self.assertEquals(elem.find('MaxParts').text, '1000') + self.assertEquals(elem.find('IsTruncated').text, 'false') + self.assertEquals(len(elem.findall('Part')), 2) + for p in elem.findall('Part'): + partnum = int(p.find('PartNumber').text) + self.assertEquals(p.find('LastModified').text, + objects_template[partnum - 1][1][:-3] + + 'Z') + self.assertEquals(p.find('ETag').text, + objects_template[partnum - 1][2]) + self.assertEquals(p.find('Size').text, + str(objects_template[partnum - 1][3])) + self.assertEquals(status.split()[0], '200') + + def test_object_list_parts_encoding_type(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object@@/X', + swob.HTTPOk, {}, None) + req = Request.blank('/bucket/object@@?uploadId=X&encoding-type=url', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + elem = fromstring(body, 'ListPartsResult') + self.assertEquals(elem.find('Key').text, quote('object@@')) + self.assertEquals(elem.find('EncodingType').text, 'url') + self.assertEquals(status.split()[0], '200') + + def test_object_list_parts_without_encoding_type(self): + self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object@@/X', + swob.HTTPOk, {}, None) + req = Request.blank('/bucket/object@@?uploadId=X', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + elem = fromstring(body, 'ListPartsResult') + self.assertEquals(elem.find('Key').text, 'object@@') + self.assertEquals(status.split()[0], '200') + + def test_object_list_parts_encoding_type_error(self): + req = Request.blank('/bucket/object?uploadId=X&encoding-type=xml', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + def test_object_list_parts_max_parts(self): + req = Request.blank('/bucket/object?uploadId=X&max-parts=1', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + elem = fromstring(body, 'ListPartsResult') + self.assertEquals(elem.find('IsTruncated').text, 'true') + self.assertEquals(len(elem.findall('Part')), 1) + self.assertEquals(status.split()[0], '200') + + def test_object_list_parts_str_max_parts(self): + req = Request.blank('/bucket/object?uploadId=X&max-parts=invalid', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + def test_object_list_parts_negative_max_parts(self): + req = Request.blank('/bucket/object?uploadId=X&max-parts=-1', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + def test_object_list_parts_over_max_parts(self): + req = Request.blank('/bucket/object?uploadId=X&max-parts=%d' % + (CONF.max_parts + 1), + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + def test_object_list_parts_with_part_number_marker(self): + req = Request.blank('/bucket/object?uploadId=X&' + 'part-number-marker=1', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + elem = fromstring(body, 'ListPartsResult') + self.assertEquals(len(elem.findall('Part')), 1) + self.assertEquals(elem.find('Part/PartNumber').text, '2') + self.assertEquals(status.split()[0], '200') + + def test_object_list_parts_invalid_part_number_marker(self): + req = Request.blank('/bucket/object?uploadId=X&part-number-marker=' + 'invalid', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + def test_object_list_parts_same_max_marts_as_objects_num(self): + req = Request.blank('/bucket/object?uploadId=X&max-parts=2', + environ={'REQUEST_METHOD': 'GET'}, + headers={'Authorization': 'AWS test:tester:hmac'}) + status, headers, body = self.call_swift3(req) + elem = fromstring(body, 'ListPartsResult') + self.assertEquals(len(elem.findall('Part')), 2) self.assertEquals(status.split()[0], '200') def _test_for_s3acl(self, method, query, account, hasObj=True, body=None):