diff --git a/swift3/controllers/multi_upload.py b/swift3/controllers/multi_upload.py index 131a2fda..6909f1a9 100644 --- a/swift3/controllers/multi_upload.py +++ b/swift3/controllers/multi_upload.py @@ -131,27 +131,62 @@ class UploadsController(Controller): """ Handles List Multipart Uploads """ + def filter_max_uploads(o): + name = o.get('name', '') + return name.count('/') == 1 + encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) - # TODO: add support for prefix, key-marker, upload-id-marker, and - # max-uploads queries. + # TODO: add support for delimiter query. + + keymarker = req.params.get('key-marker', '') + uploadid = req.params.get('upload-id-marker', '') + maxuploads = DEFAULT_MAX_UPLOADS + + if 'max-uploads' in req.params: + try: + maxuploads = int(req.params['max-uploads']) + if maxuploads < 0 or DEFAULT_MAX_UPLOADS < maxuploads: + err_msg = 'Argument max-uploads must be an integer ' \ + 'between 0 and %d' % DEFAULT_MAX_UPLOADS + raise InvalidArgument('max-uploads', maxuploads, err_msg) + except ValueError: + err_msg = 'Provided max-uploads not an integer or within ' \ + 'integer range' + raise InvalidArgument('max-uploads', req.params['max-uploads'], + err_msg) + query = { 'format': 'json', + 'limit': maxuploads + 1, } + + if uploadid and keymarker: + query.update({'marker': '%s/%s' % (keymarker, uploadid)}) + elif keymarker: + query.update({'marker': '%s/~' % (keymarker)}) + if 'prefix' in req.params: + query.update({'prefix': req.params['prefix']}) + container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, container=container, query=query) objects = loads(resp.body) - uploads = [] - for o in objects: - obj, upid = split_path('/' + o['name'], 1, 2, True) - if '/' in upid: - # This is a part object. - continue + objects = filter(filter_max_uploads, objects) + if len(objects) > maxuploads: + objects = objects[:maxuploads] + truncated = True + else: + truncated = False + + uploads = [] + prefixes = [] + for o in objects: + obj, upid = split_path('/' + o['name'], 1, 2) uploads.append( {'key': obj, 'upload_id': upid, @@ -166,17 +201,17 @@ class UploadsController(Controller): result_elem = Element('ListMultipartUploadsResult') SubElement(result_elem, 'Bucket').text = req.container_name - SubElement(result_elem, 'KeyMarker').text = '' - SubElement(result_elem, 'UploadIdMarker').text = '' + SubElement(result_elem, 'KeyMarker').text = keymarker + SubElement(result_elem, 'UploadIdMarker').text = uploadid SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker - - SubElement(result_elem, 'MaxUploads').text = str(DEFAULT_MAX_UPLOADS) - + if 'prefix' in req.params: + SubElement(result_elem, 'Prefix').text = req.params['prefix'] + SubElement(result_elem, 'MaxUploads').text = str(maxuploads) if encoding_type is not None: SubElement(result_elem, 'EncodingType').text = encoding_type - - SubElement(result_elem, 'IsTruncated').text = 'false' + SubElement(result_elem, 'IsTruncated').text = \ + 'true' if truncated else 'false' # TODO: don't show uploads which are initiated before this bucket is # created. @@ -194,6 +229,10 @@ class UploadsController(Controller): SubElement(upload_elem, 'Initiated').text = \ u['last_modified'][:-3] + 'Z' + for p in prefixes: + elem = SubElement(result_elem, 'CommonPrefixes') + SubElement(elem, 'Prefix').text = p + body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml') diff --git a/swift3/test/unit/test_multi_upload.py b/swift3/test/unit/test_multi_upload.py index 6fe39811..f28325f1 100644 --- a/swift3/test/unit/test_multi_upload.py +++ b/swift3/test/unit/test_multi_upload.py @@ -16,6 +16,7 @@ import unittest import simplejson as json from mock import patch +from urllib import quote from swift.common import swob from swift.common.swob import Request @@ -36,6 +37,17 @@ xml = '' \ '' \ '' +multiparts_template = \ + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), + ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21), + ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2), + ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12), + ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22), + ('object/Z', '2014-05-07T19:47:56.592270', 'HASH', 3), + ('object/Z/1', '2014-05-07T19:47:57.592270', 'HASH', 13), + ('object/Z/2', '2014-05-07T19:47:58.592270', 'HASH', 23)) + class TestSwift3MultiUpload(Swift3TestCase): @@ -44,21 +56,19 @@ class TestSwift3MultiUpload(Swift3TestCase): segment_bucket = '/v1/AUTH_test/bucket+segments' + objects = \ + (('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 100), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 200)) + objects = map(lambda item: {'name': item[0], 'last_modified': item[1], + 'hash': item[2], 'bytes': item[3]}, + objects) + object_list = json.dumps(objects) + self.swift.register('PUT', '/v1/AUTH_test/bucket+segments', swob.HTTPAccepted, {}, None) self.swift.register('GET', segment_bucket, swob.HTTPOk, {}, - json.dumps([{'name': 'object/X/1', - 'last_modified': - '2014-05-07T19:47:54.592270', - 'hash': 'HASH', - 'bytes': 100}, - {'name': 'object/X/2', - 'last_modified': - '2014-05-07T19:47:54.592270', - 'hash': 'HASH', - 'bytes': 100}, - ])) + object_list) self.swift.register('HEAD', segment_bucket + '/object/X', swob.HTTPOk, {}, None) self.swift.register('PUT', segment_bucket + '/object/X', @@ -74,6 +84,32 @@ class TestSwift3MultiUpload(Swift3TestCase): self.swift.register('DELETE', segment_bucket + '/object/X/2', swob.HTTPNoContent, {}, None) + self.swift.register('HEAD', segment_bucket + '/object/Y', + swob.HTTPOk, {}, None) + self.swift.register('PUT', segment_bucket + '/object/Y', + swob.HTTPCreated, {}, None) + self.swift.register('DELETE', segment_bucket + '/object/Y', + swob.HTTPNoContent, {}, None) + self.swift.register('PUT', segment_bucket + '/object/Y/1', + swob.HTTPCreated, {}, None) + self.swift.register('DELETE', segment_bucket + '/object/Y/1', + swob.HTTPNoContent, {}, None) + self.swift.register('DELETE', segment_bucket + '/object/Y/2', + swob.HTTPNoContent, {}, None) + + self.swift.register('HEAD', segment_bucket + '/object2/Z', + swob.HTTPOk, {}, None) + self.swift.register('PUT', segment_bucket + '/object2/Z', + swob.HTTPCreated, {}, None) + self.swift.register('DELETE', segment_bucket + '/object2/Z', + swob.HTTPNoContent, {}, None) + self.swift.register('PUT', segment_bucket + '/object2/Z/1', + swob.HTTPCreated, {}, None) + self.swift.register('DELETE', segment_bucket + '/object2/Z/1', + swob.HTTPNoContent, {}, None) + self.swift.register('DELETE', segment_bucket + '/object2/Z/2', + swob.HTTPNoContent, {}, None) + @s3acl def test_bucket_upload_part(self): req = Request.blank('/bucket?partNumber=1&uploadId=x', @@ -122,15 +158,181 @@ class TestSwift3MultiUpload(Swift3TestCase): status, headers, body = self.call_swift3(req) self.assertEquals(self._get_error_code(body), 'InvalidRequest') - @s3acl - def test_bucket_multipart_uploads_GET(self): - req = Request.blank('/bucket/?uploads', + def _test_bucket_multipart_uploads_GET(self, query=None, + multiparts=None): + segment_bucket = '/v1/AUTH_test/bucket+segments' + objects = multiparts or multiparts_template + objects = map(lambda item: {'name': item[0], 'last_modified': item[1], + 'hash': item[2], 'bytes': item[3]}, + objects) + object_list = json.dumps(objects) + self.swift.register('GET', segment_bucket, swob.HTTPOk, {}, + object_list) + + query = '?uploads&' + query if query else '?uploads' + req = Request.blank('/bucket/%s' % query, environ={'REQUEST_METHOD': 'GET'}, headers={'Authorization': 'AWS test:tester:hmac'}) - status, headers, body = self.call_swift3(req) - fromstring(body, 'ListMultipartUploadsResult') + return self.call_swift3(req) + + @s3acl + def test_bucket_multipart_uploads_GET(self): + status, headers, body = self._test_bucket_multipart_uploads_GET() + elem = fromstring(body, 'ListMultipartUploadsResult') + self.assertEquals(elem.find('Bucket').text, 'bucket') + self.assertEquals(elem.find('KeyMarker').text, None) + self.assertEquals(elem.find('UploadIdMarker').text, None) + self.assertEquals(elem.find('NextUploadIdMarker').text, 'Z') + self.assertEquals(elem.find('MaxUploads').text, '1000') + self.assertEquals(elem.find('IsTruncated').text, 'false') + self.assertEquals(len(elem.findall('Upload')), 3) + objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts_template] + for u in elem.findall('Upload'): + name = u.find('Key').text + '/' + u.find('UploadId').text + initiated = u.find('Initiated').text + self.assertTrue((name, initiated) in objects) + self.assertEquals(u.find('Initiator/ID').text, 'test:tester') + self.assertEquals(u.find('Initiator/DisplayName').text, + 'test:tester') + self.assertEquals(u.find('Owner/ID').text, 'test:tester') + self.assertEquals(u.find('Owner/DisplayName').text, 'test:tester') + self.assertEquals(u.find('StorageClass').text, 'STANDARD') self.assertEquals(status.split()[0], '200') + @s3acl + def test_bucket_multipart_uploads_GET_encoding_type_error(self): + query = 'encoding-type=xml' + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + @s3acl + def test_bucket_multipart_uploads_GET_maxuploads(self): + query = 'max-uploads=2' + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query) + elem = fromstring(body, 'ListMultipartUploadsResult') + self.assertEquals(len(elem.findall('Upload/UploadId')), 2) + self.assertEquals(elem.find('NextKeyMarker').text, 'object') + self.assertEquals(elem.find('NextUploadIdMarker').text, 'Y') + self.assertEquals(elem.find('MaxUploads').text, '2') + self.assertEquals(elem.find('IsTruncated').text, 'true') + self.assertEquals(status.split()[0], '200') + + @s3acl + def test_bucket_multipart_uploads_GET_str_maxuploads(self): + query = 'max-uploads=invalid' + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + @s3acl + def test_bucket_multipart_uploads_GET_negative_maxuploads(self): + query = 'max-uploads=-1' + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + @s3acl + def test_bucket_multipart_uploads_GET_maxuploads_over_default(self): + query = 'max-uploads=1001' + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query) + self.assertEquals(self._get_error_code(body), 'InvalidArgument') + + @s3acl + def test_bucket_multipart_uploads_GET_with_id_and_key_marker(self): + query = 'upload-id-marker=Y&key-marker=object' + multiparts = \ + (('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2), + ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12), + ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22)) + + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query, multiparts) + elem = fromstring(body, 'ListMultipartUploadsResult') + self.assertEquals(elem.find('KeyMarker').text, 'object') + self.assertEquals(elem.find('UploadIdMarker').text, 'Y') + self.assertEquals(len(elem.findall('Upload')), 1) + objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts] + for u in elem.findall('Upload'): + name = u.find('Key').text + '/' + u.find('UploadId').text + initiated = u.find('Initiated').text + self.assertTrue((name, initiated) in objects) + self.assertEquals(status.split()[0], '200') + + _, path, _ = self.swift.calls_with_headers[-1] + path, query_string = path.split('?', 1) + query = {} + for q in query_string.split('&'): + key, arg = q.split('=') + query[key] = arg + self.assertEquals(query['format'], 'json') + self.assertEquals(query['limit'], '1001') + self.assertEquals(query['marker'], 'object/Y') + + @s3acl + def test_bucket_multipart_uploads_GET_with_key_marker(self): + query = 'key-marker=object' + multiparts = \ + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), + ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21), + ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2), + ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12), + ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22)) + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query, multiparts) + elem = fromstring(body, 'ListMultipartUploadsResult') + self.assertEquals(elem.find('KeyMarker').text, 'object') + self.assertEquals(elem.find('NextKeyMarker').text, 'object') + self.assertEquals(elem.find('NextUploadIdMarker').text, 'Y') + self.assertEquals(len(elem.findall('Upload')), 2) + objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts] + for u in elem.findall('Upload'): + name = u.find('Key').text + '/' + u.find('UploadId').text + initiated = u.find('Initiated').text + self.assertTrue((name, initiated) in objects) + self.assertEquals(status.split()[0], '200') + + _, path, _ = self.swift.calls_with_headers[-1] + path, query_string = path.split('?', 1) + query = {} + for q in query_string.split('&'): + key, arg = q.split('=') + query[key] = arg + self.assertEquals(query['format'], 'json') + self.assertEquals(query['limit'], '1001') + self.assertEquals(query['marker'], quote('object/~')) + + @s3acl + def test_bucket_multipart_uploads_GET_with_prefix(self): + query = 'prefix=X' + multiparts = \ + (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1), + ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11), + ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21)) + status, headers, body = \ + self._test_bucket_multipart_uploads_GET(query, multiparts) + elem = fromstring(body, 'ListMultipartUploadsResult') + self.assertEquals(len(elem.findall('Upload')), 1) + objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts] + for u in elem.findall('Upload'): + name = u.find('Key').text + '/' + u.find('UploadId').text + initiated = u.find('Initiated').text + self.assertTrue((name, initiated) in objects) + self.assertEquals(status.split()[0], '200') + + _, path, _ = self.swift.calls_with_headers[-1] + path, query_string = path.split('?', 1) + query = {} + for q in query_string.split('&'): + key, arg = q.split('=') + query[key] = arg + self.assertEquals(query['format'], 'json') + self.assertEquals(query['limit'], '1001') + self.assertEquals(query['prefix'], 'X') + @s3acl @patch('swift3.controllers.multi_upload.unique_id', lambda: 'X') def test_object_multipart_upload_initiate(self):