Merge "Fix max-uploads, key-marker, prefix and upload-id-marker in queries of List Multipart Uploads"

swift3/controllers/multi_upload.py

@@ -131,27 +131,62 @@ class UploadsController(Controller):
         """
         Handles List Multipart Uploads
         """
+        def filter_max_uploads(o):
+            name = o.get('name', '')
+            return name.count('/') == 1
+
         encoding_type = req.params.get('encoding-type')
         if encoding_type is not None and encoding_type != 'url':
             err_msg = 'Invalid Encoding Method specified in Request'
             raise InvalidArgument('encoding-type', encoding_type, err_msg)
 
-        # TODO: add support for prefix, key-marker, upload-id-marker, and
+        # TODO: add support for delimiter query.
-        # max-uploads queries.
+
+        keymarker = req.params.get('key-marker', '')
+        uploadid = req.params.get('upload-id-marker', '')
+        maxuploads = DEFAULT_MAX_UPLOADS
+
+        if 'max-uploads' in req.params:
+            try:
+                maxuploads = int(req.params['max-uploads'])
+                if maxuploads < 0 or DEFAULT_MAX_UPLOADS < maxuploads:
+                    err_msg = 'Argument max-uploads must be an integer ' \
+                              'between 0 and %d' % DEFAULT_MAX_UPLOADS
+                    raise InvalidArgument('max-uploads', maxuploads, err_msg)
+            except ValueError:
+                err_msg = 'Provided max-uploads not an integer or within ' \
+                          'integer range'
+                raise InvalidArgument('max-uploads', req.params['max-uploads'],
+                                      err_msg)
+
         query = {
             'format': 'json',
+            'limit': maxuploads + 1,
         }
+
+        if uploadid and keymarker:
+            query.update({'marker': '%s/%s' % (keymarker, uploadid)})
+        elif keymarker:
+            query.update({'marker': '%s/~' % (keymarker)})
+        if 'prefix' in req.params:
+            query.update({'prefix': req.params['prefix']})
+
         container = req.container_name + MULTIUPLOAD_SUFFIX
         resp = req.get_response(self.app, container=container, query=query)
         objects = loads(resp.body)
 
-        uploads = []
+        objects = filter(filter_max_uploads, objects)
-        for o in objects:
-            obj, upid = split_path('/' + o['name'], 1, 2, True)
-            if '/' in upid:
-                # This is a part object.
-                continue
 
+        if len(objects) > maxuploads:
+            objects = objects[:maxuploads]
+            truncated = True
+        else:
+            truncated = False
+
+        uploads = []
+        prefixes = []
+        for o in objects:
+            obj, upid = split_path('/' + o['name'], 1, 2)
             uploads.append(
                 {'key': obj,
                  'upload_id': upid,
@@ -166,17 +201,17 @@ class UploadsController(Controller):
 
         result_elem = Element('ListMultipartUploadsResult')
         SubElement(result_elem, 'Bucket').text = req.container_name
-        SubElement(result_elem, 'KeyMarker').text = ''
+        SubElement(result_elem, 'KeyMarker').text = keymarker
-        SubElement(result_elem, 'UploadIdMarker').text = ''
+        SubElement(result_elem, 'UploadIdMarker').text = uploadid
         SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker
         SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker
-
+        if 'prefix' in req.params:
-        SubElement(result_elem, 'MaxUploads').text = str(DEFAULT_MAX_UPLOADS)
+            SubElement(result_elem, 'Prefix').text = req.params['prefix']
-
+        SubElement(result_elem, 'MaxUploads').text = str(maxuploads)
         if encoding_type is not None:
             SubElement(result_elem, 'EncodingType').text = encoding_type
-
+        SubElement(result_elem, 'IsTruncated').text = \
-        SubElement(result_elem, 'IsTruncated').text = 'false'
+            'true' if truncated else 'false'
 
         # TODO: don't show uploads which are initiated before this bucket is
         # created.
@@ -194,6 +229,10 @@ class UploadsController(Controller):
             SubElement(upload_elem, 'Initiated').text = \
                 u['last_modified'][:-3] + 'Z'
 
+        for p in prefixes:
+            elem = SubElement(result_elem, 'CommonPrefixes')
+            SubElement(elem, 'Prefix').text = p
+
         body = tostring(result_elem, encoding_type=encoding_type)
 
         return HTTPOk(body=body, content_type='application/xml')

swift3/test/unit/test_multi_upload.py

@@ -16,6 +16,7 @@
 import unittest
 import simplejson as json
 from mock import patch
+from urllib import quote
 
 from swift.common import swob
 from swift.common.swob import Request
@@ -36,6 +37,17 @@ xml = '<CompleteMultipartUpload>' \
     '</Part>' \
     '</CompleteMultipartUpload>'
 
+multiparts_template = \
+    (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1),
+     ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11),
+     ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21),
+     ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2),
+     ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12),
+     ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22),
+     ('object/Z', '2014-05-07T19:47:56.592270', 'HASH', 3),
+     ('object/Z/1', '2014-05-07T19:47:57.592270', 'HASH', 13),
+     ('object/Z/2', '2014-05-07T19:47:58.592270', 'HASH', 23))
+
 
 class TestSwift3MultiUpload(Swift3TestCase):
 
@@ -44,21 +56,19 @@ class TestSwift3MultiUpload(Swift3TestCase):
 
         segment_bucket = '/v1/AUTH_test/bucket+segments'
 
+        objects = \
+            (('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 100),
+             ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 200))
+        objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
+                                    'hash': item[2], 'bytes': item[3]},
+                      objects)
+        object_list = json.dumps(objects)
+
         self.swift.register('PUT',
                             '/v1/AUTH_test/bucket+segments',
                             swob.HTTPAccepted, {}, None)
         self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
-                            json.dumps([{'name': 'object/X/1',
+                            object_list)
-                                         'last_modified':
-                                             '2014-05-07T19:47:54.592270',
-                                         'hash': 'HASH',
-                                         'bytes': 100},
-                                        {'name': 'object/X/2',
-                                         'last_modified':
-                                             '2014-05-07T19:47:54.592270',
-                                         'hash': 'HASH',
-                                         'bytes': 100},
-                                        ]))
         self.swift.register('HEAD', segment_bucket + '/object/X',
                             swob.HTTPOk, {}, None)
         self.swift.register('PUT', segment_bucket + '/object/X',
@@ -74,6 +84,32 @@ class TestSwift3MultiUpload(Swift3TestCase):
         self.swift.register('DELETE', segment_bucket + '/object/X/2',
                             swob.HTTPNoContent, {}, None)
 
+        self.swift.register('HEAD', segment_bucket + '/object/Y',
+                            swob.HTTPOk, {}, None)
+        self.swift.register('PUT', segment_bucket + '/object/Y',
+                            swob.HTTPCreated, {}, None)
+        self.swift.register('DELETE', segment_bucket + '/object/Y',
+                            swob.HTTPNoContent, {}, None)
+        self.swift.register('PUT', segment_bucket + '/object/Y/1',
+                            swob.HTTPCreated, {}, None)
+        self.swift.register('DELETE', segment_bucket + '/object/Y/1',
+                            swob.HTTPNoContent, {}, None)
+        self.swift.register('DELETE', segment_bucket + '/object/Y/2',
+                            swob.HTTPNoContent, {}, None)
+
+        self.swift.register('HEAD', segment_bucket + '/object2/Z',
+                            swob.HTTPOk, {}, None)
+        self.swift.register('PUT', segment_bucket + '/object2/Z',
+                            swob.HTTPCreated, {}, None)
+        self.swift.register('DELETE', segment_bucket + '/object2/Z',
+                            swob.HTTPNoContent, {}, None)
+        self.swift.register('PUT', segment_bucket + '/object2/Z/1',
+                            swob.HTTPCreated, {}, None)
+        self.swift.register('DELETE', segment_bucket + '/object2/Z/1',
+                            swob.HTTPNoContent, {}, None)
+        self.swift.register('DELETE', segment_bucket + '/object2/Z/2',
+                            swob.HTTPNoContent, {}, None)
+
     @s3acl
     def test_bucket_upload_part(self):
         req = Request.blank('/bucket?partNumber=1&uploadId=x',
@@ -122,15 +158,181 @@ class TestSwift3MultiUpload(Swift3TestCase):
         status, headers, body = self.call_swift3(req)
         self.assertEquals(self._get_error_code(body), 'InvalidRequest')
 
-    @s3acl
+    def _test_bucket_multipart_uploads_GET(self, query=None,
-    def test_bucket_multipart_uploads_GET(self):
+                                           multiparts=None):
-        req = Request.blank('/bucket/?uploads',
+        segment_bucket = '/v1/AUTH_test/bucket+segments'
+        objects = multiparts or multiparts_template
+        objects = map(lambda item: {'name': item[0], 'last_modified': item[1],
+                                    'hash': item[2], 'bytes': item[3]},
+                      objects)
+        object_list = json.dumps(objects)
+        self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
+                            object_list)
+
+        query = '?uploads&' + query if query else '?uploads'
+        req = Request.blank('/bucket/%s' % query,
                             environ={'REQUEST_METHOD': 'GET'},
                             headers={'Authorization': 'AWS test:tester:hmac'})
-        status, headers, body = self.call_swift3(req)
+        return self.call_swift3(req)
-        fromstring(body, 'ListMultipartUploadsResult')
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET(self):
+        status, headers, body = self._test_bucket_multipart_uploads_GET()
+        elem = fromstring(body, 'ListMultipartUploadsResult')
+        self.assertEquals(elem.find('Bucket').text, 'bucket')
+        self.assertEquals(elem.find('KeyMarker').text, None)
+        self.assertEquals(elem.find('UploadIdMarker').text, None)
+        self.assertEquals(elem.find('NextUploadIdMarker').text, 'Z')
+        self.assertEquals(elem.find('MaxUploads').text, '1000')
+        self.assertEquals(elem.find('IsTruncated').text, 'false')
+        self.assertEquals(len(elem.findall('Upload')), 3)
+        objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts_template]
+        for u in elem.findall('Upload'):
+            name = u.find('Key').text + '/' + u.find('UploadId').text
+            initiated = u.find('Initiated').text
+            self.assertTrue((name, initiated) in objects)
+            self.assertEquals(u.find('Initiator/ID').text, 'test:tester')
+            self.assertEquals(u.find('Initiator/DisplayName').text,
+                              'test:tester')
+            self.assertEquals(u.find('Owner/ID').text, 'test:tester')
+            self.assertEquals(u.find('Owner/DisplayName').text, 'test:tester')
+            self.assertEquals(u.find('StorageClass').text, 'STANDARD')
         self.assertEquals(status.split()[0], '200')
 
+    @s3acl
+    def test_bucket_multipart_uploads_GET_encoding_type_error(self):
+        query = 'encoding-type=xml'
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query)
+        self.assertEquals(self._get_error_code(body), 'InvalidArgument')
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET_maxuploads(self):
+        query = 'max-uploads=2'
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query)
+        elem = fromstring(body, 'ListMultipartUploadsResult')
+        self.assertEquals(len(elem.findall('Upload/UploadId')), 2)
+        self.assertEquals(elem.find('NextKeyMarker').text, 'object')
+        self.assertEquals(elem.find('NextUploadIdMarker').text, 'Y')
+        self.assertEquals(elem.find('MaxUploads').text, '2')
+        self.assertEquals(elem.find('IsTruncated').text, 'true')
+        self.assertEquals(status.split()[0], '200')
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET_str_maxuploads(self):
+        query = 'max-uploads=invalid'
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query)
+        self.assertEquals(self._get_error_code(body), 'InvalidArgument')
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET_negative_maxuploads(self):
+        query = 'max-uploads=-1'
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query)
+        self.assertEquals(self._get_error_code(body), 'InvalidArgument')
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET_maxuploads_over_default(self):
+        query = 'max-uploads=1001'
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query)
+        self.assertEquals(self._get_error_code(body), 'InvalidArgument')
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET_with_id_and_key_marker(self):
+        query = 'upload-id-marker=Y&key-marker=object'
+        multiparts = \
+            (('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2),
+             ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12),
+             ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22))
+
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query, multiparts)
+        elem = fromstring(body, 'ListMultipartUploadsResult')
+        self.assertEquals(elem.find('KeyMarker').text, 'object')
+        self.assertEquals(elem.find('UploadIdMarker').text, 'Y')
+        self.assertEquals(len(elem.findall('Upload')), 1)
+        objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts]
+        for u in elem.findall('Upload'):
+            name = u.find('Key').text + '/' + u.find('UploadId').text
+            initiated = u.find('Initiated').text
+            self.assertTrue((name, initiated) in objects)
+        self.assertEquals(status.split()[0], '200')
+
+        _, path, _ = self.swift.calls_with_headers[-1]
+        path, query_string = path.split('?', 1)
+        query = {}
+        for q in query_string.split('&'):
+            key, arg = q.split('=')
+            query[key] = arg
+        self.assertEquals(query['format'], 'json')
+        self.assertEquals(query['limit'], '1001')
+        self.assertEquals(query['marker'], 'object/Y')
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET_with_key_marker(self):
+        query = 'key-marker=object'
+        multiparts = \
+            (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1),
+             ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11),
+             ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21),
+             ('object/Y', '2014-05-07T19:47:53.592270', 'HASH', 2),
+             ('object/Y/1', '2014-05-07T19:47:54.592270', 'HASH', 12),
+             ('object/Y/2', '2014-05-07T19:47:55.592270', 'HASH', 22))
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query, multiparts)
+        elem = fromstring(body, 'ListMultipartUploadsResult')
+        self.assertEquals(elem.find('KeyMarker').text, 'object')
+        self.assertEquals(elem.find('NextKeyMarker').text, 'object')
+        self.assertEquals(elem.find('NextUploadIdMarker').text, 'Y')
+        self.assertEquals(len(elem.findall('Upload')), 2)
+        objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts]
+        for u in elem.findall('Upload'):
+            name = u.find('Key').text + '/' + u.find('UploadId').text
+            initiated = u.find('Initiated').text
+            self.assertTrue((name, initiated) in objects)
+        self.assertEquals(status.split()[0], '200')
+
+        _, path, _ = self.swift.calls_with_headers[-1]
+        path, query_string = path.split('?', 1)
+        query = {}
+        for q in query_string.split('&'):
+            key, arg = q.split('=')
+            query[key] = arg
+        self.assertEquals(query['format'], 'json')
+        self.assertEquals(query['limit'], '1001')
+        self.assertEquals(query['marker'], quote('object/~'))
+
+    @s3acl
+    def test_bucket_multipart_uploads_GET_with_prefix(self):
+        query = 'prefix=X'
+        multiparts = \
+            (('object/X', '2014-05-07T19:47:50.592270', 'HASH', 1),
+             ('object/X/1', '2014-05-07T19:47:51.592270', 'HASH', 11),
+             ('object/X/2', '2014-05-07T19:47:52.592270', 'HASH', 21))
+        status, headers, body = \
+            self._test_bucket_multipart_uploads_GET(query, multiparts)
+        elem = fromstring(body, 'ListMultipartUploadsResult')
+        self.assertEquals(len(elem.findall('Upload')), 1)
+        objects = [(o[0], o[1][:-3] + 'Z') for o in multiparts]
+        for u in elem.findall('Upload'):
+            name = u.find('Key').text + '/' + u.find('UploadId').text
+            initiated = u.find('Initiated').text
+            self.assertTrue((name, initiated) in objects)
+        self.assertEquals(status.split()[0], '200')
+
+        _, path, _ = self.swift.calls_with_headers[-1]
+        path, query_string = path.split('?', 1)
+        query = {}
+        for q in query_string.split('&'):
+            key, arg = q.split('=')
+            query[key] = arg
+        self.assertEquals(query['format'], 'json')
+        self.assertEquals(query['limit'], '1001')
+        self.assertEquals(query['prefix'], 'X')
+
     @s3acl
     @patch('swift3.controllers.multi_upload.unique_id', lambda: 'X')
     def test_object_multipart_upload_initiate(self):