Merge "s3api: Increase max body size for Delete Multiple Objects requests"

swift/common/middleware/s3api/controllers/multi_delete.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from swift.common.constraints import MAX_OBJECT_NAME_LENGTH
 from swift.common.utils import public
 
 from swift.common.middleware.s3api.controllers.base import Controller, \
@@ -23,8 +24,6 @@ from swift.common.middleware.s3api.s3response import HTTPOk, S3NotImplemented, \
     NoSuchKey, ErrorResponse, MalformedXML, UserKeyMustBeSpecified, \
     AccessDenied, MissingRequestBodyError
 
-MAX_MULTI_DELETE_BODY_SIZE = 61365
-
 
 class MultiObjectDeleteController(Controller):
     """
@@ -61,8 +60,16 @@ class MultiObjectDeleteController(Controller):
 
                 yield key, version
 
+        max_body_size = min(
+            # FWIW, AWS limits multideletes to 1000 keys, and swift limits
+            # object names to 1024 bytes (by default). Add a factor of two to
+            # allow some slop.
+            2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
+            # But, don't let operators shoot themselves in the foot
+            10 * 1024 * 1024)
+
         try:
-            xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE)
+            xml = req.xml(max_body_size)
             if not xml:
                 raise MissingRequestBodyError()
 

test/functional/s3api/test_multi_delete.py

@@ -18,8 +18,6 @@ import os
 import test.functional as tf
 from swift.common.middleware.s3api.etree import fromstring, tostring, Element, \
     SubElement
-from swift.common.middleware.s3api.controllers.multi_delete import \
-    MAX_MULTI_DELETE_BODY_SIZE
 
 from test.functional.s3api import S3ApiBase
 from test.functional.s3api.s3_test_client import Connection
@@ -172,11 +170,12 @@ class TestS3ApiMultiDelete(S3ApiBase):
                                    query=query)
         self.assertEqual(get_error_code(body), 'UserKeyMustBeSpecified')
 
+        max_deletes = tf.cluster_info.get('s3api', {}).get(
+            'max_multi_delete_objects', 1000)
         # specified number of objects are over max_multi_delete_objects
-        # (Default 1000), but xml size is smaller than 61365 bytes.
-        req_objects = ['obj%s' for var in xrange(1001)]
+        # (Default 1000), but xml size is relatively small
+        req_objects = ['obj%s' for var in xrange(max_deletes + 1)]
         xml = self._gen_multi_delete_xml(req_objects)
-        self.assertTrue(len(xml.encode('utf-8')) <= MAX_MULTI_DELETE_BODY_SIZE)
         content_md5 = calculate_md5(xml)
         status, headers, body = \
             self.conn.make_request('POST', bucket, body=xml,
@@ -184,12 +183,11 @@ class TestS3ApiMultiDelete(S3ApiBase):
                                    query=query)
         self.assertEqual(get_error_code(body), 'MalformedXML')
 
-        # specified xml size is over 61365 bytes, but number of objects are
+        # specified xml size is large, but number of objects are
         # smaller than max_multi_delete_objects.
-        obj = 'a' * 1024
-        req_objects = [obj + str(var) for var in xrange(999)]
+        obj = 'a' * 102400
+        req_objects = [obj + str(var) for var in xrange(max_deletes - 1)]
         xml = self._gen_multi_delete_xml(req_objects)
-        self.assertTrue(len(xml.encode('utf-8')) > MAX_MULTI_DELETE_BODY_SIZE)
         content_md5 = calculate_md5(xml)
         status, headers, body = \
             self.conn.make_request('POST', bucket, body=xml,

test/unit/common/middleware/s3api/test_multi_delete.py

@@ -179,12 +179,37 @@ class TestS3ApiMultiDelete(S3ApiTestCase):
         status, headers, body = self.call_s3api(req)
         self.assertEqual(self._get_error_code(body), 'InvalidRequest')
 
+    @s3acl
+    def test_object_multi_DELETE_lots_of_keys(self):
+        elem = Element('Delete')
+        for i in range(self.conf.max_multi_delete_objects):
+            name = 'x' * 1000 + str(i)
+            self.swift.register('HEAD', '/v1/AUTH_test/bucket/%s' % name,
+                                swob.HTTPNotFound, {}, None)
+            obj = SubElement(elem, 'Object')
+            SubElement(obj, 'Key').text = name
+        body = tostring(elem, use_s3ns=False)
+        content_md5 = md5(body).digest().encode('base64').strip()
+
+        req = Request.blank('/bucket?delete',
+                            environ={'REQUEST_METHOD': 'POST'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'Content-MD5': content_md5},
+                            body=body)
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual('200 OK', status)
+
+        elem = fromstring(body)
+        self.assertEqual(len(elem.findall('Deleted')),
+                         self.conf.max_multi_delete_objects)
+
     @s3acl
     def test_object_multi_DELETE_too_many_keys(self):
         elem = Element('Delete')
         for i in range(self.conf.max_multi_delete_objects + 1):
             obj = SubElement(elem, 'Object')
-            SubElement(obj, 'Key').text = str(i)
+            SubElement(obj, 'Key').text = 'x' * 1000 + str(i)
         body = tostring(elem, use_s3ns=False)
         content_md5 = md5(body).digest().encode('base64').strip()