Ignore md5sum header during multi-part upload init

Some S3 client libs send an Etag/Content-Md5 header during multi-part object
initialization.  The S3 API reference does not mention how the MD5 checksum
header is treated at this stage, and the API itself appears to ignore the
headers.

Prior to this commit, swift3 passed the headers on, which were later compared
to the md5sum of the request's body, which is always empty.  This results in the
upload failing when the client-supplied checksum (generally the checksum for the
entire object) does not match the checksum for a null object.

After this commit, the Etag and Content-Md5 headers are ignored during the
multi-part initialization phase.  This mimics the behavior of AWS' S3 API.

Closes-Bug: 1697741
Change-Id: I2cb5376994bf270890bd9b06ec2bf521350c826d

swift3/controllers/multi_upload.py

@@ -340,6 +340,9 @@ class UploadsController(Controller):
 
         obj = '%s/%s' % (req.object_name, upload_id)
 
+        req.headers.pop('Etag', None)
+        req.headers.pop('Content-Md5', None)
+
         req.get_response(self.app, 'PUT', container, obj, body='')
 
         result_elem = Element('InitiateMultipartUploadResult')

swift3/test/functional/test_multi_upload.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import unittest
 import os
 import boto
@@ -22,7 +23,7 @@ import boto
 from distutils.version import StrictVersion
 
 from hashlib import md5
-from itertools import izip
+from itertools import izip, izip_longest
 
 from swift3.cfg import CONF
 from swift3.test.functional.utils import get_error_code, get_error_msg
@@ -47,13 +48,16 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
         return tostring(elem)
 
     def _initiate_multi_uploads_result_generator(self, bucket, keys,
-                                                 trials=1):
+                                                 headers=None, trials=1):
+        if headers is None:
+            headers = [None] * len(keys)
         self.conn.make_request('PUT', bucket)
         query = 'uploads'
-        for key in keys:
+        for key, key_headers in izip_longest(keys, headers):
             for i in xrange(trials):
                 status, resp_headers, body = \
-                    self.conn.make_request('POST', bucket, key, query=query)
+                    self.conn.make_request('POST', bucket, key,
+                                           headers=key_headers, query=query)
                 yield status, resp_headers, body
 
     def _upload_part(self, bucket, key, upload_id, content=None, part_num=1):
@@ -89,11 +93,14 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
 
     def test_object_multi_upload(self):
         bucket = 'bucket'
-        keys = ['obj1', 'obj2']
+        keys = ['obj1', 'obj2', 'obj3']
+        headers = [None,
+                   {'Content-MD5': base64.b64encode('a' * 16).strip()},
+                   {'Etag': 'nonsense'}]
         uploads = []
 
         results_generator = self._initiate_multi_uploads_result_generator(
-            bucket, keys)
+            bucket, keys, headers=headers)
 
         # Initiate Multipart Upload
         for expected_key, (status, headers, body) in \
@@ -134,7 +141,7 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
         self.assertEqual(elem.find('MaxUploads').text, '1000')
         self.assertTrue(elem.find('EncodingType') is None)
         self.assertEqual(elem.find('IsTruncated').text, 'false')
-        self.assertEqual(len(elem.findall('Upload')), 2)
+        self.assertEqual(len(elem.findall('Upload')), 3)
         for (expected_key, expected_upload_id), u in \
                 izip(uploads, elem.findall('Upload')):
             key = u.find('Key').text
@@ -259,17 +266,19 @@ class TestSwift3MultiUpload(Swift3FunctionalTestCase):
             self.assertEqual(MIN_SEGMENT_SIZE, int(p.find('Size').text))
             etags.append(p.find('ETag').text)
 
-        # Abort Multipart Upload
-        key, upload_id = uploads[1]
-        query = 'uploadId=%s' % upload_id
-        status, headers, body = \
-            self.conn.make_request('DELETE', bucket, key, query=query)
-        self.assertEqual(status, 204)
-        self.assertCommonResponseHeaders(headers)
-        self.assertTrue('content-type' in headers)
-        self.assertEqual(headers['content-type'], 'text/html; charset=UTF-8')
-        self.assertTrue('content-length' in headers)
-        self.assertEqual(headers['content-length'], '0')
+        # Abort Multipart Uploads
+        # note that uploads[1] has part data while uploads[2] does not
+        for key, upload_id in uploads[1:]:
+            query = 'uploadId=%s' % upload_id
+            status, headers, body = \
+                self.conn.make_request('DELETE', bucket, key, query=query)
+            self.assertEqual(status, 204)
+            self.assertCommonResponseHeaders(headers)
+            self.assertTrue('content-type' in headers)
+            self.assertEqual(headers['content-type'],
+                             'text/html; charset=UTF-8')
+            self.assertTrue('content-length' in headers)
+            self.assertEqual(headers['content-length'], '0')
 
         # Complete Multipart Upload
         key, upload_id = uploads[0]

swift3/test/unit/test_multi_upload.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
 import os
 import time
 import unittest
@@ -547,19 +548,29 @@ class TestSwift3MultiUpload(Swift3TestCase):
         self.assertTrue(query.get('delimiter') is None)
 
     @patch('swift3.controllers.multi_upload.unique_id', lambda: 'X')
-    def test_object_multipart_upload_initiate(self):
+    def _test_object_multipart_upload_initiate(self, headers):
+        headers.update({
+            'Authorization': 'AWS test:tester:hmac',
+            'Date': self.get_date_header(),
+            'x-amz-meta-foo': 'bar',
+        })
         req = Request.blank('/bucket/object?uploads',
                             environ={'REQUEST_METHOD': 'POST'},
-                            headers={'Authorization':
-                                     'AWS test:tester:hmac',
-                                     'Date': self.get_date_header(),
-                                     'x-amz-meta-foo': 'bar'})
+                            headers=headers)
         status, headers, body = self.call_swift3(req)
         fromstring(body, 'InitiateMultipartUploadResult')
         self.assertEqual(status.split()[0], '200')
 
         _, _, req_headers = self.swift.calls_with_headers[-1]
         self.assertEqual(req_headers.get('X-Object-Meta-Foo'), 'bar')
+        self.assertNotIn('Etag', req_headers)
+        self.assertNotIn('Content-MD5', req_headers)
+
+    def test_object_multipart_upload_initiate(self):
+        self._test_object_multipart_upload_initiate({})
+        self._test_object_multipart_upload_initiate({'Etag': 'blahblahblah'})
+        self._test_object_multipart_upload_initiate({
+            'Content-MD5': base64.b64encode('blahblahblahblah').strip()})
 
     @s3acl(s3acl_only=True)
     @patch('swift3.controllers.multi_upload.unique_id', lambda: 'X')