s3api: Stop requiring Content-MD5 for multi-deletes

...at least, provided the client sent a X-Amz-Content-SHA256 header.
Apparently Content-MD5 is no longer strictly required by AWS? Or maybe
it never was, provided the client sent a SHA256 of the content.

This also allows us to test with newer boto3, botocore, s3transfer.

Related-Bug: #2098529
Co-Authored-By: Clay Gerrard <clay.gerrard@gmail.com>
Change-Id: Ifbcde9820bee72d80cab0fe3e67ea0f5817df949

py3-constraints.txt

@@ -17,10 +17,10 @@ autopage===0.5.2
 bandit===1.7.10;python_version>='3.8'
 bandit===1.7.5;python_version=='3.7'
 bandit===1.7.1;python_version=='3.6'
-boto3===1.35.71;python_version>='3.8'
+boto3===1.36.6;python_version>='3.8'
 boto3===1.33.13;python_version=='3.7'
 boto3===1.23.10;python_version=='3.6'
-botocore===1.35.71;python_version>='3.8'
+botocore===1.36.6;python_version>='3.8'
 botocore===1.33.13;python_version=='3.7'
 botocore===1.26.10;python_version=='3.6'
 certifi===2024.8.30
@@ -188,7 +188,7 @@ rfc3986===2.0.0;python_version>='3.7'
 rfc3986===1.5.0;python_version=='3.6'
 rich===13.9.3;python_version>='3.8'
 rich===13.8.1;python_version=='3.7'
-s3transfer===0.10.4;python_version>='3.8'
+s3transfer===0.11.2;python_version>='3.8'
 s3transfer===0.8.2;python_version=='3.7'
 s3transfer===0.5.2;python_version=='3.6'
 setuptools===75.3.0;python_version>='3.12'

swift/common/middleware/s3api/controllers/multi_delete.py

@@ -77,6 +77,11 @@ class MultiObjectDeleteController(Controller):
             if not xml:
                 raise MissingRequestBodyError()
 
+            if 'x-amz-content-sha256' not in req.headers:
+                # SHA256 got checked when we read the body, so there's at
+                # least *some* verification. Recent versions of boto3 stopped
+                # sending Content-MD5, so it can't *always* be required.
+                # See https://bugs.launchpad.net/swift/+bug/2098529
                 req.check_md5(xml)
             elem = fromstring(xml, 'Delete', self.logger)
 

test/unit/common/middleware/s3api/test_multi_delete.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import base64
+import hashlib
 import json
 import unittest
 from datetime import datetime
@@ -62,6 +63,87 @@ class BaseS3ApiMultiDelete(object):
         status, headers, body = self.call_s3api(req)
         self.assertEqual(status.split()[0], '200')
 
+    def test_object_multi_DELETE_no_content_md5(self):
+        elem = Element('Delete')
+        obj = SubElement(elem, 'Object')
+        SubElement(obj, 'Key').text = 'object'
+        body = tostring(elem, use_s3ns=False)
+
+        req = Request.blank('/bucket/object?delete',
+                            environ={'REQUEST_METHOD': 'POST'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     },
+                            body=body)
+
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '400')
+        self.assertEqual(self._get_error_code(body), 'InvalidRequest')
+        self.assertIn(b'Missing required header', body)
+        self.assertIn(b'Content-MD5', body)
+
+    def test_object_multi_DELETE_sha256_invalid(self):
+        elem = Element('Delete')
+        obj = SubElement(elem, 'Object')
+        SubElement(obj, 'Key').text = 'object'
+        body = tostring(elem, use_s3ns=False)
+        content_sha256 = 'invalid'
+
+        req = Request.blank('/bucket/object?delete',
+                            environ={'REQUEST_METHOD': 'POST'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-Amz-Content-SHA256': content_sha256,
+                                     },
+                            body=body)
+
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '400')
+        self.assertEqual(self._get_error_code(body),
+                         'XAmzContentSHA256Mismatch')
+        self.assertIn(b"provided 'x-amz-content-sha256' header "
+                      b"does not match", body)
+
+    def test_object_multi_DELETE_sha256_bad(self):
+        elem = Element('Delete')
+        obj = SubElement(elem, 'Object')
+        SubElement(obj, 'Key').text = 'object'
+        body = tostring(elem, use_s3ns=False)
+        content_sha256 = hashlib.sha256(body[:-1]).hexdigest()
+
+        req = Request.blank('/bucket/object?delete',
+                            environ={'REQUEST_METHOD': 'POST'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-Amz-Content-SHA256': content_sha256,
+                                     },
+                            body=body)
+
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '400')
+        self.assertEqual(self._get_error_code(body),
+                         'XAmzContentSHA256Mismatch')
+        self.assertIn(b"provided 'x-amz-content-sha256' header "
+                      b"does not match", body)
+
+    def test_object_multi_DELETE_sha256_valid(self):
+        elem = Element('Delete')
+        obj = SubElement(elem, 'Object')
+        SubElement(obj, 'Key').text = 'object'
+        body = tostring(elem, use_s3ns=False)
+        content_sha256 = hashlib.sha256(body).hexdigest()
+
+        req = Request.blank('/bucket/object?delete',
+                            environ={'REQUEST_METHOD': 'POST'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header(),
+                                     'X-Amz-Content-SHA256': content_sha256,
+                                     },
+                            body=body)
+
+        status, headers, body = self.call_s3api(req)
+        self.assertEqual(status.split()[0], '200')
+
     def test_object_multi_DELETE(self):
         self.swift.register('DELETE', '/v1/AUTH_test/bucket/Key1',
                             swob.HTTPNoContent, {}, None)