From 33b17742a727e3bd15072b61a0f119191423f6b7 Mon Sep 17 00:00:00 2001 From: Jianjian Huo Date: Mon, 19 May 2025 16:15:12 -0700 Subject: [PATCH] s3api: more test cases for conditional writes. Change-Id: Id5583e3a1e4515ec3c8a972f647aaaabfba673bc Related-Change: I2e57dacb342b5758f16b502bb91372a2443d0182 --- .../middleware/s3api/test_multi_upload.py | 145 +++++++++++++++++- .../common/middleware/s3api/test_s3request.py | 51 +++++- 2 files changed, 189 insertions(+), 7 deletions(-) diff --git a/test/unit/common/middleware/s3api/test_multi_upload.py b/test/unit/common/middleware/s3api/test_multi_upload.py index 128087b0f1..3a4619e940 100644 --- a/test/unit/common/middleware/s3api/test_multi_upload.py +++ b/test/unit/common/middleware/s3api/test_multi_upload.py @@ -1398,14 +1398,16 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): def _do_test_object_multipart_upload_complete( self, bucket_policy_index=int(POLICIES.default), - segment_bucket_policy_index=None): + segment_bucket_policy_index=None, extra_headers=None): + extra_headers = extra_headers or {} content_md5 = base64.b64encode(md5( XML.encode('ascii'), usedforsecurity=False).digest()) req = Request.blank('/bucket/object?uploadId=X', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), - 'Content-MD5': content_md5, }, + 'Content-MD5': content_md5, + **extra_headers}, body=XML) if segment_bucket_policy_index is None: segment_bucket_policy_index = bucket_policy_index @@ -1452,11 +1454,69 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): def test_object_multipart_upload_complete(self): self._do_test_object_multipart_upload_complete() + def test_object_multipart_upload_complete_with_if_none_match_star(self): + self._do_test_object_multipart_upload_complete( + extra_headers={'If-None-Match': '*'} + ) + def test_object_multipart_upload_complete_mixed_policy(self): self._do_test_object_multipart_upload_complete( bucket_policy_index=0, segment_bucket_policy_index=1 ) + def test_mpu_complete_mixed_policy_with_if_none_match_star(self): + self._do_test_object_multipart_upload_complete( + bucket_policy_index=0, segment_bucket_policy_index=1, + extra_headers={'If-None-Match': '*'} + ) + + def _do_test_object_multipart_upload_complete_501(self, extra_headers): + bucket_policy_index = int(POLICIES.default) + segment_bucket_policy_index = bucket_policy_index + content_md5 = base64.b64encode(md5( + XML.encode('ascii'), usedforsecurity=False).digest()) + req = Request.blank('/bucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-MD5': content_md5, + **extra_headers}, + body=XML) + self._register_bucket_policy_index_head('bucket', bucket_policy_index) + self._register_bucket_policy_index_head('bucket+segments', + segment_bucket_policy_index) + status, headers, body = self.call_s3api(req) + elem = fromstring(body, 'Error') + self.assertEqual(elem.find('Code').text, 'NotImplemented') + self.assertEqual(status.split()[0], '501') + + self.assertEqual(self.swift.calls, [ + # Bucket exists + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + # And then we bail because of the 501 + ]) + + def test_object_multipart_upload_complete_with_if_match(self): + self._do_test_object_multipart_upload_complete_501( + extra_headers={'If-Match': 'not-the-etag'} + ) + + def test_object_multipart_upload_complete_with_if_none_match(self): + self._do_test_object_multipart_upload_complete_501( + extra_headers={'If-None-Match': 'not-the-etag'} + ) + + def test_object_multipart_upload_complete_with_if_modified_since(self): + self._do_test_object_multipart_upload_complete_501( + extra_headers={'If-Modified-Since': 'not-the-etag'} + ) + + def test_object_multipart_upload_complete_with_if_unmodified_since(self): + self._do_test_object_multipart_upload_complete_501( + extra_headers={'If-Unmodified-Since': 'not-the-etag'} + ) + def test_object_multipart_upload_complete_other_headers(self): headers = {'x-object-meta-foo': 'bar', 'content-type': 'application/directory', @@ -1527,7 +1587,8 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): def _do_test_object_multipart_upload_retry_complete( self, bucket_policy_index=int(POLICIES.default), - segment_bucket_policy_index=None): + segment_bucket_policy_index=None, extra_headers=None): + extra_headers = extra_headers or {} if segment_bucket_policy_index is None: segment_bucket_policy_index = bucket_policy_index self._register_bucket_policy_index_head('bucket', bucket_policy_index) @@ -1549,7 +1610,8 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), - 'Content-MD5': content_md5, }, + 'Content-MD5': content_md5, + **extra_headers}, body=XML) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'CompleteMultipartUploadResult') @@ -1575,11 +1637,22 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): def test_object_multipart_upload_retry_complete(self): self._do_test_object_multipart_upload_retry_complete() + def test_mpu_retry_complete_with_if_none_match_star(self): + self._do_test_object_multipart_upload_retry_complete( + extra_headers={'If-None-Match': '*'}) + def test_object_multipart_upload_retry_complete_mixed_policy(self): self._do_test_object_multipart_upload_retry_complete( bucket_policy_index=0, segment_bucket_policy_index=1) - def test_object_multipart_upload_retry_complete_etag_mismatch(self): + def test_mpu_retry_complete_mixed_policy_with_if_none_match_star(self): + self._do_test_object_multipart_upload_retry_complete( + bucket_policy_index=0, segment_bucket_policy_index=1, + extra_headers={'If-None-Match': '*'}) + + def _do_object_multipart_upload_retry_complete_etag_mismatch( + self, extra_headers=None): + extra_headers = extra_headers or {} content_md5 = base64.b64encode(md5( XML.encode('ascii'), usedforsecurity=False).digest()) self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object/X', @@ -1596,7 +1669,8 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac', 'Date': self.get_date_header(), - 'Content-MD5': content_md5, }, + 'Content-MD5': content_md5, + **extra_headers}, body=XML) status, headers, body = self.call_s3api(req) elem = fromstring(body, 'Error') @@ -1616,6 +1690,65 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase): self.assertEqual(req.environ['swift.backend_path'], '/v1/AUTH_test/bucket+segments/object/X') + def test_object_multipart_upload_retry_complete_etag_mismatch(self): + self._do_object_multipart_upload_retry_complete_etag_mismatch() + + def test_object_multipart_upload_retry_complete_with_if_none_match_star( + self): + self._do_object_multipart_upload_retry_complete_etag_mismatch( + {'If-None-Match': '*'}) + + def _do_object_multipart_upload_retry_complete_etag_mismatch_501( + self, extra_headers): + content_md5 = base64.b64encode(md5( + XML.encode('ascii'), usedforsecurity=False).digest()) + self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object/X', + swob.HTTPNotFound, {}, None) + recent_ts = S3Timestamp.now(delta=-1000000).internal + self.swift.register('HEAD', '/v1/AUTH_test/bucket/object', + swob.HTTPOk, + {'x-object-meta-foo': 'bar', + 'content-type': 'baz/quux', + 'x-object-sysmeta-s3api-upload-id': 'X', + 'x-object-sysmeta-s3api-etag': 'not-the-etag', + 'x-timestamp': recent_ts}, None) + req = Request.blank('/bucket/object?uploadId=X', + environ={'REQUEST_METHOD': 'POST'}, + headers={'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header(), + 'Content-MD5': content_md5, + **extra_headers}, + body=XML) + status, headers, body = self.call_s3api(req) + elem = fromstring(body, 'Error') + self.assertEqual(elem.find('Code').text, 'NotImplemented') + self.assertEqual(status.split()[0], '501') + + self.assertEqual(self.swift.calls, [ + # Bucket exists + ('HEAD', '/v1/AUTH_test'), + ('HEAD', '/v1/AUTH_test/bucket'), + # And then we bail because of the 501 + ]) + + def test_object_multipart_upload_retry_complete_with_if_match(self): + self._do_object_multipart_upload_retry_complete_etag_mismatch_501( + {'If-Match': 'not-the-etag'}) + + def test_object_multipart_upload_retry_complete_with_if_none_match(self): + self._do_object_multipart_upload_retry_complete_etag_mismatch_501( + {'If-None-Match': 'not-the-etag'}) + + def test_object_multipart_upload_retry_complete_with_if_modified_since( + self): + self._do_object_multipart_upload_retry_complete_etag_mismatch_501( + {'If-Modified-Since': 'not-the-etag'}) + + def test_object_multipart_upload_retry_complete_with_if_unmodified_since( + self): + self._do_object_multipart_upload_retry_complete_etag_mismatch_501( + {'If-Unmodified-Since': 'not-the-etag'}) + def test_object_multipart_upload_retry_complete_upload_id_mismatch(self): content_md5 = base64.b64encode(md5( XML.encode('ascii'), usedforsecurity=False).digest()) diff --git a/test/unit/common/middleware/s3api/test_s3request.py b/test/unit/common/middleware/s3api/test_s3request.py index 79912cf307..53cd5f9812 100644 --- a/test/unit/common/middleware/s3api/test_s3request.py +++ b/test/unit/common/middleware/s3api/test_s3request.py @@ -37,7 +37,7 @@ from swift.common.middleware.s3api.s3response import InvalidArgument, \ NoSuchBucket, InternalError, ServiceUnavailable, \ AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed, \ InvalidPartArgument, InvalidPartNumber, InvalidRequest, \ - XAmzContentSHA256Mismatch + XAmzContentSHA256Mismatch, ErrorResponse from test.debug_logger import debug_logger @@ -393,6 +393,55 @@ class TestRequest(S3ApiTestCase): self.assertEqual(status.split()[0], '403') self.assertEqual(body, b'') + def test_put_object_if_none_match(self): + req = Request.blank('/bucket/object', method='PUT', + headers={'If-None-Match': 'asdf', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with self.assertRaises(s3response.S3NotImplemented) as cm: + S3Request(req.environ, self.s3api.conf)._validate_headers() + self.assertIn('501 Not Implemented', str(cm.exception)) + + def test_put_object_if_match(self): + req = Request.blank('/bucket/object', method='PUT', + headers={'If-Match': '*', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with self.assertRaises(s3response.S3NotImplemented) as cm: + S3Request(req.environ, self.s3api.conf)._validate_headers() + self.assertIn('501 Not Implemented', str(cm.exception)) + + def test_put_object_if_modified_since(self): + req = Request.blank('/bucket/object', method='PUT', + headers={'If-Modified-Since': + 'Sat, 27 Jun 2015 00:00:00 GMT', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with self.assertRaises(s3response.S3NotImplemented) as cm: + S3Request(req.environ, self.s3api.conf)._validate_headers() + self.assertIn('501 Not Implemented', str(cm.exception)) + + def test_put_object_if_unmodified_since(self): + req = Request.blank('/bucket/object', method='PUT', + headers={'If-Unmodified-Since': + 'Sat, 27 Jun 2015 00:00:00 GMT', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + with self.assertRaises(s3response.S3NotImplemented) as cm: + S3Request(req.environ, self.s3api.conf)._validate_headers() + self.assertIn('501 Not Implemented', str(cm.exception)) + + def test_put_object_if_none_match_star(self): + # This is the only allowed case, should NOT raise + req = Request.blank('/bucket/object', method='PUT', + headers={'If-None-Match': '*', + 'Authorization': 'AWS test:tester:hmac', + 'Date': self.get_date_header()}) + try: + S3Request(req.environ, self.s3api.conf)._validate_headers() + except ErrorResponse as err: + self.fail('Unexpected exception raised: %s' % err) + def _test_request_timestamp_sigv4(self, date_header): # signature v4 here environ = {