openstack/swift
Code Issues Proposed changes

Merge "s3api: Allow PUT with if-none-match: *"

Browse Source
This commit is contained in:
Zuul
2025-05-19 18:32:06 +00:00
committed by Gerrit Code Review
parent f516065abd edd5eb29d7
commit 84a70769b1
8 changed files with 246 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
test/functional/s3api/test_multi_upload.py
View File

@@ -729,6 +729,66 @@ class TestS3ApiMultiUpload(S3ApiBase):
query=query)
self.assertEqual(get_error_code(body), 'InvalidPart')
def test_complete_multi_upload_conditional(self):
bucket = 'bucket'
key = 'obj'
self.conn.make_request('PUT', bucket)
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
query = 'partNumber=1&uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query)
part_etag = headers['etag']
xml = self._gen_comp_xml([part_etag])
for headers in [
{'If-Match': part_etag},
{'If-Match': '*'},
{'If-None-Match': part_etag},
{'If-Modified-Since': 'Wed, 21 Oct 2015 07:28:00 GMT'},
{'If-Unmodified-Since': 'Wed, 21 Oct 2015 07:28:00 GMT'},
]:
with self.subTest(headers=headers):
query = 'uploadId=%s' % upload_id
status, _, body = self.conn.make_request(
'POST', bucket, key, body=xml,
query=query, headers=headers)
self.assertEqual(status, 501)
self.assertEqual(get_error_code(body), 'NotImplemented')
# Can do basic existence checks, though
headers = {'If-None-Match': '*'}
query = 'uploadId=%s' % upload_id
status, _, body = self.conn.make_request(
'POST', bucket, key, body=xml,
query=query, headers=headers)
self.assertEqual(status, 200)
# And it'll prevent overwrites
query = 'uploads'
status, headers, body = \
self.conn.make_request('POST', bucket, key, query=query)
elem = fromstring(body, 'InitiateMultipartUploadResult')
upload_id = elem.find('UploadId').text
query = 'partNumber=1&uploadId=%s' % upload_id
status, headers, body = \
self.conn.make_request('PUT', bucket, key, query=query)
part_etag = headers['etag']
xml = self._gen_comp_xml([part_etag])
headers = {'If-None-Match': '*'}
query = 'uploadId=%s' % upload_id
status, _, body = self.conn.make_request(
'POST', bucket, key, body=xml,
query=query, headers=headers)
self.assertEqual(status, 412)
self.assertEqual(get_error_code(body), 'PreconditionFailed')
def test_complete_upload_min_segment_size(self):
bucket = 'bucket'
key = 'obj'

14
test/functional/s3api/test_object.py
View File

@@ -346,7 +346,7 @@ class TestS3ApiObject(S3ApiBase):
def test_put_object_conditional_requests(self):
obj = 'object'
content = b'abcdefghij'
headers = {'If-None-Match': '*'}
headers = {'If-None-Match': 'asdf'}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, headers, content)
self.assertEqual(status, 501)
@@ -371,6 +371,18 @@ class TestS3ApiObject(S3ApiBase):
self.conn.make_request('HEAD', self.bucket, obj, {}, '')
self.assertEqual(status, 404)
# But this will
headers = {'If-None-Match': '*'}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, headers, content)
self.assertEqual(status, 200)
# And the if-none-match prevents overwrites
headers = {'If-None-Match': '*'}
status, headers, body = \
self.conn.make_request('PUT', self.bucket, obj, headers, content)
self.assertEqual(status, 412)
def test_put_object_expect(self):
obj = 'object'
content = b'abcdefghij'

142
test/s3api/test_conditional_writes.py Normal file
View File

@@ -0,0 +1,142 @@
# Copyright (c) 2025 Nvidia
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from test.s3api import BaseS3TestCaseWithBucket, status_from_error, \
code_from_error
from botocore.exceptions import ClientError
class TestConditionalWrites(BaseS3TestCaseWithBucket):
def test_if_none_match_star_simple_put(self):
client = self.get_s3_client(1)
key_name = self.create_name('if-none-match-simple')
# Can create new object fine
resp = client.put_object(
Bucket=self.bucket_name,
Key=key_name,
IfNoneMatch='*',
Body=b'',
)
self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
# But overwrite is blocked
with self.assertRaises(ClientError) as caught:
client.put_object(
Bucket=self.bucket_name,
Key=key_name,
IfNoneMatch='*',
Body=b'',
)
self.assertEqual(412, status_from_error(caught.exception))
self.assertEqual('PreconditionFailed',
code_from_error(caught.exception))
def test_if_none_match_star_mpu(self):
client = self.get_s3_client(1)
key_name = self.create_name('if-none-match-mpu')
create_mpu_resp = client.create_multipart_upload(
Bucket=self.bucket_name, Key=key_name)
self.assertEqual(200, create_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
upload_id = create_mpu_resp['UploadId']
parts = []
for part_num in range(1, 4):
part_resp = client.upload_part(
Body=b'x' * 5 * 1024 * 1024,
Bucket=self.bucket_name, Key=key_name,
PartNumber=part_num, UploadId=upload_id)
self.assertEqual(200, part_resp[
'ResponseMetadata']['HTTPStatusCode'])
parts.append({
'ETag': part_resp['ETag'],
'PartNumber': part_num,
})
# Nothing there, so complete succeeds
complete_mpu_resp = client.complete_multipart_upload(
Bucket=self.bucket_name,
Key=key_name,
MultipartUpload={'Parts': parts[:2]},
UploadId=upload_id,
IfNoneMatch='*',
)
self.assertEqual(200, complete_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
# Retrying with more parts fails
with self.assertRaises(ClientError) as caught:
client.complete_multipart_upload(
Bucket=self.bucket_name,
Key=key_name,
MultipartUpload={'Parts': parts},
UploadId=upload_id,
IfNoneMatch='*',
)
self.assertEqual(404, status_from_error(caught.exception))
self.assertEqual('NoSuchUpload',
code_from_error(caught.exception))
# Ditto fewer
with self.assertRaises(ClientError) as caught:
client.complete_multipart_upload(
Bucket=self.bucket_name,
Key=key_name,
MultipartUpload={'Parts': parts[:1]},
UploadId=upload_id,
IfNoneMatch='*',
)
self.assertEqual(404, status_from_error(caught.exception))
self.assertEqual('NoSuchUpload',
code_from_error(caught.exception))
# Can retry with all the same parts and 200 though
complete_mpu_resp = client.complete_multipart_upload(
Bucket=self.bucket_name,
Key=key_name,
MultipartUpload={'Parts': parts[:2]},
UploadId=upload_id,
IfNoneMatch='*',
)
self.assertEqual(200, complete_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
# Can still start a new upload
create_mpu_resp = client.create_multipart_upload(
Bucket=self.bucket_name, Key=key_name)
self.assertEqual(200, create_mpu_resp[
'ResponseMetadata']['HTTPStatusCode'])
upload_id = create_mpu_resp['UploadId']
# And upload parts
part_resp = client.upload_part(
Body=b'', Bucket=self.bucket_name, Key=key_name,
PartNumber=1, UploadId=upload_id)
self.assertEqual(200, part_resp[
'ResponseMetadata']['HTTPStatusCode'])
parts = [{
'ETag': part_resp['ETag'],
'PartNumber': 1,
}]
# But completion will be blocked
with self.assertRaises(ClientError) as caught:
client.complete_multipart_upload(
Bucket=self.bucket_name,
Key=key_name,
MultipartUpload={'Parts': parts},
UploadId=upload_id,
IfNoneMatch='*',
)
self.assertEqual(412, status_from_error(caught.exception))
self.assertEqual('PreconditionFailed',
code_from_error(caught.exception))

22
test/unit/common/middleware/s3api/test_multi_upload.py
View File

@@ -1599,10 +1599,9 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase):
'Content-MD5': content_md5, },
body=XML)
status, headers, body = self.call_s3api(req)
elem = fromstring(body, 'CompleteMultipartUploadResult')
self.assertNotIn('Etag', headers)
self.assertEqual(elem.find('ETag').text, S3_ETAG)
self.assertEqual(status.split()[0], '200')
elem = fromstring(body, 'Error')
self.assertEqual(elem.find('Code').text, 'NoSuchUpload')
self.assertEqual(status.split()[0], '404')
self.assertEqual(self.swift.calls, [
# Bucket exists
@@ -1612,24 +1611,11 @@ class TestS3ApiMultiUpload(BaseS3ApiMultiUpload, S3ApiTestCase):
('HEAD', '/v1/AUTH_test/bucket+segments/object/X'),
# But the object does, and with the same upload ID
('HEAD', '/v1/AUTH_test/bucket/object'),
# Create the SLO
('PUT', '/v1/AUTH_test/bucket/object'
'?heartbeat=on&multipart-manifest=put'),
# Retry deleting the marker for the sake of completeness
('DELETE', '/v1/AUTH_test/bucket+segments/object/X')
# And then we bail
])
self.assertEqual(req.environ['swift.backend_path'],
'/v1/AUTH_test/bucket+segments/object/X')
_, _, headers = self.swift.calls_with_headers[-2]
self.assertEqual(headers.get('X-Object-Meta-Foo'), 'bar')
self.assertEqual(headers.get('Content-Type'), 'baz/quux')
# SLO will provide a base value
override_etag = '; s3_etag=%s' % S3_ETAG.strip('"')
h = 'X-Object-Sysmeta-Container-Update-Override-Etag'
self.assertEqual(headers.get(h), override_etag)
self.assertEqual(headers.get('X-Object-Sysmeta-S3Api-Upload-Id'), 'X')
def test_object_multipart_upload_retry_complete_upload_id_mismatch(self):
content_md5 = base64.b64encode(md5(
XML.encode('ascii'), usedforsecurity=False).digest())