42b4cdc538
Ordinary objects in S3 use the MD5 of the object as the ETag, just like Swift. Multipart Uploads follow a different format, notably including a dash followed by the number of segments. Several clients use this difference to change their behavior based upon the presence of a dash in object ETags, not only during object download but during upload and listing, too. In particular, this can disable upload/download integrity checks or cause the client to issue HEAD requests to see whether the MD5 was stored in metadata on the object. The goal of this patch is to get as many of the benefits of the dash as we can, even for data that was uploaded via the Swift API or that predated the related-changes. To that end (and for S3 API requests *only*), look for any indication that an object may be an SLO and tack on a literal '-N' to the end of the ETag. Why 'N'? Two main reasons: - We don't necessarily know how many segments there are, and don't want to use additional backend requests to find out (particularly when it would require *multiple* additional requests as in a bucket listing). - We don't want to provide an arbitrary number (as ProxyFS does [1]) because it would look *too much* like an S3 ETag, and if any client actually cares about the *exact* ETag generation algorithm, I'd prefer to have a way to distinguish between the two. This modified ETag will be used in key GET/HEAD responses via the S3 API, where SLOs are always indicated with a X-Static-Large-Object header. Either the modified or original ETag may be used for conditional requests via the S3 API. Bucket listings via the S3 API *may* present the modified ETag, but only if the JSON container listing includes an 'slo_etag' key for the object; see the related SLO patch for when that started happening. There should be no impact for the Swift API. [1] https://github.com/swiftstack/ProxyFS/blob/1.6.4/pfs_middleware/pfs_middleware/middleware.py#L443-L455 Change-Id: If4c47d7b13dcb4b3d04c52bb08b15ca2069cd15c Related-Change: Ibe68c44bef6c17605863e9084503e8f5dc577fab Related-Change: I67478923619b00ec1a37d56b6fec6a218453dafc
86 lines
3.9 KiB
Python
86 lines
3.9 KiB
Python
# Copyright (c) 2014 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import unittest
|
|
|
|
from swift.common.swob import Response
|
|
from swift.common.utils import HeaderKeyDict
|
|
from swift.common.middleware.s3api.s3response import S3Response
|
|
from swift.common.middleware.s3api.utils import sysmeta_prefix
|
|
|
|
|
|
class TestResponse(unittest.TestCase):
|
|
def test_from_swift_resp_slo(self):
|
|
for expected, header_vals in \
|
|
((True, ('true', '1')), (False, ('false', 'ugahhh', None))):
|
|
for val in header_vals:
|
|
resp = Response(headers={'X-Static-Large-Object': val,
|
|
'Etag': 'theetag'})
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
self.assertEqual(expected, s3resp.is_slo)
|
|
if s3resp.is_slo:
|
|
self.assertEqual('"theetag-N"', s3resp.headers['ETag'])
|
|
else:
|
|
self.assertEqual('"theetag"', s3resp.headers['ETag'])
|
|
|
|
def test_response_s3api_sysmeta_headers(self):
|
|
for _server_type in ('object', 'container'):
|
|
swift_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
self.assertEqual(swift_headers, s3resp.sysmeta_headers)
|
|
|
|
def test_response_s3api_sysmeta_headers_ignore_other_sysmeta(self):
|
|
for _server_type in ('object', 'container'):
|
|
swift_headers = HeaderKeyDict(
|
|
# sysmeta not leading sysmeta_prefix even including s3api word
|
|
{'x-%s-sysmeta-test-s3api' % _server_type: 'ok',
|
|
sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
expected_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
|
|
|
def test_response_s3api_sysmeta_from_swift3_sysmeta(self):
|
|
for _server_type in ('object', 'container'):
|
|
# swift could return older swift3 sysmeta
|
|
swift_headers = HeaderKeyDict(
|
|
{('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
expected_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
# but Response class should translates as s3api sysmeta
|
|
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
|
|
|
def test_response_swift3_sysmeta_does_not_overwrite_s3api_sysmeta(self):
|
|
for _server_type in ('object', 'container'):
|
|
# same key name except sysmeta prefix
|
|
swift_headers = HeaderKeyDict(
|
|
{('x-%s-sysmeta-swift3-' % _server_type) + 'test': 'ng',
|
|
sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
resp = Response(headers=swift_headers)
|
|
s3resp = S3Response.from_swift_resp(resp)
|
|
expected_headers = HeaderKeyDict(
|
|
{sysmeta_prefix(_server_type) + 'test': 'ok'})
|
|
# but only s3api sysmeta remains in the response sysmeta_headers
|
|
self.assertEqual(expected_headers, s3resp.sysmeta_headers)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|