diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index c996a5ee74..b4f662acb5 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -225,8 +225,14 @@ class BucketController(Controller): if 's3_etag' in o: # New-enough MUs are already in the right format etag = o['s3_etag'] + elif 'slo_etag' in o: + # SLOs may be in something *close* to the MU format + etag = '"%s-N"' % o['slo_etag'].strip('"') else: + # Normal objects just use the MD5 etag = '"%s"' % o['hash'] + # This also catches sufficiently-old SLOs, but we have + # no way to identify those from container listings SubElement(contents, 'ETag').text = etag SubElement(contents, 'Size').text = str(o['bytes']) if fetch_owner or listing_type != 'version-2': diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py index e199482b14..0a5a341d50 100644 --- a/swift/common/middleware/s3api/controllers/obj.py +++ b/swift/common/middleware/s3api/controllers/obj.py @@ -16,7 +16,7 @@ from swift.common.http import HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_NO_CONTENT from swift.common.request_helpers import update_etag_is_at_header from swift.common.swob import Range, content_range_header_value -from swift.common.utils import public +from swift.common.utils import public, list_from_csv from swift.common.middleware.s3api.utils import S3Timestamp, sysmeta_header from swift.common.middleware.s3api.controllers.base import Controller @@ -62,8 +62,19 @@ class ObjectController(Controller): return resp def GETorHEAD(self, req): - if any(match_header in req.headers - for match_header in ('if-match', 'if-none-match')): + had_match = False + for match_header in ('if-match', 'if-none-match'): + if match_header not in req.headers: + continue + had_match = True + for value in list_from_csv(req.headers[match_header]): + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + if value.endswith('-N'): + # Deal with fake S3-like etags for SLOs uploaded via Swift + req.headers[match_header] += ', ' + value[:-2] + + if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) diff --git a/swift/common/middleware/s3api/s3response.py b/swift/common/middleware/s3api/s3response.py index da79ce3b54..23192ef2b4 100644 --- a/swift/common/middleware/s3api/s3response.py +++ b/swift/common/middleware/s3api/s3response.py @@ -138,6 +138,12 @@ class S3Response(S3ResponseBase, swob.Response): # Multipart uploads in AWS have ETags like # - headers['etag'] = override_etag + elif self.is_slo and 'etag' in headers: + # Many AWS clients use the presence of a '-' to decide whether + # to attempt client-side download validation, so even if we + # didn't store the AWS-style header, tack on a '-N'. (Use 'N' + # because we don't actually know how many parts there are.) + headers['etag'] += '-N' self.headers = headers diff --git a/test/unit/common/middleware/s3api/test_bucket.py b/test/unit/common/middleware/s3api/test_bucket.py index d48deca9a1..b53d936f17 100644 --- a/test/unit/common/middleware/s3api/test_bucket.py +++ b/test/unit/common/middleware/s3api/test_bucket.py @@ -42,6 +42,8 @@ class TestS3ApiBucket(S3ApiTestCase): (u'lily-\u062a', '2011-01-05T02:19:14.275290', 0, 390), ('mu', '2011-01-05T02:19:14.275290', 'md5-of-the-manifest; s3_etag=0', '3909'), + ('slo', '2011-01-05T02:19:14.275290', + 'md5-of-the-manifest', '3909'), ('with space', '2011-01-05T02:19:14.275290', 0, 390), ('with%20space', '2011-01-05T02:19:14.275290', 0, 390)) @@ -49,6 +51,7 @@ class TestS3ApiBucket(S3ApiTestCase): {'name': item[0], 'last_modified': str(item[1]), 'hash': str(item[2]), 'bytes': str(item[3])} for item in self.objects] + objects[5]['slo_etag'] = '"0"' object_list = json.dumps(objects) self.prefixes = ['rose', 'viola', 'lily'] @@ -159,15 +162,14 @@ class TestS3ApiBucket(S3ApiTestCase): objects = elem.iterchildren('Contents') - names = [] + items = [] for o in objects: - names.append(o.find('./Key').text) + items.append((o.find('./Key').text, o.find('./ETag').text)) self.assertEqual('2011-01-05T02:19:14.275Z', o.find('./LastModified').text) - self.assertEqual('"0"', o.find('./ETag').text) - - self.assertEqual( - names, [obj[0].encode('utf-8') for obj in self.objects]) + self.assertEqual(items, [ + (i[0].encode('utf-8'), '"0-N"' if i[0] == 'slo' else '"0"') + for i in self.objects]) def test_bucket_GET_url_encoded(self): bucket_name = 'junk' @@ -184,16 +186,15 @@ class TestS3ApiBucket(S3ApiTestCase): objects = elem.iterchildren('Contents') - names = [] + items = [] for o in objects: - names.append(o.find('./Key').text) + items.append((o.find('./Key').text, o.find('./ETag').text)) self.assertEqual('2011-01-05T02:19:14.275Z', o.find('./LastModified').text) - self.assertEqual('"0"', o.find('./ETag').text) - self.assertEqual(len(names), len(self.objects)) - for i in self.objects: - self.assertIn(quote(i[0].encode('utf-8')), names) + self.assertEqual(items, [ + (quote(i[0].encode('utf-8')), '"0-N"' if i[0] == 'slo' else '"0"') + for i in self.objects]) def test_bucket_GET_subdir(self): bucket_name = 'junk-subdir' @@ -529,7 +530,8 @@ class TestS3ApiBucket(S3ApiTestCase): self.assertEqual([v.find('./LastModified').text for v in versions], [v[1][:-3] + 'Z' for v in objects]) self.assertEqual([v.find('./ETag').text for v in versions], - ['"0"' for v in objects]) + ['"0-N"' if v[0] == 'slo' else '"0"' + for v in objects]) self.assertEqual([v.find('./Size').text for v in versions], [str(v[3]) for v in objects]) self.assertEqual([v.find('./Owner/ID').text for v in versions], diff --git a/test/unit/common/middleware/s3api/test_s3response.py b/test/unit/common/middleware/s3api/test_s3response.py index 2e2f7d825c..9ffdaf1db1 100644 --- a/test/unit/common/middleware/s3api/test_s3response.py +++ b/test/unit/common/middleware/s3api/test_s3response.py @@ -30,7 +30,10 @@ class TestResponse(unittest.TestCase): 'Etag': 'theetag'}) s3resp = S3Response.from_swift_resp(resp) self.assertEqual(expected, s3resp.is_slo) - self.assertEqual('"theetag"', s3resp.headers['ETag']) + if s3resp.is_slo: + self.assertEqual('"theetag-N"', s3resp.headers['ETag']) + else: + self.assertEqual('"theetag"', s3resp.headers['ETag']) def test_response_s3api_sysmeta_headers(self): for _server_type in ('object', 'container'):