Make SLO manifest copies retain correct content-type
When copying an SLO manifest with multipart-manifest=get the actual manifest content-type should get copied to the destination, rather than the application/json value that is synthesised by SLO in a GET response. That way the result of a HEAD on the copied manifest is the same as a HEAD to the source, and the container listings for the two are consistent. This patch also un-skips a functional test and adds functional tests that verify this patch and also verify that etags and size also get correctly copied and updated in destination container (bug #1260446). Closes-Bug: #1260446 Closes-Bug: #1583756 Change-Id: Ie7fa82f70b3ec3ef568f5355c69f6bce460ba25d
This commit is contained in:
Alistair Coles
parent
40eace99c3
commit
d0ec1adb78
@ -582,6 +582,7 @@ class SloGetContext(WSGIContext):
|
||||
if req.params.get('format') == 'raw':
|
||||
resp_iter = self.convert_segment_listing(
|
||||
self._response_headers, resp_iter)
|
||||
else:
|
||||
new_headers = []
|
||||
for header, value in self._response_headers:
|
||||
if header.lower() == 'content-type':
|
||||
|
||||
@ -2875,18 +2875,26 @@ class TestSlo(Base):
|
||||
def test_slo_container_listing(self):
|
||||
# the listing object size should equal the sum of the size of the
|
||||
# segments, not the size of the manifest body
|
||||
raise SkipTest('Only passes with object_post_as_copy=False')
|
||||
file_item = self.env.container.file(Utils.create_name)
|
||||
file_item.write(
|
||||
json.dumps([self.env.seg_info['seg_a']]),
|
||||
parms={'multipart-manifest': 'put'})
|
||||
# The container listing has the etag of the actual manifest object
|
||||
# contents which we get using multipart-manifest=get. Arguably this
|
||||
# should be the etag that we get when NOT using multipart-manifest=get,
|
||||
# to be consistent with size and content-type. But here we at least
|
||||
# verify that it remains consistent when the object is updated with a
|
||||
# POST.
|
||||
file_item.initialize(parms={'multipart-manifest': 'get'})
|
||||
expected_etag = file_item.etag
|
||||
|
||||
files = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in files:
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] == file_item.name:
|
||||
self.assertEqual(1024 * 1024, f_dict['bytes'])
|
||||
self.assertEqual('application/octet-stream',
|
||||
f_dict['content_type'])
|
||||
self.assertEqual(expected_etag, f_dict['hash'])
|
||||
break
|
||||
else:
|
||||
self.fail('Failed to find manifest file in container listing')
|
||||
@ -2898,12 +2906,31 @@ class TestSlo(Base):
|
||||
self.assertEqual('image/jpeg', file_item.content_type) # sanity
|
||||
|
||||
# verify that the container listing is consistent with the file
|
||||
files = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in files:
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] == file_item.name:
|
||||
self.assertEqual(1024 * 1024, f_dict['bytes'])
|
||||
self.assertEqual(file_item.content_type,
|
||||
f_dict['content_type'])
|
||||
self.assertEqual(expected_etag, f_dict['hash'])
|
||||
break
|
||||
else:
|
||||
self.fail('Failed to find manifest file in container listing')
|
||||
|
||||
# now POST with no change to content-type
|
||||
file_item.sync_metadata({'X-Object-Meta-Test': 'blah'},
|
||||
cfg={'no_content_type': True})
|
||||
file_item.initialize()
|
||||
self.assertEqual('image/jpeg', file_item.content_type) # sanity
|
||||
|
||||
# verify that the container listing is consistent with the file
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] == file_item.name:
|
||||
self.assertEqual(1024 * 1024, f_dict['bytes'])
|
||||
self.assertEqual(file_item.content_type,
|
||||
f_dict['content_type'])
|
||||
self.assertEqual(expected_etag, f_dict['hash'])
|
||||
break
|
||||
else:
|
||||
self.fail('Failed to find manifest file in container listing')
|
||||
@ -3127,17 +3154,109 @@ class TestSlo(Base):
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, len(copied_contents))
|
||||
|
||||
def test_slo_copy_the_manifest(self):
|
||||
file_item = self.env.container.file("manifest-abcde")
|
||||
self.assertTrue(file_item.copy(self.env.container.name,
|
||||
source = self.env.container.file("manifest-abcde")
|
||||
source_contents = source.read(parms={'multipart-manifest': 'get'})
|
||||
source_json = json.loads(source_contents)
|
||||
source.initialize()
|
||||
self.assertEqual('application/octet-stream', source.content_type)
|
||||
source.initialize(parms={'multipart-manifest': 'get'})
|
||||
source_hash = hashlib.md5()
|
||||
source_hash.update(source_contents)
|
||||
self.assertEqual(source_hash.hexdigest(), source.etag)
|
||||
|
||||
self.assertTrue(source.copy(self.env.container.name,
|
||||
"copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'}))
|
||||
|
||||
copied = self.env.container.file("copied-abcde-manifest-only")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
json.loads(copied_contents)
|
||||
copied_json = json.loads(copied_contents)
|
||||
except ValueError:
|
||||
self.fail("COPY didn't copy the manifest (invalid json on GET)")
|
||||
self.assertEqual(source_json, copied_json)
|
||||
copied.initialize()
|
||||
self.assertEqual('application/octet-stream', copied.content_type)
|
||||
copied.initialize(parms={'multipart-manifest': 'get'})
|
||||
copied_hash = hashlib.md5()
|
||||
copied_hash.update(copied_contents)
|
||||
self.assertEqual(copied_hash.hexdigest(), copied.etag)
|
||||
|
||||
# verify the listing metadata
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
names = {}
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] in ('manifest-abcde',
|
||||
'copied-abcde-manifest-only'):
|
||||
names[f_dict['name']] = f_dict
|
||||
|
||||
self.assertIn('manifest-abcde', names)
|
||||
actual = names['manifest-abcde']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('application/octet-stream', actual['content_type'])
|
||||
self.assertEqual(source.etag, actual['hash'])
|
||||
|
||||
self.assertIn('copied-abcde-manifest-only', names)
|
||||
actual = names['copied-abcde-manifest-only']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('application/octet-stream', actual['content_type'])
|
||||
self.assertEqual(copied.etag, actual['hash'])
|
||||
|
||||
def test_slo_copy_the_manifest_updating_metadata(self):
|
||||
source = self.env.container.file("manifest-abcde")
|
||||
source.content_type = 'application/octet-stream'
|
||||
source.sync_metadata({'test': 'original'})
|
||||
source_contents = source.read(parms={'multipart-manifest': 'get'})
|
||||
source_json = json.loads(source_contents)
|
||||
source.initialize()
|
||||
self.assertEqual('application/octet-stream', source.content_type)
|
||||
source.initialize(parms={'multipart-manifest': 'get'})
|
||||
source_hash = hashlib.md5()
|
||||
source_hash.update(source_contents)
|
||||
self.assertEqual(source_hash.hexdigest(), source.etag)
|
||||
self.assertEqual(source.metadata['test'], 'original')
|
||||
|
||||
self.assertTrue(
|
||||
source.copy(self.env.container.name, "copied-abcde-manifest-only",
|
||||
parms={'multipart-manifest': 'get'},
|
||||
hdrs={'Content-Type': 'image/jpeg',
|
||||
'X-Object-Meta-Test': 'updated'}))
|
||||
|
||||
copied = self.env.container.file("copied-abcde-manifest-only")
|
||||
copied_contents = copied.read(parms={'multipart-manifest': 'get'})
|
||||
try:
|
||||
copied_json = json.loads(copied_contents)
|
||||
except ValueError:
|
||||
self.fail("COPY didn't copy the manifest (invalid json on GET)")
|
||||
self.assertEqual(source_json, copied_json)
|
||||
copied.initialize()
|
||||
self.assertEqual('image/jpeg', copied.content_type)
|
||||
copied.initialize(parms={'multipart-manifest': 'get'})
|
||||
copied_hash = hashlib.md5()
|
||||
copied_hash.update(copied_contents)
|
||||
self.assertEqual(copied_hash.hexdigest(), copied.etag)
|
||||
self.assertEqual(copied.metadata['test'], 'updated')
|
||||
|
||||
# verify the listing metadata
|
||||
listing = self.env.container.files(parms={'format': 'json'})
|
||||
names = {}
|
||||
for f_dict in listing:
|
||||
if f_dict['name'] in ('manifest-abcde',
|
||||
'copied-abcde-manifest-only'):
|
||||
names[f_dict['name']] = f_dict
|
||||
|
||||
self.assertIn('manifest-abcde', names)
|
||||
actual = names['manifest-abcde']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('application/octet-stream', actual['content_type'])
|
||||
# the container listing should have the etag of the manifest contents
|
||||
self.assertEqual(source.etag, actual['hash'])
|
||||
|
||||
self.assertIn('copied-abcde-manifest-only', names)
|
||||
actual = names['copied-abcde-manifest-only']
|
||||
self.assertEqual(4 * 1024 * 1024 + 1, actual['bytes'])
|
||||
self.assertEqual('image/jpeg', actual['content_type'])
|
||||
self.assertEqual(copied.etag, actual['hash'])
|
||||
|
||||
def test_slo_copy_the_manifest_account(self):
|
||||
acct = self.env.conn.account_name
|
||||
@ -3295,8 +3414,8 @@ class TestSlo(Base):
|
||||
got_body = manifest.read(parms={'multipart-manifest': 'get',
|
||||
'format': 'raw'})
|
||||
|
||||
self.assertEqual('application/json; charset=utf-8',
|
||||
manifest.content_type)
|
||||
# raw format should have the actual manifest object content-type
|
||||
self.assertEqual('application/octet-stream', manifest.content_type)
|
||||
try:
|
||||
value = json.loads(got_body)
|
||||
except ValueError:
|
||||
|
||||
@ -1112,7 +1112,8 @@ class TestSloGetRawManifest(SloTestCase):
|
||||
self.bc_etag = md5hex(_bc_manifest_json)
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-bc',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=35',
|
||||
# proxy obj controller removes swift_bytes from content-type
|
||||
swob.HTTPOk, {'Content-Type': 'text/plain',
|
||||
'X-Static-Large-Object': 'true',
|
||||
'X-Object-Meta-Plant': 'Ficus',
|
||||
'Etag': md5hex(_bc_manifest_json)},
|
||||
@ -1127,7 +1128,8 @@ class TestSloGetRawManifest(SloTestCase):
|
||||
'content_type': 'text/plain', 'range': '100-200'}])
|
||||
self.app.register(
|
||||
'GET', '/v1/AUTH_test/gettest/manifest-bc-r',
|
||||
swob.HTTPOk, {'Content-Type': 'application/json;swift_bytes=25',
|
||||
# proxy obj controller removes swift_bytes from content-type
|
||||
swob.HTTPOk, {'Content-Type': 'text/plain',
|
||||
'X-Static-Large-Object': 'true',
|
||||
'X-Object-Meta-Plant': 'Ficus',
|
||||
'Etag': md5hex(_bc_manifest_json_ranges)},
|
||||
@ -1144,9 +1146,8 @@ class TestSloGetRawManifest(SloTestCase):
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertTrue(('Etag', self.bc_etag) in headers, headers)
|
||||
self.assertTrue(('X-Static-Large-Object', 'true') in headers, headers)
|
||||
self.assertTrue(
|
||||
('Content-Type', 'application/json; charset=utf-8') in headers,
|
||||
headers)
|
||||
# raw format should return the actual manifest object content-type
|
||||
self.assertIn(('Content-Type', 'text/plain'), headers)
|
||||
|
||||
try:
|
||||
resp_data = json.loads(body)
|
||||
@ -1172,9 +1173,8 @@ class TestSloGetRawManifest(SloTestCase):
|
||||
status, headers, body = self.call_slo(req)
|
||||
|
||||
self.assertEqual(status, '200 OK')
|
||||
self.assertTrue(
|
||||
('Content-Type', 'application/json; charset=utf-8') in headers,
|
||||
headers)
|
||||
# raw format should return the actual manifest object content-type
|
||||
self.assertIn(('Content-Type', 'text/plain'), headers)
|
||||
try:
|
||||
resp_data = json.loads(body)
|
||||
except ValueError:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user