openstack/swift
Code Issues Proposed changes

Additionally break container servers during fast-post probes

Browse Source 
The existing probetests were already doing brainsplitting on the object
servers in order to introduce composite metdata timestamp reconciliation.

But they were not introducing failures at the container server level, and
therefore not covering functional testing of those servers replication
consistency repair.

This change adds a brain splitter for the container servers to additionally
exercise the consistency engine at that layer under the existing probetest so
that you can observe their correct behavior while exercising the probetests
[1].

1. I used a script like this to watch the object servers and container db's
repair themselves -> https://gist.github.com/clayg/a8077d9d29ff68bb0edd

Change-Id: Ic5bb03dffef7e30c58338cef969a105577b84620
This commit is contained in:
Clay Gerrard 2016-02-29 13:14:59 +00:00 committed by Alistair Coles
parent e91de49d68
commit fa0c5f244f
1 changed files with 83 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
test/probe/test_object_metadata_replication.py
View File

@ -45,6 +45,8 @@ class Test(ReplProbeTest):
self.brain = BrainSplitter(self.url, self.token, self.container_name, self.brain = BrainSplitter(self.url, self.token, self.container_name,
self.object_name, 'object', self.object_name, 'object',
policy=self.policy) policy=self.policy)
self.container_brain = BrainSplitter(self.url, self.token,
self.container_name)
self.int_client = self.make_internal_client(object_post_as_copy=False) self.int_client = self.make_internal_client(object_post_as_copy=False)
def tearDown(self): def tearDown(self):
@ -182,40 +184,51 @@ class Test(ReplProbeTest):
# put newer object with sysmeta to first server subset # put newer object with sysmeta to first server subset
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object() self._put_object()
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# delete object on second server subset # delete object on second server subset
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._delete_object() self._delete_object()
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# run replicator # run replicator
self.get_to_final_state() self.get_to_final_state()
# check object deletion has been replicated on first server set # check object deletion has been replicated on first server set
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._get_object(expect_statuses=(4,)) self._get_object(expect_statuses=(4,))
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# check object deletion persists on second server set # check object deletion persists on second server set
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._get_object(expect_statuses=(4,)) self._get_object(expect_statuses=(4,))
# put newer object to second server set # put newer object to second server set
self._put_object() self._put_object()
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# run replicator # run replicator
self.get_to_final_state() self.get_to_final_state()
# check new object has been replicated on first server set # check new object has been replicated on first server set
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._get_object() self._get_object()
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# check new object persists on second server set # check new object persists on second server set
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._get_object() self._get_object()
def test_object_after_replication_with_subsequent_post(self): def test_object_after_replication_with_subsequent_post(self):
@ -226,10 +239,12 @@ class Test(ReplProbeTest):
# put newer object to first server subset # put newer object to first server subset
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar'}, body=u'newer') self._put_object(headers={'Content-Type': 'bar'}, body=u'newer')
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
etag = metadata['etag'] etag = metadata['etag']
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# post some user meta to all servers # post some user meta to all servers
self._post_object({'x-object-meta-bar': 'meta-bar'}) self._post_object({'x-object-meta-bar': 'meta-bar'})
@ -239,11 +254,13 @@ class Test(ReplProbeTest):
# check that newer data has been replicated to second server subset # check that newer data has been replicated to second server subset
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
self.assertEqual(etag, metadata['etag']) self.assertEqual(etag, metadata['etag'])
self.assertEqual('bar', metadata['content-type']) self.assertEqual('bar', metadata['content-type'])
self.assertEqual('meta-bar', metadata['x-object-meta-bar']) self.assertEqual('meta-bar', metadata['x-object-meta-bar'])
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self._assert_consistent_object_metadata() self._assert_consistent_object_metadata()
self._assert_consistent_container_dbs() self._assert_consistent_container_dbs()
@ -257,15 +274,18 @@ class Test(ReplProbeTest):
# put object with sysmeta to first server subset # put object with sysmeta to first server subset
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers=sysmeta) self._put_object(headers=sysmeta)
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in sysmeta: for key in sysmeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
self.assertEqual(metadata[key], sysmeta[key]) self.assertEqual(metadata[key], sysmeta[key])
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# put object with updated sysmeta to second server subset # put object with updated sysmeta to second server subset
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers=sysmeta2) self._put_object(headers=sysmeta2)
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in sysmeta2: for key in sysmeta2:
@ -281,12 +301,14 @@ class Test(ReplProbeTest):
self.assertEqual(metadata[key], sysmeta2[key]) self.assertEqual(metadata[key], sysmeta2[key])
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# run replicator # run replicator
self.get_to_final_state() self.get_to_final_state()
# check sysmeta has been replicated to first server subset # check sysmeta has been replicated to first server subset
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in usermeta: for key in usermeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
@ -295,9 +317,11 @@ class Test(ReplProbeTest):
self.assertTrue(key in metadata, key) self.assertTrue(key in metadata, key)
self.assertEqual(metadata[key], sysmeta2[key]) self.assertEqual(metadata[key], sysmeta2[key])
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# check user sysmeta ok on second server subset # check user sysmeta ok on second server subset
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in usermeta: for key in usermeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
@ -306,6 +330,7 @@ class Test(ReplProbeTest):
self.assertTrue(key in metadata, key) self.assertTrue(key in metadata, key)
self.assertEqual(metadata[key], sysmeta2[key]) self.assertEqual(metadata[key], sysmeta2[key])
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self._assert_consistent_object_metadata() self._assert_consistent_object_metadata()
self._assert_consistent_container_dbs() self._assert_consistent_container_dbs()
@ -319,15 +344,18 @@ class Test(ReplProbeTest):
self._put_object() self._put_object()
# put newer object with sysmeta to first server subset # put newer object with sysmeta to first server subset
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers=sysmeta) self._put_object(headers=sysmeta)
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in sysmeta: for key in sysmeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
self.assertEqual(metadata[key], sysmeta[key]) self.assertEqual(metadata[key], sysmeta[key])
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# post some user meta to second server subset # post some user meta to second server subset
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(usermeta) self._post_object(usermeta)
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in usermeta: for key in usermeta:
@ -336,6 +364,7 @@ class Test(ReplProbeTest):
for key in sysmeta: for key in sysmeta:
self.assertFalse(key in metadata) self.assertFalse(key in metadata)
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# run replicator # run replicator
self.get_to_final_state() self.get_to_final_state()
@ -343,6 +372,7 @@ class Test(ReplProbeTest):
# check user metadata has been replicated to first server subset # check user metadata has been replicated to first server subset
# and sysmeta is unchanged # and sysmeta is unchanged
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
expected = dict(sysmeta) expected = dict(sysmeta)
expected.update(usermeta) expected.update(usermeta)
@ -350,14 +380,17 @@ class Test(ReplProbeTest):
self.assertTrue(key in metadata, key) self.assertTrue(key in metadata, key)
self.assertEqual(metadata[key], expected[key]) self.assertEqual(metadata[key], expected[key])
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# check user metadata and sysmeta both on second server subset # check user metadata and sysmeta both on second server subset
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in expected.keys(): for key in expected.keys():
self.assertTrue(key in metadata, key) self.assertTrue(key in metadata, key)
self.assertEqual(metadata[key], expected[key]) self.assertEqual(metadata[key], expected[key])
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self._assert_consistent_object_metadata() self._assert_consistent_object_metadata()
self._assert_consistent_container_dbs() self._assert_consistent_container_dbs()
@ -372,21 +405,25 @@ class Test(ReplProbeTest):
# put user meta to first server subset # put user meta to first server subset
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers=usermeta) self._post_object(headers=usermeta)
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in usermeta: for key in usermeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
self.assertEqual(metadata[key], usermeta[key]) self.assertEqual(metadata[key], usermeta[key])
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# put newer object with sysmeta to second server subset # put newer object with sysmeta to second server subset
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers=sysmeta) self._put_object(headers=sysmeta)
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in sysmeta: for key in sysmeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
self.assertEqual(metadata[key], sysmeta[key]) self.assertEqual(metadata[key], sysmeta[key])
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# run replicator # run replicator
self.get_to_final_state() self.get_to_final_state()
@ -394,6 +431,7 @@ class Test(ReplProbeTest):
# check stale user metadata is not replicated to first server subset # check stale user metadata is not replicated to first server subset
# and sysmeta is unchanged # and sysmeta is unchanged
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in sysmeta: for key in sysmeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
@ -401,10 +439,12 @@ class Test(ReplProbeTest):
for key in usermeta: for key in usermeta:
self.assertFalse(key in metadata) self.assertFalse(key in metadata)
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# check stale user metadata is removed from second server subset # check stale user metadata is removed from second server subset
# and sysmeta is replicated # and sysmeta is replicated
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
for key in sysmeta: for key in sysmeta:
self.assertTrue(key in metadata) self.assertTrue(key in metadata)
@ -412,6 +452,7 @@ class Test(ReplProbeTest):
for key in usermeta: for key in usermeta:
self.assertFalse(key in metadata) self.assertFalse(key in metadata)
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self._assert_consistent_object_metadata() self._assert_consistent_object_metadata()
self._assert_consistent_container_dbs() self._assert_consistent_container_dbs()
@ -432,18 +473,24 @@ class Test(ReplProbeTest):
# incomplete write to primary half # incomplete write to primary half
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'foo'}) self._put_object(headers={'Content-Type': 'foo'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff write # handoff write
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar'}) self._put_object(headers={'Content-Type': 'bar'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# content-type update to primary half # content-type update to primary half
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers={'Content-Type': 'baz'}) self._post_object(headers={'Content-Type': 'baz'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self.get_to_final_state() self.get_to_final_state()
@ -481,18 +528,24 @@ class Test(ReplProbeTest):
# incomplete write # incomplete write
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'foo'}) self._put_object(headers={'Content-Type': 'foo'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff write # handoff write
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar'}) self._put_object(headers={'Content-Type': 'bar'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# metadata update with newest data unavailable # metadata update with newest data unavailable
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers={'X-Object-Meta-Color': 'Blue'}) self._post_object(headers={'X-Object-Meta-Color': 'Blue'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self.get_to_final_state() self.get_to_final_state()
@ -535,26 +588,34 @@ class Test(ReplProbeTest):
# incomplete write # incomplete write
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'foo', self._put_object(headers={'Content-Type': 'foo',
'X-Object-Sysmeta-Test': 'older'}) 'X-Object-Sysmeta-Test': 'older'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff write # handoff write
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar', self._put_object(headers={'Content-Type': 'bar',
'X-Object-Sysmeta-Test': 'newer'}) 'X-Object-Sysmeta-Test': 'newer'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# incomplete post with content type # incomplete post with content type
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers={'Content-Type': 'bif'}) self._post_object(headers={'Content-Type': 'bif'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# incomplete post to handoff with content type # incomplete post to handoff with content type
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._post_object(headers={'Content-Type': 'baz', self._post_object(headers={'Content-Type': 'baz',
'X-Object-Meta-Color': 'Red'}) 'X-Object-Meta-Color': 'Red'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# complete post with no content type # complete post with no content type
self._post_object(headers={'X-Object-Meta-Color': 'Blue', self._post_object(headers={'X-Object-Meta-Color': 'Blue',
@ -601,20 +662,26 @@ class Test(ReplProbeTest):
# incomplete write to handoff half # incomplete write to handoff half
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar', self._put_object(headers={'Content-Type': 'bar',
'X-Object-Sysmeta-Test': 'newer'}) 'X-Object-Sysmeta-Test': 'newer'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# incomplete post with no content type to primary half # incomplete post with no content type to primary half
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers={'X-Object-Meta-Color': 'Red', self._post_object(headers={'X-Object-Meta-Color': 'Red',
'X-Object-Sysmeta-Test': 'ignored'}) 'X-Object-Sysmeta-Test': 'ignored'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# incomplete post with no content type to handoff half # incomplete post with no content type to handoff half
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._post_object(headers={'X-Object-Meta-Color': 'Blue'}) self._post_object(headers={'X-Object-Meta-Color': 'Blue'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
self.get_to_final_state() self.get_to_final_state()
@ -645,38 +712,48 @@ class Test(ReplProbeTest):
self.brain.put_container(policy_index=0) self.brain.put_container(policy_index=0)
# incomplete put to handoff # incomplete put to handoff
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'oldest', self._put_object(headers={'Content-Type': 'oldest',
'X-Object-Sysmeta-Test': 'oldest', 'X-Object-Sysmeta-Test': 'oldest',
'X-Object-Meta-Test': 'oldest'}) 'X-Object-Meta-Test': 'oldest'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# incomplete put to primary # incomplete put to primary
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'oldest', self._put_object(headers={'Content-Type': 'oldest',
'X-Object-Sysmeta-Test': 'oldest', 'X-Object-Sysmeta-Test': 'oldest',
'X-Object-Meta-Test': 'oldest'}) 'X-Object-Meta-Test': 'oldest'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# incomplete post with content-type to handoff # incomplete post with content-type to handoff
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._post_object(headers={'Content-Type': 'newer', self._post_object(headers={'Content-Type': 'newer',
'X-Object-Meta-Test': 'newer'}) 'X-Object-Meta-Test': 'newer'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# incomplete put to primary # incomplete put to primary
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'newest', self._put_object(headers={'Content-Type': 'newest',
'X-Object-Sysmeta-Test': 'newest', 'X-Object-Sysmeta-Test': 'newest',
'X-Object-Meta-Test': 'newer'}) 'X-Object-Meta-Test': 'newer'})
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# incomplete post with no content-type to handoff which still has # incomplete post with no content-type to handoff which still has
# out of date content-type # out of date content-type
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._post_object(headers={'X-Object-Meta-Test': 'newest'}) self._post_object(headers={'X-Object-Meta-Test': 'newest'})
metadata = self._get_object_metadata() metadata = self._get_object_metadata()
self.assertEqual(metadata['x-object-meta-test'], 'newest') self.assertEqual(metadata['x-object-meta-test'], 'newest')
self.assertEqual(metadata['content-type'], 'newer') self.assertEqual(metadata['content-type'], 'newer')
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
self.get_to_final_state() self.get_to_final_state()
@ -707,21 +784,26 @@ class Test(ReplProbeTest):
self.brain.put_container(policy_index=0) self.brain.put_container(policy_index=0)
# incomplete put # incomplete put
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'oldest', self._put_object(headers={'Content-Type': 'oldest',
'X-Object-Sysmeta-Test': 'oldest', 'X-Object-Sysmeta-Test': 'oldest',
'X-Object-Meta-Test': 'oldest'}) 'X-Object-Meta-Test': 'oldest'})
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# incomplete put then delete # incomplete put then delete
self.brain.stop_handoff_half() self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'oldest', self._put_object(headers={'Content-Type': 'oldest',
'X-Object-Sysmeta-Test': 'oldest', 'X-Object-Sysmeta-Test': 'oldest',
'X-Object-Meta-Test': 'oldest'}) 'X-Object-Meta-Test': 'oldest'})
self._delete_object() self._delete_object()
self.brain.start_handoff_half() self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff post # handoff post
self.brain.stop_primary_half() self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._post_object(headers={'Content-Type': 'newest', self._post_object(headers={'Content-Type': 'newest',
'X-Object-Sysmeta-Test': 'ignored', 'X-Object-Sysmeta-Test': 'ignored',
'X-Object-Meta-Test': 'newest'}) 'X-Object-Meta-Test': 'newest'})
@ -733,6 +815,7 @@ class Test(ReplProbeTest):
self.assertEqual(metadata['content-type'], 'newest') self.assertEqual(metadata['content-type'], 'newest')
self.brain.start_primary_half() self.brain.start_primary_half()
self.container_brain.start_primary_half()
# delete trumps later post # delete trumps later post
self.get_to_final_state() self.get_to_final_state()