diff --git a/ceilometer/coordination.py b/ceilometer/coordination.py index e4b98aa3..50c9b202 100644 --- a/ceilometer/coordination.py +++ b/ceilometer/coordination.py @@ -130,6 +130,9 @@ class PartitionCoordinator(object): create_grp_req.get() except tooz.coordination.GroupAlreadyExist: pass + except tooz.coordination.ToozError: + LOG.exception(_LE('Error joining partitioning group %s,' + ' re-trying'), group_id) self._groups.add(group_id) def leave_group(self, group_id): diff --git a/ceilometer/tests/unit/test_coordination.py b/ceilometer/tests/unit/test_coordination.py index 5171d67f..086f62a8 100644 --- a/ceilometer/tests/unit/test_coordination.py +++ b/ceilometer/tests/unit/test_coordination.py @@ -83,6 +83,21 @@ class MockToozCoordExceptionRaiser(MockToozCoordinator): raise tooz.coordination.ToozError('error') +class MockToozCoordExceptionOnJoinRaiser(MockToozCoordinator): + def __init__(self, member_id, shared_storage, retry_count=None): + super(MockToozCoordExceptionOnJoinRaiser, + self).__init__(member_id, shared_storage) + self.tooz_error_count = retry_count + self.count = 0 + + def join_group(self, group_id, capabilities=b''): + if self.count == self.tooz_error_count: + return MockAsyncResult(None) + else: + self.count += 1 + raise tooz.coordination.ToozError('error') + + class MockAsyncResult(tooz.coordination.CoordAsyncResult): def __init__(self, result): self.result = result @@ -135,12 +150,14 @@ class TestPartitioning(base.BaseTestCase): self.shared_storage = {} def _get_new_started_coordinator(self, shared_storage, agent_id=None, - coordinator_cls=None): + coordinator_cls=None, retry_count=None): coordinator_cls = coordinator_cls or MockToozCoordinator self.CONF.set_override('backend_url', 'xxx://yyy', group='coordination') with mock.patch('tooz.coordination.get_coordinator', lambda _, member_id: + coordinator_cls(member_id, shared_storage, + retry_count) if retry_count else coordinator_cls(member_id, shared_storage)): pc = coordination.PartitionCoordinator(agent_id) pc.start() @@ -210,6 +227,20 @@ class TestPartitioning(base.BaseTestCase): for e in expected_errors: self.assertIn(e, self.str_handler.messages['error']) + def test_coordination_backend_connection_fail_on_join(self): + coord = self._get_new_started_coordinator( + {'group'}, 'agent1', MockToozCoordExceptionOnJoinRaiser, + retry_count=2) + with mock.patch('tooz.coordination.get_coordinator', + return_value=MockToozCoordExceptionOnJoinRaiser): + coord.join_group(group_id='group') + + expected_errors = ['Error joining partitioning group group,' + ' re-trying', + 'Error joining partitioning group group,' + ' re-trying'] + self.assertEqual(expected_errors, self.str_handler.messages['error']) + def test_reconnect(self): coord = self._get_new_started_coordinator({}, 'a', MockToozCoordExceptionRaiser)