openstack
/
senlin
Code Issues Proposed changes

Fix health policy attach/detach action when messaging.MessagingTimeout occurred 

Health policy attach/detach from cluster still successful when send
notification to health manager service with messaging.MessagingTimeout.
This is very bad for the cluster health manager，the following things may happen:
1. When do attach from cluster when messaging.MessagingTimeout occurred，
   Register cluster to health registry will be failed,
   and then the cluster will not do health check, even though
   the policy has attached.
2. When do detach from cluster when messaging.MessagingTimeout occurred,
   Unregister cluster to health registry will be failed,
   and then the cluster will still do health check, even though the policy
   has detached.

Change-Id: Iedf1f0b77e9034ccb81a9073b936875c2259fb10
Closes-Bug: #1897443
This commit is contained in:
wangyu 2020-09-27 17:56:23 +08:00
parent 35b5c65bd0
commit 40830c8834
2 changed files with 27 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
senlin/policies/health_policy.py
View File

@ -381,7 +381,12 @@ class HealthPolicy(base.Policy):
detection_mode = {'detection_modes': converted_detection_modes} detection_mode = {'detection_modes': converted_detection_modes}
kwargs['params'].update(detection_mode) kwargs['params'].update(detection_mode)
health_manager.register(cluster.id, engine_id=None, **kwargs) ret = health_manager.register(cluster.id, engine_id=None, **kwargs)
if not ret:
LOG.warning('Registering health manager for cluster %s '
'timed out.', cluster.id)
err_msg = _("Registering health manager for cluster timed out.")
return False, err_msg
data = { data = {
'interval': self.interval, 'interval': self.interval,
@ -405,6 +410,8 @@ class HealthPolicy(base.Policy):
if not ret: if not ret:
LOG.warning('Unregistering health manager for cluster %s ' LOG.warning('Unregistering health manager for cluster %s '
'timed out.', cluster.id) 'timed out.', cluster.id)
err_msg = _("Unregistering health manager for cluster timed out.")
return False, err_msg
return True, '' return True, ''
def pre_op(self, cluster_id, action, **args): def pre_op(self, cluster_id, action, **args):

21
senlin/tests/unit/policies/test_health_policy.py
View File

@ -278,12 +278,29 @@ class TestHealthPolicy(base.SenlinTestCase):
self.assertEqual("Recovery action REBOOT is only applicable to " self.assertEqual("Recovery action REBOOT is only applicable to "
"os.nova.server clusters.", data) "os.nova.server clusters.", data)
@mock.patch.object(health_manager, 'register')
def test_attach_failed_with_notify_timeout(self, mock_hm_reg):
mock_hm_reg.return_value = False
res, data = self.hp.attach(self.cluster)
self.assertFalse(res)
self.assertEqual("Registering health manager for cluster timed "
"out.", data)
@mock.patch.object(health_manager, 'unregister') @mock.patch.object(health_manager, 'unregister')
def test_detach(self, mock_hm_reg): def test_detach(self, mock_hm_unreg):
res, data = self.hp.detach(self.cluster) res, data = self.hp.detach(self.cluster)
self.assertTrue(res) self.assertTrue(res)
self.assertEqual('', data) self.assertEqual('', data)
mock_hm_reg.assert_called_once_with('CLUSTER_ID') mock_hm_unreg.assert_called_once_with('CLUSTER_ID')
@mock.patch.object(health_manager, 'unregister')
def test_detach_failed_with_notify_timeout(self, mock_hm_unreg):
mock_hm_unreg.return_value = False
res, data = self.hp.detach(self.cluster)
self.assertFalse(res)
self.assertEqual("Unregistering health manager for cluster timed "
"out.", data)
mock_hm_unreg.assert_called_once_with('CLUSTER_ID')
def test_pre_op_default(self): def test_pre_op_default(self):
action = mock.Mock(context='action_context', data={}, action = mock.Mock(context='action_context', data={},