openstack/senlin
Code Issues Proposed changes

Add cluster option cluster.stop_node_before_delete

Browse Source 
This adds 'cluster.stop_node_before_delete' as a new boolean cluster
config option. If true, a cluster will issue a stop to a node and then a
delete for all cluster operations that trigger a node delete. If not
set, the config option defaults to False and the cluster will perform a
regular node delete. The cluster operations affected by this change are
cluster delete, scale-in and resize.

Change-Id: I2248419cc7b1e359c9a189fae97d7883bab3ec84
This commit is contained in:
Duc Truong 2018-06-26 20:50:08 +00:00
parent a19579a6c5
commit e7dae6c458
2 changed files with 212 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
senlin/engine/actions/cluster_action.py
View File

@ -305,7 +305,8 @@ class ClusterAction(base.Action):
ao.Action.update(self.context, action_id,
{'status': base.Action.READY})
def _delete_nodes_with_hook(self, action_name, node_ids, lifecycle_hook):
def _remove_nodes_with_hook(self, action_name, node_ids, lifecycle_hook,
inputs=None):
lifecycle_hook_timeout = lifecycle_hook.get('timeout')
lifecycle_hook_type = lifecycle_hook.get('type', None)
lifecycle_hook_params = lifecycle_hook.get('params')
@ -320,6 +321,7 @@ class ClusterAction(base.Action):
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': consts.CAUSE_DERIVED_LCH,
'inputs': inputs or {},
}
action_id = base.Action.create(self.context, node_id, action_name,
@ -378,12 +380,13 @@ class ClusterAction(base.Action):
return self.RES_OK, ''
def _delete_nodes_normally(self, action_name, node_ids):
def _remove_nodes_normally(self, action_name, node_ids, inputs=None):
child = []
for node_id in node_ids:
kwargs = {
'name': 'node_delete_%s' % node_id[:8],
'cause': consts.CAUSE_DERIVED,
'inputs': inputs or {},
}
action_id = base.Action.create(self.context, node_id, action_name,
@ -412,13 +415,30 @@ class ClusterAction(base.Action):
if destroy is False:
action_name = consts.NODE_LEAVE
stop_node_before_delete = self.entity.config.get(
"cluster.stop_node_before_delete", False)
# get lifecycle hook properties if specified
lifecycle_hook = self.data.get('hooks')
if lifecycle_hook:
res, reason = self._delete_nodes_with_hook(action_name, node_ids,
lifecycle_hook)
if stop_node_before_delete:
res, reason = self._remove_nodes_with_hook(
consts.NODE_OPERATION, node_ids, lifecycle_hook,
{'operation': 'stop'})
if res != self.RES_OK:
return res, 'Failed in stopping nodes:%s' % reason
res, reason = self._remove_nodes_normally(action_name,
node_ids)
else:
res, reason = self._delete_nodes_normally(action_name, node_ids)
res, reason = self._remove_nodes_with_hook(
action_name, node_ids, lifecycle_hook)
else:
if stop_node_before_delete:
res, reason = self._remove_nodes_normally(
consts.NODE_OPERATION, node_ids, {'operation': 'stop'})
if res != self.RES_OK:
return res, 'Failed in stopping nodes:%s' % reason
res, reason = self._remove_nodes_normally(action_name, node_ids)
if res == self.RES_OK:
self.outputs['nodes_removed'] = node_ids

225
senlin/tests/unit/engine/actions/test_delete.py
View File

@ -40,7 +40,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
def test__delete_nodes_single(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='FAKE_CLUSTER', desired_capacity=100)
cluster = mock.Mock(id='FAKE_CLUSTER', desired_capacity=100, config={})
# cluster action is real
mock_load.return_value = cluster
@ -58,7 +58,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual('All dependents completed', res_msg)
mock_action.assert_called_once_with(
action.context, 'NODE_ID', 'NODE_DELETE',
name='node_delete_NODE_ID', cause='Derived Action')
name='node_delete_NODE_ID', cause='Derived Action', inputs={})
mock_dep.assert_called_once_with(action.context, ['NODE_ACTION_ID'],
'CLUSTER_ACTION_ID')
mock_update.assert_called_once_with(action.context, 'NODE_ACTION_ID',
@ -68,6 +68,57 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual(['NODE_ID'], action.outputs['nodes_removed'])
cluster.remove_node.assert_called_once_with('NODE_ID')
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@mock.patch.object(dobj.Dependency, 'create')
@mock.patch.object(dispatcher, 'start_action')
@mock.patch.object(ca.ClusterAction, '_wait_for_dependents')
def test__delete_nodes_single_stop_node(self, mock_wait, mock_start,
mock_dep, mock_action, mock_update,
mock_load):
# prepare mocks
cluster = mock.Mock(id='FAKE_CLUSTER', desired_capacity=100,
config={'cluster.stop_node_before_delete': True})
# cluster action is real
mock_load.return_value = cluster
action = ca.ClusterAction(cluster.id, 'CLUSTER_ACTION', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
mock_wait.return_value = (action.RES_OK, 'All dependents completed')
mock_action.return_value = 'NODE_ACTION_ID'
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions
self.assertEqual(action.RES_OK, res_code)
self.assertEqual('All dependents completed', res_msg)
create_actions = [
mock.call(action.context, 'NODE_ID', 'NODE_OPERATION',
name='node_delete_NODE_ID',
cause='Derived Action',
inputs={'operation': 'stop'}),
mock.call(action.context, 'NODE_ID', 'NODE_DELETE',
name='node_delete_NODE_ID',
cause='Derived Action', inputs={})
]
mock_action.assert_has_calls(create_actions)
dep_calls = [
mock.call(action.context, ['NODE_ACTION_ID'], 'CLUSTER_ACTION_ID'),
mock.call(action.context, ['NODE_ACTION_ID'], 'CLUSTER_ACTION_ID'),
]
mock_dep.assert_has_calls(dep_calls)
update_calls = [
mock.call(action.context, 'NODE_ACTION_ID', {'status': 'READY'}),
mock.call(action.context, 'NODE_ACTION_ID', {'status': 'READY'})
]
mock_update.assert_has_calls(update_calls)
mock_start.assert_has_calls([mock.call(), mock.call()])
mock_wait.assert_has_calls([mock.call(), mock.call()])
self.assertEqual(['NODE_ID'], action.outputs['nodes_removed'])
cluster.remove_node.assert_called_once_with('NODE_ID')
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@mock.patch.object(dobj.Dependency, 'create')
@ -76,7 +127,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
def test__delete_nodes_multi(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
@ -128,7 +179,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
def test__delete_nodes_with_pd(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
@ -149,7 +200,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual('All dependents completed', res_msg)
mock_action.assert_called_once_with(
action.context, 'NODE_ID', 'NODE_LEAVE',
name='node_delete_NODE_ID', cause='Derived Action')
name='node_delete_NODE_ID', cause='Derived Action', inputs={})
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@ -164,7 +215,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_action, mock_update,
mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
@ -182,6 +233,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_action.return_value = 'NODE_ACTION_ID'
mock_node_get.return_value = mock.Mock(
status=consts.NS_ACTIVE, id='NODE_ID', physical_id="nova-server")
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
@ -191,8 +243,9 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual(1, mock_dep.call_count)
mock_action.assert_called_once_with(
action.context, 'NODE_ID', 'NODE_DELETE',
name='node_delete_NODE_ID', cause='Derived Action with '
'Lifecycle Hook')
name='node_delete_NODE_ID',
cause='Derived Action with Lifecycle Hook',
inputs={})
update_calls = [
mock.call(action.context, 'NODE_ACTION_ID',
{'status': 'WAITING_LIFECYCLE_COMPLETION'}),
@ -237,7 +290,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
self, mock_node_obj, mock_wait, mock_start, mock_post, mock_dep,
mock_node_get, mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
@ -263,8 +316,8 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual(1, mock_dep.call_count)
mock_action.assert_called_once_with(
action.context, 'NODE_ID', 'NODE_DELETE',
name='node_delete_NODE_ID', cause='Derived Action with '
'Lifecycle Hook')
name='node_delete_NODE_ID',
cause='Derived Action with Lifecycle Hook', inputs={})
update_calls = [
mock.call(action.context, 'NODE_ACTION_ID',
{'status': 'READY'}),
@ -291,7 +344,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_check_status,
mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
@ -322,8 +375,8 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual(1, mock_dep.call_count)
mock_action.assert_called_once_with(
action.context, 'NODE_ID', 'NODE_DELETE',
name='node_delete_NODE_ID', cause='Derived Action with '
'Lifecycle Hook')
name='node_delete_NODE_ID',
cause='Derived Action with Lifecycle Hook', inputs={})
update_calls = [
mock.call(action.context, 'NODE_ACTION_ID',
{'status': 'WAITING_LIFECYCLE_COMPLETION'}),
@ -346,7 +399,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_action,
mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
@ -376,7 +429,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_update,
mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100, config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
@ -399,31 +452,64 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual("Failed in deleting nodes:Lifecycle hook type "
"'webhook' is not implemented", res_msg)
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@mock.patch.object(dobj.Dependency, 'create')
@mock.patch.object(dispatcher, 'start_action')
@mock.patch.object(ca.ClusterAction, '_wait_for_dependents')
def test__delete_nodes_failed_wait(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
@mock.patch.object(ca.ClusterAction, '_remove_nodes_normally')
def test__delete_nodes_failed_remove_stop_node(self, mock_remove,
mock_load):
# prepare mocks
cluster = mock.Mock(id='ID')
cluster = mock.Mock(id='ID',
config={'cluster.stop_node_before_delete': True})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
action.data = {}
mock_wait.return_value = (action.RES_TIMEOUT, 'Timeout!')
mock_action.return_value = 'NODE_ACTION_ID'
mock_remove.return_value = (action.RES_TIMEOUT, 'Timeout!')
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_TIMEOUT, res_code)
self.assertEqual('Failed in deleting nodes:Timeout!', res_msg)
self.assertEqual('Failed in stopping nodes:Timeout!', res_msg)
self.assertEqual({}, action.data)
mock_remove.assert_called_once_with('NODE_OPERATION', ['NODE_ID'],
{'operation': 'stop'})
@mock.patch.object(ca.ClusterAction, '_remove_nodes_with_hook')
@mock.patch.object(ca.ClusterAction, '_remove_nodes_normally')
def test__delete_nodes_with_lifecycle_hook_failed_remove_stop_node(
self, mock_remove_normally, mock_remove_hook, mock_load):
# prepare mocks
cluster = mock.Mock(id='ID',
config={'cluster.stop_node_before_delete': True})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
lifecycle_hook = {
'timeout': 10,
'type': 'zaqar',
'params': {
'queue': 'myqueue'
}
}
action.data = {
'hooks': lifecycle_hook,
}
mock_remove_hook.return_value = (action.RES_TIMEOUT, 'Timeout!')
# do it
res_code, res_msg = action._delete_nodes(['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_TIMEOUT, res_code)
self.assertEqual('Failed in stopping nodes:Timeout!', res_msg)
mock_remove_hook.assert_called_once_with(
'NODE_OPERATION', ['NODE_ID'], lifecycle_hook,
{'operation': 'stop'})
mock_remove_normally.assert_not_called()
@mock.patch.object(ao.Action, 'delete_by_target')
def test_do_delete_success(self, mock_action, mock_load):
@ -683,7 +769,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
@mock.patch.object(dobj.Dependency, 'create')
@mock.patch.object(dispatcher, 'start_action')
@mock.patch.object(ca.ClusterAction, '_wait_for_dependents')
def test__delete_nodes_normally(self, mock_wait, mock_start, mock_dep,
def test__remove_nodes_normally(self, mock_wait, mock_start, mock_dep,
mock_action, mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='CLUSTER_ID', desired_capacity=100)
@ -697,7 +783,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_action.side_effect = ['NODE_ACTION_1', 'NODE_ACTION_2']
# do it
res_code, res_msg = action._delete_nodes_normally('NODE_REMOVE',
res_code, res_msg = action._remove_nodes_normally('NODE_REMOVE',
['NODE_1', 'NODE_2'])
# assertions
@ -720,7 +806,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
@mock.patch.object(msg.Message, 'post_lifecycle_hook_message')
@mock.patch.object(dispatcher, 'start_action')
@mock.patch.object(ca.ClusterAction, '_wait_for_dependents')
def test__delete_nodes_with_hook(self, mock_wait, mock_start, mock_post,
def test__remove_nodes_with_hook(self, mock_wait, mock_start, mock_post,
mock_dep, mock_node_get, mock_action,
mock_update, mock_load):
# prepare mocks
@ -743,7 +829,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_node_get.return_value = mock.Mock(
status=consts.NS_ACTIVE, id='NODE_ID', physical_id="nova-server")
# do it
res_code, res_msg = action._delete_nodes_with_hook(
res_code, res_msg = action._remove_nodes_with_hook(
'NODE_DELETE', ['NODE_ID'], action.data['hooks'])
# assertions (other assertions are skipped)
@ -752,8 +838,8 @@ class ClusterDeleteTest(base.SenlinTestCase):
self.assertEqual(1, mock_dep.call_count)
mock_action.assert_called_once_with(
action.context, 'NODE_ID', 'NODE_DELETE',
name='node_delete_NODE_ID', cause='Derived Action with '
'Lifecycle Hook')
name='node_delete_NODE_ID',
cause='Derived Action with Lifecycle Hook', inputs={})
update_calls = [
mock.call(action.context, 'NODE_ACTION_ID',
{'status': 'WAITING_LIFECYCLE_COMPLETION'}),
@ -765,6 +851,69 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_start.assert_called_once_with()
mock_wait.assert_called_once_with(action.data['hooks']['timeout'])
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@mock.patch.object(dobj.Dependency, 'create')
@mock.patch.object(dispatcher, 'start_action')
@mock.patch.object(ca.ClusterAction, '_wait_for_dependents')
def test__remove_nodes_normally_failed_wait(self, mock_wait, mock_start,
mock_dep, mock_action,
mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='ID', config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
action.data = {}
mock_wait.return_value = (action.RES_TIMEOUT, 'Timeout!')
mock_action.return_value = 'NODE_ACTION_ID'
# do it
res_code, res_msg = action._remove_nodes_normally('NODE_REMOVE',
['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_TIMEOUT, res_code)
self.assertEqual('Timeout!', res_msg)
self.assertEqual({}, action.data)
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@mock.patch.object(dobj.Dependency, 'create')
@mock.patch.object(dispatcher, 'start_action')
@mock.patch.object(ca.ClusterAction, '_wait_for_dependents')
def test__remove_nodes_hook_failed_wait(self, mock_wait, mock_start,
mock_dep, mock_action,
mock_update, mock_load):
# prepare mocks
cluster = mock.Mock(id='ID', config={})
mock_load.return_value = cluster
# cluster action is real
action = ca.ClusterAction(cluster.id, 'CLUSTER_DELETE', self.ctx)
action.id = 'CLUSTER_ACTION_ID'
action.inputs = {'destroy_after_deletion': False}
action.data = {
'hooks': {
'timeout': 10,
'type': 'zaqar',
'params': {
'queue': 'myqueue'
}
}
}
mock_wait.return_value = (action.RES_TIMEOUT, 'Timeout!')
mock_action.return_value = 'NODE_ACTION_ID'
# do it
res_code, res_msg = action._remove_nodes_normally('NODE_REMOVE',
['NODE_ID'])
# assertions (other assertions are skipped)
self.assertEqual(action.RES_TIMEOUT, res_code)
self.assertEqual('Timeout!', res_msg)
@mock.patch.object(ao.Action, 'update')
@mock.patch.object(ab.Action, 'create')
@mock.patch.object(no.Node, 'get')
@ -800,7 +949,7 @@ class ClusterDeleteTest(base.SenlinTestCase):
physical_id=None)
mock_node_get.side_effect = [node1, node2]
# do it
res_code, res_msg = action._delete_nodes_with_hook(
res_code, res_msg = action._remove_nodes_with_hook(
'NODE_DELETE', ['NODE_1', 'NODE_2'], action.data['hooks'])
# assertions
@ -814,11 +963,11 @@ class ClusterDeleteTest(base.SenlinTestCase):
mock_update.assert_has_calls(update_calls)
create_actions = [
mock.call(action.context, 'NODE_1', 'NODE_DELETE',
name='node_delete_NODE_1', cause='Derived Action with '
'Lifecycle Hook'),
name='node_delete_NODE_1',
cause='Derived Action with Lifecycle Hook', inputs={}),
mock.call(action.context, 'NODE_2', 'NODE_DELETE',
name='node_delete_NODE_2', cause='Derived Action with '
'Lifecycle Hook')
name='node_delete_NODE_2',
cause='Derived Action with Lifecycle Hook', inputs={})
]
mock_action.assert_has_calls(create_actions)