Convergence cancel update implementation
Implements: (1) stack-cancel-update <stack_id> will start another update using the previous template/environment. We'll start rolling back; in-progress resources will be allowed to complete normally. (2) stack-cancel-update <stack_id> --no-rollback will set the traversal_id to None so no further resources will be updated; in-progress resources will be allowed to complete normally. Change-Id: I46ebdebb130be7410abe3e0b62f85da9856287b6
This commit is contained in:
parent
78d56b1e0d
commit
084d0eb20f
|
@ -13,6 +13,7 @@
|
||||||
|
|
||||||
import collections
|
import collections
|
||||||
import datetime
|
import datetime
|
||||||
|
import functools
|
||||||
import itertools
|
import itertools
|
||||||
import os
|
import os
|
||||||
import socket
|
import socket
|
||||||
|
@ -1140,6 +1141,16 @@ class EngineService(service.Service):
|
||||||
msg = _("Cancelling update when stack is %s") % str(state)
|
msg = _("Cancelling update when stack is %s") % str(state)
|
||||||
raise exception.NotSupported(feature=msg)
|
raise exception.NotSupported(feature=msg)
|
||||||
LOG.info(_LI('Starting cancel of updating stack %s'), db_stack.name)
|
LOG.info(_LI('Starting cancel of updating stack %s'), db_stack.name)
|
||||||
|
|
||||||
|
if current_stack.convergence:
|
||||||
|
if cancel_with_rollback:
|
||||||
|
func = current_stack.rollback
|
||||||
|
else:
|
||||||
|
func = functools.partial(self.worker_service.stop_traversal,
|
||||||
|
current_stack)
|
||||||
|
self.thread_group_mgr.start(current_stack.id, func)
|
||||||
|
return
|
||||||
|
|
||||||
# stop the running update and take the lock
|
# stop the running update and take the lock
|
||||||
# as we cancel only running update, the acquire_result is
|
# as we cancel only running update, the acquire_result is
|
||||||
# always some engine_id, not None
|
# always some engine_id, not None
|
||||||
|
|
|
@ -924,10 +924,17 @@ class Stack(collections.Mapping):
|
||||||
self._send_notification_and_add_event()
|
self._send_notification_and_add_event()
|
||||||
if self.convergence:
|
if self.convergence:
|
||||||
# do things differently for convergence
|
# do things differently for convergence
|
||||||
|
exp_trvsl = self.current_traversal
|
||||||
|
if self.status == self.FAILED:
|
||||||
|
self.current_traversal = ''
|
||||||
|
values['current_traversal'] = self.current_traversal
|
||||||
|
|
||||||
updated = stack_object.Stack.select_and_update(
|
updated = stack_object.Stack.select_and_update(
|
||||||
self.context, self.id, values,
|
self.context, self.id, values,
|
||||||
exp_trvsl=self.current_traversal)
|
exp_trvsl=exp_trvsl)
|
||||||
|
|
||||||
return updated
|
return updated
|
||||||
|
|
||||||
else:
|
else:
|
||||||
stack.update_and_save(values)
|
stack.update_and_save(values)
|
||||||
|
|
||||||
|
@ -2008,17 +2015,13 @@ class Stack(collections.Mapping):
|
||||||
"""
|
"""
|
||||||
resource_objects.Resource.purge_deleted(self.context, self.id)
|
resource_objects.Resource.purge_deleted(self.context, self.id)
|
||||||
|
|
||||||
exp_trvsl = self.current_traversal
|
|
||||||
if self.status == self.FAILED:
|
|
||||||
self.current_traversal = ''
|
|
||||||
|
|
||||||
prev_tmpl_id = None
|
prev_tmpl_id = None
|
||||||
if (self.prev_raw_template_id is not None and
|
if (self.prev_raw_template_id is not None and
|
||||||
self.status != self.FAILED):
|
self.status != self.FAILED):
|
||||||
prev_tmpl_id = self.prev_raw_template_id
|
prev_tmpl_id = self.prev_raw_template_id
|
||||||
self.prev_raw_template_id = None
|
self.prev_raw_template_id = None
|
||||||
|
|
||||||
stack_id = self.store(exp_trvsl=exp_trvsl)
|
stack_id = self.store()
|
||||||
if stack_id is None:
|
if stack_id is None:
|
||||||
# Failed concurrent update
|
# Failed concurrent update
|
||||||
LOG.warning(_LW("Failed to store stack %(name)s with traversal ID "
|
LOG.warning(_LW("Failed to store stack %(name)s with traversal ID "
|
||||||
|
|
|
@ -21,6 +21,7 @@ from osprofiler import profiler
|
||||||
from heat.common import context
|
from heat.common import context
|
||||||
from heat.common.i18n import _LE
|
from heat.common.i18n import _LE
|
||||||
from heat.common.i18n import _LI
|
from heat.common.i18n import _LI
|
||||||
|
from heat.common.i18n import _LW
|
||||||
from heat.common import messaging as rpc_messaging
|
from heat.common import messaging as rpc_messaging
|
||||||
from heat.engine import check_resource
|
from heat.engine import check_resource
|
||||||
from heat.engine import sync_point
|
from heat.engine import sync_point
|
||||||
|
@ -88,6 +89,22 @@ class WorkerService(service.Service):
|
||||||
|
|
||||||
super(WorkerService, self).stop()
|
super(WorkerService, self).stop()
|
||||||
|
|
||||||
|
def stop_traversal(self, stack):
|
||||||
|
"""Update current traversal to stop workers from propagating.
|
||||||
|
|
||||||
|
Marks the stack as FAILED due to cancellation, but, allows all
|
||||||
|
in_progress resources to complete normally; no worker is stopped
|
||||||
|
abruptly.
|
||||||
|
"""
|
||||||
|
reason = 'User cancelled stack %s ' % stack.action
|
||||||
|
# state_set will update the current traversal to '' for FAILED state
|
||||||
|
old_trvsl = stack.current_traversal
|
||||||
|
updated = stack.state_set(stack.action, stack.FAILED, reason)
|
||||||
|
if not updated:
|
||||||
|
LOG.warning(_LW("Failed to stop traversal %(trvsl)s of stack "
|
||||||
|
"%(name)s while cancelling the operation."),
|
||||||
|
{'name': stack.name, 'trvsl': old_trvsl})
|
||||||
|
|
||||||
@context.request_context
|
@context.request_context
|
||||||
def check_resource(self, cnxt, resource_id, current_traversal, data,
|
def check_resource(self, cnxt, resource_id, current_traversal, data,
|
||||||
is_update, adopt_stack_data):
|
is_update, adopt_stack_data):
|
||||||
|
|
|
@ -370,14 +370,13 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
|
||||||
stack.mark_complete()
|
stack.mark_complete()
|
||||||
self.assertTrue(stack.purge_db.called)
|
self.assertTrue(stack.purge_db.called)
|
||||||
|
|
||||||
def test_purge_db_sets_curr_trvsl_to_none_for_failed_stack(
|
def test_state_set_sets_empty_curr_trvsl_for_failed_stack(
|
||||||
self, mock_cr):
|
self, mock_cr):
|
||||||
stack = tools.get_stack('test_stack', utils.dummy_context(),
|
stack = tools.get_stack('test_stack', utils.dummy_context(),
|
||||||
template=tools.string_template_five,
|
template=tools.string_template_five,
|
||||||
convergence=True)
|
convergence=True)
|
||||||
stack.status = stack.FAILED
|
|
||||||
stack.store()
|
stack.store()
|
||||||
stack.purge_db()
|
stack.state_set(stack.action, stack.FAILED, 'test-reason')
|
||||||
self.assertEqual('', stack.current_traversal)
|
self.assertEqual('', stack.current_traversal)
|
||||||
|
|
||||||
@mock.patch.object(raw_template_object.RawTemplate, 'delete')
|
@mock.patch.object(raw_template_object.RawTemplate, 'delete')
|
||||||
|
|
|
@ -1418,3 +1418,57 @@ class StackServiceTest(common.HeatTestCase):
|
||||||
stack = self.eng._parse_template_and_validate_stack(
|
stack = self.eng._parse_template_and_validate_stack(
|
||||||
self.ctx, 'stack_name', template, {}, {}, None, args)
|
self.ctx, 'stack_name', template, {}, {}, None, args)
|
||||||
self.assertEqual(1, stack.parameters['volsize'])
|
self.assertEqual(1, stack.parameters['volsize'])
|
||||||
|
|
||||||
|
@mock.patch('heat.engine.service.ThreadGroupManager',
|
||||||
|
return_value=mock.Mock())
|
||||||
|
@mock.patch.object(stack_object.Stack, 'get_by_id')
|
||||||
|
@mock.patch.object(parser.Stack, 'load')
|
||||||
|
def test_stack_cancel_update_convergence_with_no_rollback(
|
||||||
|
self, mock_load, mock_get_by_id, mock_tg):
|
||||||
|
stk = mock.MagicMock()
|
||||||
|
stk.id = 1
|
||||||
|
stk.UPDATE = 'UPDATE'
|
||||||
|
stk.IN_PROGRESS = 'IN_PROGRESS'
|
||||||
|
stk.state = ('UPDATE', 'IN_PROGRESS')
|
||||||
|
stk.status = stk.IN_PROGRESS
|
||||||
|
stk.action = stk.UPDATE
|
||||||
|
stk.convergence = True
|
||||||
|
mock_load.return_value = stk
|
||||||
|
self.patchobject(self.eng, '_get_stack')
|
||||||
|
self.eng.thread_group_mgr.start = mock.MagicMock()
|
||||||
|
with mock.patch.object(self.eng, 'worker_service') as mock_ws:
|
||||||
|
mock_ws.stop_traversal = mock.Mock()
|
||||||
|
# with rollback as false
|
||||||
|
self.eng.stack_cancel_update(self.ctx, 1,
|
||||||
|
cancel_with_rollback=False)
|
||||||
|
self.assertTrue(self.eng.thread_group_mgr.start.called)
|
||||||
|
call_args, _ = self.eng.thread_group_mgr.start.call_args
|
||||||
|
# test ID of stack
|
||||||
|
self.assertEqual(call_args[0], 1)
|
||||||
|
# ensure stop_traversal should be called with stack
|
||||||
|
self.assertEqual(call_args[1].func, mock_ws.stop_traversal)
|
||||||
|
self.assertEqual(call_args[1].args[0], stk)
|
||||||
|
|
||||||
|
@mock.patch('heat.engine.service.ThreadGroupManager',
|
||||||
|
return_value=mock.Mock())
|
||||||
|
@mock.patch.object(stack_object.Stack, 'get_by_id')
|
||||||
|
@mock.patch.object(parser.Stack, 'load')
|
||||||
|
def test_stack_cancel_update_convergence_with_rollback(
|
||||||
|
self, mock_load, mock_get_by_id, mock_tg):
|
||||||
|
stk = mock.MagicMock()
|
||||||
|
stk.id = 1
|
||||||
|
stk.UPDATE = 'UPDATE'
|
||||||
|
stk.IN_PROGRESS = 'IN_PROGRESS'
|
||||||
|
stk.state = ('UPDATE', 'IN_PROGRESS')
|
||||||
|
stk.status = stk.IN_PROGRESS
|
||||||
|
stk.action = stk.UPDATE
|
||||||
|
stk.convergence = True
|
||||||
|
stk.rollback = mock.MagicMock(return_value=None)
|
||||||
|
mock_load.return_value = stk
|
||||||
|
self.patchobject(self.eng, '_get_stack')
|
||||||
|
self.eng.thread_group_mgr.start = mock.MagicMock()
|
||||||
|
# with rollback as true
|
||||||
|
self.eng.stack_cancel_update(self.ctx, 1,
|
||||||
|
cancel_with_rollback=True)
|
||||||
|
self.eng.thread_group_mgr.start.assert_called_once_with(
|
||||||
|
1, stk.rollback)
|
||||||
|
|
Loading…
Reference in New Issue