Convergence cancel update implementation
Implements: (1) stack-cancel-update <stack_id> will start another update using the previous template/environment. We'll start rolling back; in-progress resources will be allowed to complete normally. (2) stack-cancel-update <stack_id> --no-rollback will set the traversal_id to None so no further resources will be updated; in-progress resources will be allowed to complete normally. Change-Id: I46ebdebb130be7410abe3e0b62f85da9856287b6
This commit is contained in:
parent
78d56b1e0d
commit
084d0eb20f
|
@ -13,6 +13,7 @@
|
|||
|
||||
import collections
|
||||
import datetime
|
||||
import functools
|
||||
import itertools
|
||||
import os
|
||||
import socket
|
||||
|
@ -1140,6 +1141,16 @@ class EngineService(service.Service):
|
|||
msg = _("Cancelling update when stack is %s") % str(state)
|
||||
raise exception.NotSupported(feature=msg)
|
||||
LOG.info(_LI('Starting cancel of updating stack %s'), db_stack.name)
|
||||
|
||||
if current_stack.convergence:
|
||||
if cancel_with_rollback:
|
||||
func = current_stack.rollback
|
||||
else:
|
||||
func = functools.partial(self.worker_service.stop_traversal,
|
||||
current_stack)
|
||||
self.thread_group_mgr.start(current_stack.id, func)
|
||||
return
|
||||
|
||||
# stop the running update and take the lock
|
||||
# as we cancel only running update, the acquire_result is
|
||||
# always some engine_id, not None
|
||||
|
|
|
@ -924,10 +924,17 @@ class Stack(collections.Mapping):
|
|||
self._send_notification_and_add_event()
|
||||
if self.convergence:
|
||||
# do things differently for convergence
|
||||
exp_trvsl = self.current_traversal
|
||||
if self.status == self.FAILED:
|
||||
self.current_traversal = ''
|
||||
values['current_traversal'] = self.current_traversal
|
||||
|
||||
updated = stack_object.Stack.select_and_update(
|
||||
self.context, self.id, values,
|
||||
exp_trvsl=self.current_traversal)
|
||||
exp_trvsl=exp_trvsl)
|
||||
|
||||
return updated
|
||||
|
||||
else:
|
||||
stack.update_and_save(values)
|
||||
|
||||
|
@ -2008,17 +2015,13 @@ class Stack(collections.Mapping):
|
|||
"""
|
||||
resource_objects.Resource.purge_deleted(self.context, self.id)
|
||||
|
||||
exp_trvsl = self.current_traversal
|
||||
if self.status == self.FAILED:
|
||||
self.current_traversal = ''
|
||||
|
||||
prev_tmpl_id = None
|
||||
if (self.prev_raw_template_id is not None and
|
||||
self.status != self.FAILED):
|
||||
prev_tmpl_id = self.prev_raw_template_id
|
||||
self.prev_raw_template_id = None
|
||||
|
||||
stack_id = self.store(exp_trvsl=exp_trvsl)
|
||||
stack_id = self.store()
|
||||
if stack_id is None:
|
||||
# Failed concurrent update
|
||||
LOG.warning(_LW("Failed to store stack %(name)s with traversal ID "
|
||||
|
|
|
@ -21,6 +21,7 @@ from osprofiler import profiler
|
|||
from heat.common import context
|
||||
from heat.common.i18n import _LE
|
||||
from heat.common.i18n import _LI
|
||||
from heat.common.i18n import _LW
|
||||
from heat.common import messaging as rpc_messaging
|
||||
from heat.engine import check_resource
|
||||
from heat.engine import sync_point
|
||||
|
@ -88,6 +89,22 @@ class WorkerService(service.Service):
|
|||
|
||||
super(WorkerService, self).stop()
|
||||
|
||||
def stop_traversal(self, stack):
|
||||
"""Update current traversal to stop workers from propagating.
|
||||
|
||||
Marks the stack as FAILED due to cancellation, but, allows all
|
||||
in_progress resources to complete normally; no worker is stopped
|
||||
abruptly.
|
||||
"""
|
||||
reason = 'User cancelled stack %s ' % stack.action
|
||||
# state_set will update the current traversal to '' for FAILED state
|
||||
old_trvsl = stack.current_traversal
|
||||
updated = stack.state_set(stack.action, stack.FAILED, reason)
|
||||
if not updated:
|
||||
LOG.warning(_LW("Failed to stop traversal %(trvsl)s of stack "
|
||||
"%(name)s while cancelling the operation."),
|
||||
{'name': stack.name, 'trvsl': old_trvsl})
|
||||
|
||||
@context.request_context
|
||||
def check_resource(self, cnxt, resource_id, current_traversal, data,
|
||||
is_update, adopt_stack_data):
|
||||
|
|
|
@ -370,14 +370,13 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
|
|||
stack.mark_complete()
|
||||
self.assertTrue(stack.purge_db.called)
|
||||
|
||||
def test_purge_db_sets_curr_trvsl_to_none_for_failed_stack(
|
||||
def test_state_set_sets_empty_curr_trvsl_for_failed_stack(
|
||||
self, mock_cr):
|
||||
stack = tools.get_stack('test_stack', utils.dummy_context(),
|
||||
template=tools.string_template_five,
|
||||
convergence=True)
|
||||
stack.status = stack.FAILED
|
||||
stack.store()
|
||||
stack.purge_db()
|
||||
stack.state_set(stack.action, stack.FAILED, 'test-reason')
|
||||
self.assertEqual('', stack.current_traversal)
|
||||
|
||||
@mock.patch.object(raw_template_object.RawTemplate, 'delete')
|
||||
|
|
|
@ -1418,3 +1418,57 @@ class StackServiceTest(common.HeatTestCase):
|
|||
stack = self.eng._parse_template_and_validate_stack(
|
||||
self.ctx, 'stack_name', template, {}, {}, None, args)
|
||||
self.assertEqual(1, stack.parameters['volsize'])
|
||||
|
||||
@mock.patch('heat.engine.service.ThreadGroupManager',
|
||||
return_value=mock.Mock())
|
||||
@mock.patch.object(stack_object.Stack, 'get_by_id')
|
||||
@mock.patch.object(parser.Stack, 'load')
|
||||
def test_stack_cancel_update_convergence_with_no_rollback(
|
||||
self, mock_load, mock_get_by_id, mock_tg):
|
||||
stk = mock.MagicMock()
|
||||
stk.id = 1
|
||||
stk.UPDATE = 'UPDATE'
|
||||
stk.IN_PROGRESS = 'IN_PROGRESS'
|
||||
stk.state = ('UPDATE', 'IN_PROGRESS')
|
||||
stk.status = stk.IN_PROGRESS
|
||||
stk.action = stk.UPDATE
|
||||
stk.convergence = True
|
||||
mock_load.return_value = stk
|
||||
self.patchobject(self.eng, '_get_stack')
|
||||
self.eng.thread_group_mgr.start = mock.MagicMock()
|
||||
with mock.patch.object(self.eng, 'worker_service') as mock_ws:
|
||||
mock_ws.stop_traversal = mock.Mock()
|
||||
# with rollback as false
|
||||
self.eng.stack_cancel_update(self.ctx, 1,
|
||||
cancel_with_rollback=False)
|
||||
self.assertTrue(self.eng.thread_group_mgr.start.called)
|
||||
call_args, _ = self.eng.thread_group_mgr.start.call_args
|
||||
# test ID of stack
|
||||
self.assertEqual(call_args[0], 1)
|
||||
# ensure stop_traversal should be called with stack
|
||||
self.assertEqual(call_args[1].func, mock_ws.stop_traversal)
|
||||
self.assertEqual(call_args[1].args[0], stk)
|
||||
|
||||
@mock.patch('heat.engine.service.ThreadGroupManager',
|
||||
return_value=mock.Mock())
|
||||
@mock.patch.object(stack_object.Stack, 'get_by_id')
|
||||
@mock.patch.object(parser.Stack, 'load')
|
||||
def test_stack_cancel_update_convergence_with_rollback(
|
||||
self, mock_load, mock_get_by_id, mock_tg):
|
||||
stk = mock.MagicMock()
|
||||
stk.id = 1
|
||||
stk.UPDATE = 'UPDATE'
|
||||
stk.IN_PROGRESS = 'IN_PROGRESS'
|
||||
stk.state = ('UPDATE', 'IN_PROGRESS')
|
||||
stk.status = stk.IN_PROGRESS
|
||||
stk.action = stk.UPDATE
|
||||
stk.convergence = True
|
||||
stk.rollback = mock.MagicMock(return_value=None)
|
||||
mock_load.return_value = stk
|
||||
self.patchobject(self.eng, '_get_stack')
|
||||
self.eng.thread_group_mgr.start = mock.MagicMock()
|
||||
# with rollback as true
|
||||
self.eng.stack_cancel_update(self.ctx, 1,
|
||||
cancel_with_rollback=True)
|
||||
self.eng.thread_group_mgr.start.assert_called_once_with(
|
||||
1, stk.rollback)
|
||||
|
|
Loading…
Reference in New Issue