Convergence cancel update implementation

Implements:
(1) stack-cancel-update <stack_id> will start another update using the
previous template/environment. We'll start rolling back; in-progress
resources will be allowed to complete normally.
(2) stack-cancel-update <stack_id> --no-rollback will set the
traversal_id to None so no further resources will be updated;
in-progress resources will be allowed to complete normally.

Change-Id: I46ebdebb130be7410abe3e0b62f85da9856287b6
This commit is contained in:
Anant Patil 2016-08-18 17:28:29 +05:30
parent 78d56b1e0d
commit 084d0eb20f
5 changed files with 93 additions and 9 deletions

View File

@ -13,6 +13,7 @@
import collections
import datetime
import functools
import itertools
import os
import socket
@ -1140,6 +1141,16 @@ class EngineService(service.Service):
msg = _("Cancelling update when stack is %s") % str(state)
raise exception.NotSupported(feature=msg)
LOG.info(_LI('Starting cancel of updating stack %s'), db_stack.name)
if current_stack.convergence:
if cancel_with_rollback:
func = current_stack.rollback
else:
func = functools.partial(self.worker_service.stop_traversal,
current_stack)
self.thread_group_mgr.start(current_stack.id, func)
return
# stop the running update and take the lock
# as we cancel only running update, the acquire_result is
# always some engine_id, not None

View File

@ -924,10 +924,17 @@ class Stack(collections.Mapping):
self._send_notification_and_add_event()
if self.convergence:
# do things differently for convergence
exp_trvsl = self.current_traversal
if self.status == self.FAILED:
self.current_traversal = ''
values['current_traversal'] = self.current_traversal
updated = stack_object.Stack.select_and_update(
self.context, self.id, values,
exp_trvsl=self.current_traversal)
exp_trvsl=exp_trvsl)
return updated
else:
stack.update_and_save(values)
@ -2008,17 +2015,13 @@ class Stack(collections.Mapping):
"""
resource_objects.Resource.purge_deleted(self.context, self.id)
exp_trvsl = self.current_traversal
if self.status == self.FAILED:
self.current_traversal = ''
prev_tmpl_id = None
if (self.prev_raw_template_id is not None and
self.status != self.FAILED):
prev_tmpl_id = self.prev_raw_template_id
self.prev_raw_template_id = None
stack_id = self.store(exp_trvsl=exp_trvsl)
stack_id = self.store()
if stack_id is None:
# Failed concurrent update
LOG.warning(_LW("Failed to store stack %(name)s with traversal ID "

View File

@ -21,6 +21,7 @@ from osprofiler import profiler
from heat.common import context
from heat.common.i18n import _LE
from heat.common.i18n import _LI
from heat.common.i18n import _LW
from heat.common import messaging as rpc_messaging
from heat.engine import check_resource
from heat.engine import sync_point
@ -88,6 +89,22 @@ class WorkerService(service.Service):
super(WorkerService, self).stop()
def stop_traversal(self, stack):
"""Update current traversal to stop workers from propagating.
Marks the stack as FAILED due to cancellation, but, allows all
in_progress resources to complete normally; no worker is stopped
abruptly.
"""
reason = 'User cancelled stack %s ' % stack.action
# state_set will update the current traversal to '' for FAILED state
old_trvsl = stack.current_traversal
updated = stack.state_set(stack.action, stack.FAILED, reason)
if not updated:
LOG.warning(_LW("Failed to stop traversal %(trvsl)s of stack "
"%(name)s while cancelling the operation."),
{'name': stack.name, 'trvsl': old_trvsl})
@context.request_context
def check_resource(self, cnxt, resource_id, current_traversal, data,
is_update, adopt_stack_data):

View File

@ -370,14 +370,13 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
stack.mark_complete()
self.assertTrue(stack.purge_db.called)
def test_purge_db_sets_curr_trvsl_to_none_for_failed_stack(
def test_state_set_sets_empty_curr_trvsl_for_failed_stack(
self, mock_cr):
stack = tools.get_stack('test_stack', utils.dummy_context(),
template=tools.string_template_five,
convergence=True)
stack.status = stack.FAILED
stack.store()
stack.purge_db()
stack.state_set(stack.action, stack.FAILED, 'test-reason')
self.assertEqual('', stack.current_traversal)
@mock.patch.object(raw_template_object.RawTemplate, 'delete')

View File

@ -1418,3 +1418,57 @@ class StackServiceTest(common.HeatTestCase):
stack = self.eng._parse_template_and_validate_stack(
self.ctx, 'stack_name', template, {}, {}, None, args)
self.assertEqual(1, stack.parameters['volsize'])
@mock.patch('heat.engine.service.ThreadGroupManager',
return_value=mock.Mock())
@mock.patch.object(stack_object.Stack, 'get_by_id')
@mock.patch.object(parser.Stack, 'load')
def test_stack_cancel_update_convergence_with_no_rollback(
self, mock_load, mock_get_by_id, mock_tg):
stk = mock.MagicMock()
stk.id = 1
stk.UPDATE = 'UPDATE'
stk.IN_PROGRESS = 'IN_PROGRESS'
stk.state = ('UPDATE', 'IN_PROGRESS')
stk.status = stk.IN_PROGRESS
stk.action = stk.UPDATE
stk.convergence = True
mock_load.return_value = stk
self.patchobject(self.eng, '_get_stack')
self.eng.thread_group_mgr.start = mock.MagicMock()
with mock.patch.object(self.eng, 'worker_service') as mock_ws:
mock_ws.stop_traversal = mock.Mock()
# with rollback as false
self.eng.stack_cancel_update(self.ctx, 1,
cancel_with_rollback=False)
self.assertTrue(self.eng.thread_group_mgr.start.called)
call_args, _ = self.eng.thread_group_mgr.start.call_args
# test ID of stack
self.assertEqual(call_args[0], 1)
# ensure stop_traversal should be called with stack
self.assertEqual(call_args[1].func, mock_ws.stop_traversal)
self.assertEqual(call_args[1].args[0], stk)
@mock.patch('heat.engine.service.ThreadGroupManager',
return_value=mock.Mock())
@mock.patch.object(stack_object.Stack, 'get_by_id')
@mock.patch.object(parser.Stack, 'load')
def test_stack_cancel_update_convergence_with_rollback(
self, mock_load, mock_get_by_id, mock_tg):
stk = mock.MagicMock()
stk.id = 1
stk.UPDATE = 'UPDATE'
stk.IN_PROGRESS = 'IN_PROGRESS'
stk.state = ('UPDATE', 'IN_PROGRESS')
stk.status = stk.IN_PROGRESS
stk.action = stk.UPDATE
stk.convergence = True
stk.rollback = mock.MagicMock(return_value=None)
mock_load.return_value = stk
self.patchobject(self.eng, '_get_stack')
self.eng.thread_group_mgr.start = mock.MagicMock()
# with rollback as true
self.eng.stack_cancel_update(self.ctx, 1,
cancel_with_rollback=True)
self.eng.thread_group_mgr.start.assert_called_once_with(
1, stk.rollback)