diff --git a/heat/engine/service.py b/heat/engine/service.py index 49788d4da5..76be7032f6 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -13,6 +13,7 @@ import collections import datetime +import functools import itertools import os import socket @@ -1140,6 +1141,16 @@ class EngineService(service.Service): msg = _("Cancelling update when stack is %s") % str(state) raise exception.NotSupported(feature=msg) LOG.info(_LI('Starting cancel of updating stack %s'), db_stack.name) + + if current_stack.convergence: + if cancel_with_rollback: + func = current_stack.rollback + else: + func = functools.partial(self.worker_service.stop_traversal, + current_stack) + self.thread_group_mgr.start(current_stack.id, func) + return + # stop the running update and take the lock # as we cancel only running update, the acquire_result is # always some engine_id, not None diff --git a/heat/engine/stack.py b/heat/engine/stack.py index b09465ad8f..5482e4b18c 100644 --- a/heat/engine/stack.py +++ b/heat/engine/stack.py @@ -924,10 +924,17 @@ class Stack(collections.Mapping): self._send_notification_and_add_event() if self.convergence: # do things differently for convergence + exp_trvsl = self.current_traversal + if self.status == self.FAILED: + self.current_traversal = '' + values['current_traversal'] = self.current_traversal + updated = stack_object.Stack.select_and_update( self.context, self.id, values, - exp_trvsl=self.current_traversal) + exp_trvsl=exp_trvsl) + return updated + else: stack.update_and_save(values) @@ -2008,17 +2015,13 @@ class Stack(collections.Mapping): """ resource_objects.Resource.purge_deleted(self.context, self.id) - exp_trvsl = self.current_traversal - if self.status == self.FAILED: - self.current_traversal = '' - prev_tmpl_id = None if (self.prev_raw_template_id is not None and self.status != self.FAILED): prev_tmpl_id = self.prev_raw_template_id self.prev_raw_template_id = None - stack_id = self.store(exp_trvsl=exp_trvsl) + stack_id = self.store() if stack_id is None: # Failed concurrent update LOG.warning(_LW("Failed to store stack %(name)s with traversal ID " diff --git a/heat/engine/worker.py b/heat/engine/worker.py index d8d2645918..7702774de4 100644 --- a/heat/engine/worker.py +++ b/heat/engine/worker.py @@ -21,6 +21,7 @@ from osprofiler import profiler from heat.common import context from heat.common.i18n import _LE from heat.common.i18n import _LI +from heat.common.i18n import _LW from heat.common import messaging as rpc_messaging from heat.engine import check_resource from heat.engine import sync_point @@ -88,6 +89,22 @@ class WorkerService(service.Service): super(WorkerService, self).stop() + def stop_traversal(self, stack): + """Update current traversal to stop workers from propagating. + + Marks the stack as FAILED due to cancellation, but, allows all + in_progress resources to complete normally; no worker is stopped + abruptly. + """ + reason = 'User cancelled stack %s ' % stack.action + # state_set will update the current traversal to '' for FAILED state + old_trvsl = stack.current_traversal + updated = stack.state_set(stack.action, stack.FAILED, reason) + if not updated: + LOG.warning(_LW("Failed to stop traversal %(trvsl)s of stack " + "%(name)s while cancelling the operation."), + {'name': stack.name, 'trvsl': old_trvsl}) + @context.request_context def check_resource(self, cnxt, resource_id, current_traversal, data, is_update, adopt_stack_data): diff --git a/heat/tests/test_convg_stack.py b/heat/tests/test_convg_stack.py index 269acc04fa..5009d0efe6 100644 --- a/heat/tests/test_convg_stack.py +++ b/heat/tests/test_convg_stack.py @@ -370,14 +370,13 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase): stack.mark_complete() self.assertTrue(stack.purge_db.called) - def test_purge_db_sets_curr_trvsl_to_none_for_failed_stack( + def test_state_set_sets_empty_curr_trvsl_for_failed_stack( self, mock_cr): stack = tools.get_stack('test_stack', utils.dummy_context(), template=tools.string_template_five, convergence=True) - stack.status = stack.FAILED stack.store() - stack.purge_db() + stack.state_set(stack.action, stack.FAILED, 'test-reason') self.assertEqual('', stack.current_traversal) @mock.patch.object(raw_template_object.RawTemplate, 'delete') diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index 60a1bb0abe..8da2f4a5ea 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -1418,3 +1418,57 @@ class StackServiceTest(common.HeatTestCase): stack = self.eng._parse_template_and_validate_stack( self.ctx, 'stack_name', template, {}, {}, None, args) self.assertEqual(1, stack.parameters['volsize']) + + @mock.patch('heat.engine.service.ThreadGroupManager', + return_value=mock.Mock()) + @mock.patch.object(stack_object.Stack, 'get_by_id') + @mock.patch.object(parser.Stack, 'load') + def test_stack_cancel_update_convergence_with_no_rollback( + self, mock_load, mock_get_by_id, mock_tg): + stk = mock.MagicMock() + stk.id = 1 + stk.UPDATE = 'UPDATE' + stk.IN_PROGRESS = 'IN_PROGRESS' + stk.state = ('UPDATE', 'IN_PROGRESS') + stk.status = stk.IN_PROGRESS + stk.action = stk.UPDATE + stk.convergence = True + mock_load.return_value = stk + self.patchobject(self.eng, '_get_stack') + self.eng.thread_group_mgr.start = mock.MagicMock() + with mock.patch.object(self.eng, 'worker_service') as mock_ws: + mock_ws.stop_traversal = mock.Mock() + # with rollback as false + self.eng.stack_cancel_update(self.ctx, 1, + cancel_with_rollback=False) + self.assertTrue(self.eng.thread_group_mgr.start.called) + call_args, _ = self.eng.thread_group_mgr.start.call_args + # test ID of stack + self.assertEqual(call_args[0], 1) + # ensure stop_traversal should be called with stack + self.assertEqual(call_args[1].func, mock_ws.stop_traversal) + self.assertEqual(call_args[1].args[0], stk) + + @mock.patch('heat.engine.service.ThreadGroupManager', + return_value=mock.Mock()) + @mock.patch.object(stack_object.Stack, 'get_by_id') + @mock.patch.object(parser.Stack, 'load') + def test_stack_cancel_update_convergence_with_rollback( + self, mock_load, mock_get_by_id, mock_tg): + stk = mock.MagicMock() + stk.id = 1 + stk.UPDATE = 'UPDATE' + stk.IN_PROGRESS = 'IN_PROGRESS' + stk.state = ('UPDATE', 'IN_PROGRESS') + stk.status = stk.IN_PROGRESS + stk.action = stk.UPDATE + stk.convergence = True + stk.rollback = mock.MagicMock(return_value=None) + mock_load.return_value = stk + self.patchobject(self.eng, '_get_stack') + self.eng.thread_group_mgr.start = mock.MagicMock() + # with rollback as true + self.eng.stack_cancel_update(self.ctx, 1, + cancel_with_rollback=True) + self.eng.thread_group_mgr.start.assert_called_once_with( + 1, stk.rollback)