Merge "Convergence: Handle resource failure"
This commit is contained in:
commit
b4ae1aeefa
|
@ -1588,18 +1588,27 @@ class Stack(collections.Mapping):
|
|||
LOG.info('[%s(%s)] update traversal %s complete',
|
||||
self.name, self.id, traversal_id)
|
||||
|
||||
prev_prev_id = self.prev_raw_template_id
|
||||
self.prev_raw_template_id = None
|
||||
self.store()
|
||||
|
||||
if (prev_prev_id is not None and
|
||||
prev_prev_id != self.t.id):
|
||||
raw_template_object.RawTemplate.delete(self.context,
|
||||
prev_prev_id)
|
||||
|
||||
reason = 'Stack %s completed successfully' % self.action
|
||||
self.state_set(self.action, self.COMPLETE, reason)
|
||||
if self.action == self.DELETE:
|
||||
self.purge_db()
|
||||
|
||||
def purge_db(self):
|
||||
'''Cleanup database after stack has completed/failed.
|
||||
|
||||
1. Delete previous raw template if stack completes successfully.
|
||||
2. Deletes all sync points. They are no longer needed after stack
|
||||
has completed/failed.
|
||||
3. Delete the stack is the action is DELETE.
|
||||
'''
|
||||
if self.prev_raw_template_id is not None:
|
||||
prev_tmpl_id = self.prev_raw_template_id
|
||||
self.prev_raw_template_id = None
|
||||
self.store()
|
||||
raw_template_object.RawTemplate.delete(self.context, prev_tmpl_id)
|
||||
|
||||
sync_point.delete_all(self.context, self.id, self.current_traversal)
|
||||
|
||||
if (self.action, self.status) == (self.DELETE, self.COMPLETE):
|
||||
try:
|
||||
stack_object.Stack.delete(self.context, self.id)
|
||||
except exception.NotFound:
|
||||
|
|
|
@ -26,6 +26,7 @@ from heat.common.i18n import _LI
|
|||
from heat.common import messaging as rpc_messaging
|
||||
from heat.engine import dependencies
|
||||
from heat.engine import resource
|
||||
from heat.engine import stack as parser
|
||||
from heat.engine import sync_point
|
||||
from heat.rpc import worker_client as rpc_client
|
||||
|
||||
|
@ -83,6 +84,24 @@ class WorkerService(service.Service):
|
|||
|
||||
super(WorkerService, self).stop()
|
||||
|
||||
def _trigger_rollback(self, cnxt, stack):
|
||||
# TODO(ananta) convergence-rollback implementation
|
||||
pass
|
||||
|
||||
def _handle_resource_failure(self, cnxt, stack_id, traversal_id,
|
||||
failure_reason):
|
||||
stack = parser.Stack.load(cnxt, stack_id=stack_id)
|
||||
# make sure no new stack operation was triggered
|
||||
if stack.current_traversal != traversal_id:
|
||||
return
|
||||
|
||||
stack.state_set(stack.action, stack.FAILED, failure_reason)
|
||||
if (not stack.disable_rollback and
|
||||
stack.action in (stack.CREATE, stack.UPDATE)):
|
||||
self._trigger_rollback(cnxt, stack)
|
||||
else:
|
||||
stack.purge_db()
|
||||
|
||||
@context.request_context
|
||||
def check_resource(self, cnxt, resource_id, current_traversal, data,
|
||||
is_update):
|
||||
|
@ -127,6 +146,11 @@ class WorkerService(service.Service):
|
|||
return
|
||||
except resource.UpdateInProgress:
|
||||
return
|
||||
except exception.ResourceFailure as e:
|
||||
reason = six.text_type(e)
|
||||
self._handle_resource_failure(
|
||||
cnxt, stack.id, current_traversal, reason)
|
||||
return
|
||||
|
||||
input_data = construct_input_data(rsrc)
|
||||
else:
|
||||
|
@ -134,6 +158,11 @@ class WorkerService(service.Service):
|
|||
check_resource_cleanup(rsrc, tmpl.id, data)
|
||||
except resource.UpdateInProgress:
|
||||
return
|
||||
except exception.ResourceFailure as e:
|
||||
reason = six.text_type(e)
|
||||
self._handle_resource_failure(
|
||||
cnxt, stack.id, current_traversal, reason)
|
||||
return
|
||||
|
||||
graph_key = (rsrc.id, is_update)
|
||||
if graph_key not in graph and rsrc.replaces is not None:
|
||||
|
|
|
@ -491,6 +491,50 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
|
|||
{}, is_update))
|
||||
self.assertEqual(expected_calls, mock_cr.mock_calls)
|
||||
|
||||
def test_mark_complete_purges_db(self, mock_cr):
|
||||
stack = tools.get_stack('test_stack', utils.dummy_context(),
|
||||
template=tools.string_template_five,
|
||||
convergence=True)
|
||||
stack.store()
|
||||
stack.purge_db = mock.Mock()
|
||||
stack.mark_complete(stack.current_traversal)
|
||||
self.assertTrue(stack.purge_db.called)
|
||||
|
||||
def test_purge_db_deletes_previous_template(self, mock_cr):
|
||||
stack = tools.get_stack('test_stack', utils.dummy_context(),
|
||||
template=tools.string_template_five,
|
||||
convergence=True)
|
||||
prev_tmpl = templatem.Template.create_empty_template()
|
||||
prev_tmpl.store()
|
||||
stack.prev_raw_template_id = prev_tmpl.id
|
||||
stack.store()
|
||||
stack.purge_db()
|
||||
self.assertRaises(exception.NotFound,
|
||||
templatem.Template.load,
|
||||
stack.context, prev_tmpl.id)
|
||||
|
||||
def test_purge_db_deletes_sync_points(self, mock_cr):
|
||||
stack = tools.get_stack('test_stack', utils.dummy_context(),
|
||||
template=tools.string_template_five,
|
||||
convergence=True)
|
||||
stack.store()
|
||||
stack.purge_db()
|
||||
rows = sync_point_object.SyncPoint.delete_all_by_stack_and_traversal(
|
||||
stack.context, stack.id, stack.current_traversal)
|
||||
self.assertEqual(0, rows)
|
||||
|
||||
def test_purge_db_deletes_stack_for_deleted_stack(self, mock_cr):
|
||||
stack = tools.get_stack('test_stack', utils.dummy_context(),
|
||||
template=tools.string_template_five,
|
||||
convergence=True)
|
||||
stack.store()
|
||||
stack.state_set(stack.DELETE, stack.COMPLETE, 'test reason')
|
||||
stack.purge_db()
|
||||
self.assertRaises(exception.NotFound,
|
||||
parser.Stack.load,
|
||||
stack.context, stack_id=stack.id,
|
||||
show_deleted=False)
|
||||
|
||||
|
||||
class StackCreateTest(common.HeatTestCase):
|
||||
def setUp(self):
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
import mock
|
||||
|
||||
from heat.common import exception
|
||||
from heat.engine import resource
|
||||
from heat.engine import stack
|
||||
from heat.engine import sync_point
|
||||
|
@ -112,6 +113,12 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
|||
self.resource = self.stack['A']
|
||||
self.is_update = True
|
||||
self.graph_key = (self.resource.id, self.is_update)
|
||||
self.orig_load_method = stack.Stack.load
|
||||
stack.Stack.load = mock.Mock(return_value=self.stack)
|
||||
|
||||
def tearDown(self):
|
||||
super(CheckWorkflowUpdateTest, self).tearDown()
|
||||
stack.Stack.load = self.orig_load_method
|
||||
|
||||
def test_resource_not_available(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
|
@ -181,6 +188,135 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
|||
self.assertFalse(mock_pcr.called)
|
||||
self.assertFalse(mock_csc.called)
|
||||
|
||||
def test_resource_update_failure_sets_stack_state_as_failed(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.stack.state_set(self.stack.UPDATE, self.stack.IN_PROGRESS, '')
|
||||
self.worker._trigger_rollback = mock.Mock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_cru.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.resource.UPDATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
s = self.stack.load(self.ctx, stack_id=self.stack.id)
|
||||
self.assertEqual((s.UPDATE, s.FAILED), (s.action, s.status))
|
||||
self.assertEqual(u'ResourceNotAvailable: resources.A: The Resource (A)'
|
||||
' is not available.', s.status_reason)
|
||||
|
||||
def test_resource_cleanup_failure_sets_stack_state_as_failed(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.is_update = False # invokes check_resource_cleanup
|
||||
self.stack.state_set(self.stack.UPDATE, self.stack.IN_PROGRESS, '')
|
||||
self.worker._trigger_rollback = mock.Mock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_crc.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.resource.UPDATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
s = self.stack.load(self.ctx, stack_id=self.stack.id)
|
||||
self.assertEqual((s.UPDATE, s.FAILED), (s.action, s.status))
|
||||
self.assertEqual(u'ResourceNotAvailable: resources.A: The Resource (A)'
|
||||
' is not available.', s.status_reason)
|
||||
|
||||
def test_resource_update_failure_triggers_rollback_if_enabled(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.stack.disable_rollback = False
|
||||
self.stack.store()
|
||||
self.worker._trigger_rollback = mock.Mock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_cru.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.resource.UPDATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
self.assertTrue(self.worker._trigger_rollback.called)
|
||||
# make sure the rollback is called on given stack
|
||||
call_args, call_kwargs = self.worker._trigger_rollback.call_args
|
||||
called_stack = call_args[1]
|
||||
self.assertEqual(self.stack.id, called_stack.id)
|
||||
|
||||
def test_resource_cleanup_failure_triggers_rollback_if_enabled(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.is_update = False # invokes check_resource_cleanup
|
||||
self.stack.disable_rollback = False
|
||||
self.stack.store()
|
||||
self.worker._trigger_rollback = mock.Mock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_crc.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.resource.UPDATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
self.assertTrue(self.worker._trigger_rollback.called)
|
||||
# make sure the rollback is called on given stack
|
||||
call_args, call_kwargs = self.worker._trigger_rollback.call_args
|
||||
called_stack = call_args[1]
|
||||
self.assertEqual(self.stack.id, called_stack.id)
|
||||
|
||||
def test_rollback_is_not_triggered_on_rollback_disabled_stack(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.stack.disable_rollback = True
|
||||
self.stack.store()
|
||||
self.worker._trigger_rollback = mock.Mock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_cru.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.stack.CREATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
self.assertFalse(self.worker._trigger_rollback.called)
|
||||
|
||||
def test_rollback_not_re_triggered_for_a_rolling_back_stack(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.stack.disable_rollback = False
|
||||
self.stack.action = self.stack.ROLLBACK
|
||||
self.stack.status = self.stack.IN_PROGRESS
|
||||
self.stack.store()
|
||||
self.worker._trigger_rollback = mock.MagicMock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_cru.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.stack.CREATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
self.assertFalse(self.worker._trigger_rollback.called)
|
||||
|
||||
def test_resource_update_failure_purges_db_for_stack_failure(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.stack.disable_rollback = True
|
||||
self.stack.store()
|
||||
self.stack.purge_db = mock.Mock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_cru.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.resource.UPDATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
self.assertTrue(self.stack.purge_db.called)
|
||||
|
||||
def test_resource_cleanup_failure_purges_db_for_stack_failure(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
self.is_update = False
|
||||
self.stack.disable_rollback = True
|
||||
self.stack.store()
|
||||
self.stack.purge_db = mock.Mock()
|
||||
dummy_ex = exception.ResourceNotAvailable(
|
||||
resource_name=self.resource.name)
|
||||
mock_crc.side_effect = exception.ResourceFailure(
|
||||
dummy_ex, self.resource, action=self.resource.UPDATE)
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal, {},
|
||||
self.is_update)
|
||||
self.assertTrue(self.stack.purge_db.called)
|
||||
|
||||
|
||||
@mock.patch.object(worker, 'construct_input_data')
|
||||
@mock.patch.object(worker, 'check_stack_complete')
|
||||
|
|
Loading…
Reference in New Issue