Merge "Convergence: Handle resource failure"

This commit is contained in:
Jenkins 2015-06-25 00:15:34 +00:00 committed by Gerrit Code Review
commit b4ae1aeefa
4 changed files with 228 additions and 10 deletions

View File

@ -1588,18 +1588,27 @@ class Stack(collections.Mapping):
LOG.info('[%s(%s)] update traversal %s complete',
self.name, self.id, traversal_id)
prev_prev_id = self.prev_raw_template_id
self.prev_raw_template_id = None
self.store()
if (prev_prev_id is not None and
prev_prev_id != self.t.id):
raw_template_object.RawTemplate.delete(self.context,
prev_prev_id)
reason = 'Stack %s completed successfully' % self.action
self.state_set(self.action, self.COMPLETE, reason)
if self.action == self.DELETE:
self.purge_db()
def purge_db(self):
'''Cleanup database after stack has completed/failed.
1. Delete previous raw template if stack completes successfully.
2. Deletes all sync points. They are no longer needed after stack
has completed/failed.
3. Delete the stack is the action is DELETE.
'''
if self.prev_raw_template_id is not None:
prev_tmpl_id = self.prev_raw_template_id
self.prev_raw_template_id = None
self.store()
raw_template_object.RawTemplate.delete(self.context, prev_tmpl_id)
sync_point.delete_all(self.context, self.id, self.current_traversal)
if (self.action, self.status) == (self.DELETE, self.COMPLETE):
try:
stack_object.Stack.delete(self.context, self.id)
except exception.NotFound:

View File

@ -26,6 +26,7 @@ from heat.common.i18n import _LI
from heat.common import messaging as rpc_messaging
from heat.engine import dependencies
from heat.engine import resource
from heat.engine import stack as parser
from heat.engine import sync_point
from heat.rpc import worker_client as rpc_client
@ -83,6 +84,24 @@ class WorkerService(service.Service):
super(WorkerService, self).stop()
def _trigger_rollback(self, cnxt, stack):
# TODO(ananta) convergence-rollback implementation
pass
def _handle_resource_failure(self, cnxt, stack_id, traversal_id,
failure_reason):
stack = parser.Stack.load(cnxt, stack_id=stack_id)
# make sure no new stack operation was triggered
if stack.current_traversal != traversal_id:
return
stack.state_set(stack.action, stack.FAILED, failure_reason)
if (not stack.disable_rollback and
stack.action in (stack.CREATE, stack.UPDATE)):
self._trigger_rollback(cnxt, stack)
else:
stack.purge_db()
@context.request_context
def check_resource(self, cnxt, resource_id, current_traversal, data,
is_update):
@ -127,6 +146,11 @@ class WorkerService(service.Service):
return
except resource.UpdateInProgress:
return
except exception.ResourceFailure as e:
reason = six.text_type(e)
self._handle_resource_failure(
cnxt, stack.id, current_traversal, reason)
return
input_data = construct_input_data(rsrc)
else:
@ -134,6 +158,11 @@ class WorkerService(service.Service):
check_resource_cleanup(rsrc, tmpl.id, data)
except resource.UpdateInProgress:
return
except exception.ResourceFailure as e:
reason = six.text_type(e)
self._handle_resource_failure(
cnxt, stack.id, current_traversal, reason)
return
graph_key = (rsrc.id, is_update)
if graph_key not in graph and rsrc.replaces is not None:

View File

@ -491,6 +491,50 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
{}, is_update))
self.assertEqual(expected_calls, mock_cr.mock_calls)
def test_mark_complete_purges_db(self, mock_cr):
stack = tools.get_stack('test_stack', utils.dummy_context(),
template=tools.string_template_five,
convergence=True)
stack.store()
stack.purge_db = mock.Mock()
stack.mark_complete(stack.current_traversal)
self.assertTrue(stack.purge_db.called)
def test_purge_db_deletes_previous_template(self, mock_cr):
stack = tools.get_stack('test_stack', utils.dummy_context(),
template=tools.string_template_five,
convergence=True)
prev_tmpl = templatem.Template.create_empty_template()
prev_tmpl.store()
stack.prev_raw_template_id = prev_tmpl.id
stack.store()
stack.purge_db()
self.assertRaises(exception.NotFound,
templatem.Template.load,
stack.context, prev_tmpl.id)
def test_purge_db_deletes_sync_points(self, mock_cr):
stack = tools.get_stack('test_stack', utils.dummy_context(),
template=tools.string_template_five,
convergence=True)
stack.store()
stack.purge_db()
rows = sync_point_object.SyncPoint.delete_all_by_stack_and_traversal(
stack.context, stack.id, stack.current_traversal)
self.assertEqual(0, rows)
def test_purge_db_deletes_stack_for_deleted_stack(self, mock_cr):
stack = tools.get_stack('test_stack', utils.dummy_context(),
template=tools.string_template_five,
convergence=True)
stack.store()
stack.state_set(stack.DELETE, stack.COMPLETE, 'test reason')
stack.purge_db()
self.assertRaises(exception.NotFound,
parser.Stack.load,
stack.context, stack_id=stack.id,
show_deleted=False)
class StackCreateTest(common.HeatTestCase):
def setUp(self):

View File

@ -15,6 +15,7 @@
import mock
from heat.common import exception
from heat.engine import resource
from heat.engine import stack
from heat.engine import sync_point
@ -112,6 +113,12 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
self.resource = self.stack['A']
self.is_update = True
self.graph_key = (self.resource.id, self.is_update)
self.orig_load_method = stack.Stack.load
stack.Stack.load = mock.Mock(return_value=self.stack)
def tearDown(self):
super(CheckWorkflowUpdateTest, self).tearDown()
stack.Stack.load = self.orig_load_method
def test_resource_not_available(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
@ -181,6 +188,135 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
self.assertFalse(mock_pcr.called)
self.assertFalse(mock_csc.called)
def test_resource_update_failure_sets_stack_state_as_failed(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.stack.state_set(self.stack.UPDATE, self.stack.IN_PROGRESS, '')
self.worker._trigger_rollback = mock.Mock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_cru.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.resource.UPDATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
s = self.stack.load(self.ctx, stack_id=self.stack.id)
self.assertEqual((s.UPDATE, s.FAILED), (s.action, s.status))
self.assertEqual(u'ResourceNotAvailable: resources.A: The Resource (A)'
' is not available.', s.status_reason)
def test_resource_cleanup_failure_sets_stack_state_as_failed(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.is_update = False # invokes check_resource_cleanup
self.stack.state_set(self.stack.UPDATE, self.stack.IN_PROGRESS, '')
self.worker._trigger_rollback = mock.Mock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_crc.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.resource.UPDATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
s = self.stack.load(self.ctx, stack_id=self.stack.id)
self.assertEqual((s.UPDATE, s.FAILED), (s.action, s.status))
self.assertEqual(u'ResourceNotAvailable: resources.A: The Resource (A)'
' is not available.', s.status_reason)
def test_resource_update_failure_triggers_rollback_if_enabled(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.stack.disable_rollback = False
self.stack.store()
self.worker._trigger_rollback = mock.Mock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_cru.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.resource.UPDATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
self.assertTrue(self.worker._trigger_rollback.called)
# make sure the rollback is called on given stack
call_args, call_kwargs = self.worker._trigger_rollback.call_args
called_stack = call_args[1]
self.assertEqual(self.stack.id, called_stack.id)
def test_resource_cleanup_failure_triggers_rollback_if_enabled(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.is_update = False # invokes check_resource_cleanup
self.stack.disable_rollback = False
self.stack.store()
self.worker._trigger_rollback = mock.Mock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_crc.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.resource.UPDATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
self.assertTrue(self.worker._trigger_rollback.called)
# make sure the rollback is called on given stack
call_args, call_kwargs = self.worker._trigger_rollback.call_args
called_stack = call_args[1]
self.assertEqual(self.stack.id, called_stack.id)
def test_rollback_is_not_triggered_on_rollback_disabled_stack(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.stack.disable_rollback = True
self.stack.store()
self.worker._trigger_rollback = mock.Mock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_cru.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.stack.CREATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
self.assertFalse(self.worker._trigger_rollback.called)
def test_rollback_not_re_triggered_for_a_rolling_back_stack(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.stack.disable_rollback = False
self.stack.action = self.stack.ROLLBACK
self.stack.status = self.stack.IN_PROGRESS
self.stack.store()
self.worker._trigger_rollback = mock.MagicMock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_cru.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.stack.CREATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
self.assertFalse(self.worker._trigger_rollback.called)
def test_resource_update_failure_purges_db_for_stack_failure(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.stack.disable_rollback = True
self.stack.store()
self.stack.purge_db = mock.Mock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_cru.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.resource.UPDATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
self.assertTrue(self.stack.purge_db.called)
def test_resource_cleanup_failure_purges_db_for_stack_failure(
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
self.is_update = False
self.stack.disable_rollback = True
self.stack.store()
self.stack.purge_db = mock.Mock()
dummy_ex = exception.ResourceNotAvailable(
resource_name=self.resource.name)
mock_crc.side_effect = exception.ResourceFailure(
dummy_ex, self.resource, action=self.resource.UPDATE)
self.worker.check_resource(self.ctx, self.resource.id,
self.stack.current_traversal, {},
self.is_update)
self.assertTrue(self.stack.purge_db.called)
@mock.patch.object(worker, 'construct_input_data')
@mock.patch.object(worker, 'check_stack_complete')