Fix for resources stuck in progress after engine crash
When a stack is IN_PROGRESS and an UPDATE or RESTORE is called after an engine crash, we set status of the stack and all of its IN_PROGRESS resources to FAILED Change-Id: Ia3adbfeff16c69719f9e5365657ab46a0932ec9b Closes-Bug: #1570576
This commit is contained in:
parent
841492ea32
commit
d6a90cc6ac
|
@ -2397,17 +2397,6 @@ class EngineService(service.ServiceBase):
|
|||
LOG.debug('Service %s was aborted' % service_ref['id'])
|
||||
service_objects.Service.delete(cnxt, service_ref['id'])
|
||||
|
||||
def set_stack_and_resource_to_failed(self, stack):
|
||||
for name, rsrc in six.iteritems(stack.resources):
|
||||
if rsrc.status == rsrc.IN_PROGRESS:
|
||||
status_reason = ('Engine went down '
|
||||
'during resource %s' % rsrc.action)
|
||||
rsrc.state_set(rsrc.action,
|
||||
rsrc.FAILED,
|
||||
six.text_type(status_reason))
|
||||
reason = ('Engine went down during stack %s' % stack.action)
|
||||
stack.state_set(stack.action, stack.FAILED, six.text_type(reason))
|
||||
|
||||
def reset_stack_status(self):
|
||||
cnxt = context.get_admin_context()
|
||||
filters = {
|
||||
|
@ -2440,12 +2429,14 @@ class EngineService(service.ServiceBase):
|
|||
{'engine': engine_id, 'action': stk.action,
|
||||
'stack_id': stk.id})
|
||||
|
||||
reason = _('Engine went down during stack %s') % stk.action
|
||||
|
||||
# Set stack and resources status to FAILED in sub thread
|
||||
self.thread_group_mgr.start_with_acquired_lock(
|
||||
stk,
|
||||
lock,
|
||||
self.set_stack_and_resource_to_failed,
|
||||
stk
|
||||
stk.reset_stack_and_resources_in_progress,
|
||||
reason
|
||||
)
|
||||
except exception.ActionInProgress:
|
||||
continue
|
||||
|
|
|
@ -1426,6 +1426,14 @@ class Stack(collections.Mapping):
|
|||
|
||||
return self._convg_deps
|
||||
|
||||
def reset_stack_and_resources_in_progress(self, reason):
|
||||
for name, rsrc in six.iteritems(self.resources):
|
||||
if rsrc.status == rsrc.IN_PROGRESS:
|
||||
rsrc.state_set(rsrc.action,
|
||||
rsrc.FAILED,
|
||||
six.text_type(reason))
|
||||
self.state_set(self.action, self.FAILED, six.text_type(reason))
|
||||
|
||||
@scheduler.wrappertask
|
||||
def update_task(self, newstack, action=UPDATE, msg_queue=None):
|
||||
if action not in (self.UPDATE, self.ROLLBACK, self.RESTORE):
|
||||
|
@ -1445,8 +1453,9 @@ class Stack(collections.Mapping):
|
|||
if action == self.ROLLBACK:
|
||||
LOG.debug("Starting update rollback for %s" % self.name)
|
||||
else:
|
||||
self.state_set(action, self.FAILED,
|
||||
'State invalid for %s' % action)
|
||||
reason = _('Attempted to %s an IN_PROGRESS '
|
||||
'stack') % action
|
||||
self.reset_stack_and_resources_in_progress(reason)
|
||||
return
|
||||
|
||||
# Save a copy of the new template. To avoid two DB writes
|
||||
|
|
|
@ -1125,3 +1125,41 @@ resources:
|
|||
environment_files=environment_files)
|
||||
|
||||
# Assertions done in _test_stack_update_preview
|
||||
|
||||
def test_reset_stack_and_resources_in_progress(self):
|
||||
|
||||
def mock_stack_resource(name, action, status):
|
||||
rs = mock.MagicMock()
|
||||
rs.name = name
|
||||
rs.action = action
|
||||
rs.status = status
|
||||
rs.IN_PROGRESS = 'IN_PROGRESS'
|
||||
rs.FAILED = 'FAILED'
|
||||
|
||||
def mock_resource_state_set(a, s, reason='engine_down'):
|
||||
rs.status = s
|
||||
rs.action = a
|
||||
rs.status_reason = reason
|
||||
|
||||
rs.state_set = mock_resource_state_set
|
||||
|
||||
return rs
|
||||
|
||||
stk_name = 'test_stack'
|
||||
stk = tools.get_stack(stk_name, self.ctx)
|
||||
stk.action = 'CREATE'
|
||||
stk.status = 'IN_PROGRESS'
|
||||
|
||||
resources = {'r1': mock_stack_resource('r1', 'UPDATE', 'COMPLETE'),
|
||||
'r2': mock_stack_resource('r2', 'UPDATE', 'IN_PROGRESS'),
|
||||
'r3': mock_stack_resource('r3', 'UPDATE', 'FAILED')}
|
||||
|
||||
stk._resources = resources
|
||||
|
||||
reason = 'Test resetting stack and resources in progress'
|
||||
|
||||
stk.reset_stack_and_resources_in_progress(reason)
|
||||
self.assertEqual('FAILED', stk.status)
|
||||
self.assertEqual('COMPLETE', stk.resources.get('r1').status)
|
||||
self.assertEqual('FAILED', stk.resources.get('r2').status)
|
||||
self.assertEqual('FAILED', stk.resources.get('r3').status)
|
||||
|
|
|
@ -1340,59 +1340,12 @@ class StackServiceTest(common.HeatTestCase):
|
|||
service_check_defer=True,
|
||||
resource_validate=False)
|
||||
self.assertTrue(lock2.release.called)
|
||||
reason = ('Engine went down during stack %s' % fake_stack.action)
|
||||
mock_thread.start_with_acquired_lock.assert_called_once_with(
|
||||
fake_stack, lock1,
|
||||
self.eng.set_stack_and_resource_to_failed, fake_stack
|
||||
fake_stack.reset_stack_and_resources_in_progress, reason
|
||||
)
|
||||
|
||||
def test_set_stack_and_resource_to_failed(self):
|
||||
|
||||
def fake_stack():
|
||||
stk = mock.MagicMock()
|
||||
stk.action = 'CREATE'
|
||||
stk.id = 'foo'
|
||||
stk.status = 'IN_PROGRESS'
|
||||
stk.FAILED = 'FAILED'
|
||||
|
||||
def mock_stack_state_set(a, s, reason):
|
||||
stk.status = s
|
||||
stk.action = a
|
||||
stk.status_reason = reason
|
||||
|
||||
stk.state_set = mock_stack_state_set
|
||||
|
||||
return stk
|
||||
|
||||
def fake_stack_resource(name, action, status):
|
||||
rs = mock.MagicMock()
|
||||
rs.name = name
|
||||
rs.action = action
|
||||
rs.status = status
|
||||
rs.IN_PROGRESS = 'IN_PROGRESS'
|
||||
rs.FAILED = 'FAILED'
|
||||
|
||||
def mock_resource_state_set(a, s, reason='engine_down'):
|
||||
rs.status = s
|
||||
rs.action = a
|
||||
rs.status_reason = reason
|
||||
|
||||
rs.state_set = mock_resource_state_set
|
||||
|
||||
return rs
|
||||
|
||||
test_stack = fake_stack()
|
||||
|
||||
test_stack.resources = {
|
||||
'r1': fake_stack_resource('r1', 'UPDATE', 'COMPLETE'),
|
||||
'r2': fake_stack_resource('r2', 'UPDATE', 'IN_PROGRESS'),
|
||||
'r3': fake_stack_resource('r3', 'UPDATE', 'FAILED')}
|
||||
|
||||
self.eng.set_stack_and_resource_to_failed(test_stack)
|
||||
self.assertEqual('FAILED', test_stack.status)
|
||||
self.assertEqual('COMPLETE', test_stack.resources.get('r1').status)
|
||||
self.assertEqual('FAILED', test_stack.resources.get('r2').status)
|
||||
self.assertEqual('FAILED', test_stack.resources.get('r3').status)
|
||||
|
||||
def test_parse_adopt_stack_data_without_parameters(self):
|
||||
cfg.CONF.set_override('enable_stack_adopt', True, enforce_type=True)
|
||||
template = {"heat_template_version": "2015-04-30",
|
||||
|
|
Loading…
Reference in New Issue