Fix for resources stuck in progress after engine crash
When a stack is IN_PROGRESS and an UPDATE or RESTORE is called after an engine crash, we set status of the stack and all of its IN_PROGRESS resources to FAILED Change-Id: Ia3adbfeff16c69719f9e5365657ab46a0932ec9b Closes-Bug: #1570576
This commit is contained in:
parent
841492ea32
commit
d6a90cc6ac
|
@ -2397,17 +2397,6 @@ class EngineService(service.ServiceBase):
|
||||||
LOG.debug('Service %s was aborted' % service_ref['id'])
|
LOG.debug('Service %s was aborted' % service_ref['id'])
|
||||||
service_objects.Service.delete(cnxt, service_ref['id'])
|
service_objects.Service.delete(cnxt, service_ref['id'])
|
||||||
|
|
||||||
def set_stack_and_resource_to_failed(self, stack):
|
|
||||||
for name, rsrc in six.iteritems(stack.resources):
|
|
||||||
if rsrc.status == rsrc.IN_PROGRESS:
|
|
||||||
status_reason = ('Engine went down '
|
|
||||||
'during resource %s' % rsrc.action)
|
|
||||||
rsrc.state_set(rsrc.action,
|
|
||||||
rsrc.FAILED,
|
|
||||||
six.text_type(status_reason))
|
|
||||||
reason = ('Engine went down during stack %s' % stack.action)
|
|
||||||
stack.state_set(stack.action, stack.FAILED, six.text_type(reason))
|
|
||||||
|
|
||||||
def reset_stack_status(self):
|
def reset_stack_status(self):
|
||||||
cnxt = context.get_admin_context()
|
cnxt = context.get_admin_context()
|
||||||
filters = {
|
filters = {
|
||||||
|
@ -2440,12 +2429,14 @@ class EngineService(service.ServiceBase):
|
||||||
{'engine': engine_id, 'action': stk.action,
|
{'engine': engine_id, 'action': stk.action,
|
||||||
'stack_id': stk.id})
|
'stack_id': stk.id})
|
||||||
|
|
||||||
|
reason = _('Engine went down during stack %s') % stk.action
|
||||||
|
|
||||||
# Set stack and resources status to FAILED in sub thread
|
# Set stack and resources status to FAILED in sub thread
|
||||||
self.thread_group_mgr.start_with_acquired_lock(
|
self.thread_group_mgr.start_with_acquired_lock(
|
||||||
stk,
|
stk,
|
||||||
lock,
|
lock,
|
||||||
self.set_stack_and_resource_to_failed,
|
stk.reset_stack_and_resources_in_progress,
|
||||||
stk
|
reason
|
||||||
)
|
)
|
||||||
except exception.ActionInProgress:
|
except exception.ActionInProgress:
|
||||||
continue
|
continue
|
||||||
|
|
|
@ -1426,6 +1426,14 @@ class Stack(collections.Mapping):
|
||||||
|
|
||||||
return self._convg_deps
|
return self._convg_deps
|
||||||
|
|
||||||
|
def reset_stack_and_resources_in_progress(self, reason):
|
||||||
|
for name, rsrc in six.iteritems(self.resources):
|
||||||
|
if rsrc.status == rsrc.IN_PROGRESS:
|
||||||
|
rsrc.state_set(rsrc.action,
|
||||||
|
rsrc.FAILED,
|
||||||
|
six.text_type(reason))
|
||||||
|
self.state_set(self.action, self.FAILED, six.text_type(reason))
|
||||||
|
|
||||||
@scheduler.wrappertask
|
@scheduler.wrappertask
|
||||||
def update_task(self, newstack, action=UPDATE, msg_queue=None):
|
def update_task(self, newstack, action=UPDATE, msg_queue=None):
|
||||||
if action not in (self.UPDATE, self.ROLLBACK, self.RESTORE):
|
if action not in (self.UPDATE, self.ROLLBACK, self.RESTORE):
|
||||||
|
@ -1445,8 +1453,9 @@ class Stack(collections.Mapping):
|
||||||
if action == self.ROLLBACK:
|
if action == self.ROLLBACK:
|
||||||
LOG.debug("Starting update rollback for %s" % self.name)
|
LOG.debug("Starting update rollback for %s" % self.name)
|
||||||
else:
|
else:
|
||||||
self.state_set(action, self.FAILED,
|
reason = _('Attempted to %s an IN_PROGRESS '
|
||||||
'State invalid for %s' % action)
|
'stack') % action
|
||||||
|
self.reset_stack_and_resources_in_progress(reason)
|
||||||
return
|
return
|
||||||
|
|
||||||
# Save a copy of the new template. To avoid two DB writes
|
# Save a copy of the new template. To avoid two DB writes
|
||||||
|
|
|
@ -1125,3 +1125,41 @@ resources:
|
||||||
environment_files=environment_files)
|
environment_files=environment_files)
|
||||||
|
|
||||||
# Assertions done in _test_stack_update_preview
|
# Assertions done in _test_stack_update_preview
|
||||||
|
|
||||||
|
def test_reset_stack_and_resources_in_progress(self):
|
||||||
|
|
||||||
|
def mock_stack_resource(name, action, status):
|
||||||
|
rs = mock.MagicMock()
|
||||||
|
rs.name = name
|
||||||
|
rs.action = action
|
||||||
|
rs.status = status
|
||||||
|
rs.IN_PROGRESS = 'IN_PROGRESS'
|
||||||
|
rs.FAILED = 'FAILED'
|
||||||
|
|
||||||
|
def mock_resource_state_set(a, s, reason='engine_down'):
|
||||||
|
rs.status = s
|
||||||
|
rs.action = a
|
||||||
|
rs.status_reason = reason
|
||||||
|
|
||||||
|
rs.state_set = mock_resource_state_set
|
||||||
|
|
||||||
|
return rs
|
||||||
|
|
||||||
|
stk_name = 'test_stack'
|
||||||
|
stk = tools.get_stack(stk_name, self.ctx)
|
||||||
|
stk.action = 'CREATE'
|
||||||
|
stk.status = 'IN_PROGRESS'
|
||||||
|
|
||||||
|
resources = {'r1': mock_stack_resource('r1', 'UPDATE', 'COMPLETE'),
|
||||||
|
'r2': mock_stack_resource('r2', 'UPDATE', 'IN_PROGRESS'),
|
||||||
|
'r3': mock_stack_resource('r3', 'UPDATE', 'FAILED')}
|
||||||
|
|
||||||
|
stk._resources = resources
|
||||||
|
|
||||||
|
reason = 'Test resetting stack and resources in progress'
|
||||||
|
|
||||||
|
stk.reset_stack_and_resources_in_progress(reason)
|
||||||
|
self.assertEqual('FAILED', stk.status)
|
||||||
|
self.assertEqual('COMPLETE', stk.resources.get('r1').status)
|
||||||
|
self.assertEqual('FAILED', stk.resources.get('r2').status)
|
||||||
|
self.assertEqual('FAILED', stk.resources.get('r3').status)
|
||||||
|
|
|
@ -1340,59 +1340,12 @@ class StackServiceTest(common.HeatTestCase):
|
||||||
service_check_defer=True,
|
service_check_defer=True,
|
||||||
resource_validate=False)
|
resource_validate=False)
|
||||||
self.assertTrue(lock2.release.called)
|
self.assertTrue(lock2.release.called)
|
||||||
|
reason = ('Engine went down during stack %s' % fake_stack.action)
|
||||||
mock_thread.start_with_acquired_lock.assert_called_once_with(
|
mock_thread.start_with_acquired_lock.assert_called_once_with(
|
||||||
fake_stack, lock1,
|
fake_stack, lock1,
|
||||||
self.eng.set_stack_and_resource_to_failed, fake_stack
|
fake_stack.reset_stack_and_resources_in_progress, reason
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_set_stack_and_resource_to_failed(self):
|
|
||||||
|
|
||||||
def fake_stack():
|
|
||||||
stk = mock.MagicMock()
|
|
||||||
stk.action = 'CREATE'
|
|
||||||
stk.id = 'foo'
|
|
||||||
stk.status = 'IN_PROGRESS'
|
|
||||||
stk.FAILED = 'FAILED'
|
|
||||||
|
|
||||||
def mock_stack_state_set(a, s, reason):
|
|
||||||
stk.status = s
|
|
||||||
stk.action = a
|
|
||||||
stk.status_reason = reason
|
|
||||||
|
|
||||||
stk.state_set = mock_stack_state_set
|
|
||||||
|
|
||||||
return stk
|
|
||||||
|
|
||||||
def fake_stack_resource(name, action, status):
|
|
||||||
rs = mock.MagicMock()
|
|
||||||
rs.name = name
|
|
||||||
rs.action = action
|
|
||||||
rs.status = status
|
|
||||||
rs.IN_PROGRESS = 'IN_PROGRESS'
|
|
||||||
rs.FAILED = 'FAILED'
|
|
||||||
|
|
||||||
def mock_resource_state_set(a, s, reason='engine_down'):
|
|
||||||
rs.status = s
|
|
||||||
rs.action = a
|
|
||||||
rs.status_reason = reason
|
|
||||||
|
|
||||||
rs.state_set = mock_resource_state_set
|
|
||||||
|
|
||||||
return rs
|
|
||||||
|
|
||||||
test_stack = fake_stack()
|
|
||||||
|
|
||||||
test_stack.resources = {
|
|
||||||
'r1': fake_stack_resource('r1', 'UPDATE', 'COMPLETE'),
|
|
||||||
'r2': fake_stack_resource('r2', 'UPDATE', 'IN_PROGRESS'),
|
|
||||||
'r3': fake_stack_resource('r3', 'UPDATE', 'FAILED')}
|
|
||||||
|
|
||||||
self.eng.set_stack_and_resource_to_failed(test_stack)
|
|
||||||
self.assertEqual('FAILED', test_stack.status)
|
|
||||||
self.assertEqual('COMPLETE', test_stack.resources.get('r1').status)
|
|
||||||
self.assertEqual('FAILED', test_stack.resources.get('r2').status)
|
|
||||||
self.assertEqual('FAILED', test_stack.resources.get('r3').status)
|
|
||||||
|
|
||||||
def test_parse_adopt_stack_data_without_parameters(self):
|
def test_parse_adopt_stack_data_without_parameters(self):
|
||||||
cfg.CONF.set_override('enable_stack_adopt', True, enforce_type=True)
|
cfg.CONF.set_override('enable_stack_adopt', True, enforce_type=True)
|
||||||
template = {"heat_template_version": "2015-04-30",
|
template = {"heat_template_version": "2015-04-30",
|
||||||
|
|
Loading…
Reference in New Issue