Merge "Reset stack status even when lock engine_id is None"
This commit is contained in:
commit
13d27403ea
@ -1826,30 +1826,48 @@ class EngineService(service.Service):
|
|||||||
|
|
||||||
def reset_stack_status(self):
|
def reset_stack_status(self):
|
||||||
cnxt = context.get_admin_context()
|
cnxt = context.get_admin_context()
|
||||||
filters = {'status': parser.Stack.IN_PROGRESS}
|
filters = {
|
||||||
|
'status': parser.Stack.IN_PROGRESS,
|
||||||
|
'convergence': False
|
||||||
|
}
|
||||||
stacks = stack_object.Stack.get_all(cnxt,
|
stacks = stack_object.Stack.get_all(cnxt,
|
||||||
filters=filters,
|
filters=filters,
|
||||||
tenant_safe=False,
|
tenant_safe=False,
|
||||||
show_nested=True) or []
|
show_nested=True) or []
|
||||||
for s in stacks:
|
for s in stacks:
|
||||||
lock = stack_lock.StackLock(cnxt, s.id, self.engine_id)
|
stack_id = s.id
|
||||||
# If stacklock is released, means stack status may changed.
|
lock = stack_lock.StackLock(cnxt, stack_id, self.engine_id)
|
||||||
engine_id = lock.get_engine_id()
|
engine_id = lock.get_engine_id()
|
||||||
if not engine_id:
|
|
||||||
continue
|
|
||||||
# Try to steal the lock and set status to failed.
|
|
||||||
try:
|
try:
|
||||||
lock.acquire(retry=False)
|
with lock.thread_lock(retry=False):
|
||||||
except exception.ActionInProgress:
|
|
||||||
|
# refetch stack and confirm it is still IN_PROGRESS
|
||||||
|
s = stack_object.Stack.get_by_id(
|
||||||
|
cnxt,
|
||||||
|
stack_id,
|
||||||
|
tenant_safe=False,
|
||||||
|
eager_load=True)
|
||||||
|
if s.status != parser.Stack.IN_PROGRESS:
|
||||||
|
lock.release()
|
||||||
continue
|
continue
|
||||||
|
|
||||||
stk = parser.Stack.load(cnxt, stack=s,
|
stk = parser.Stack.load(cnxt, stack=s,
|
||||||
use_stored_context=True)
|
use_stored_context=True)
|
||||||
LOG.info(_LI('Engine %(engine)s went down when stack %(stack_id)s'
|
LOG.info(_LI('Engine %(engine)s went down when stack '
|
||||||
' was in action %(action)s'),
|
'%(stack_id)s was in action %(action)s'),
|
||||||
{'engine': engine_id, 'action': stk.action,
|
{'engine': engine_id, 'action': stk.action,
|
||||||
'stack_id': stk.id})
|
'stack_id': stk.id})
|
||||||
|
|
||||||
# Set stack and resources status to FAILED in sub thread
|
# Set stack and resources status to FAILED in sub thread
|
||||||
self.thread_group_mgr.start_with_acquired_lock(
|
self.thread_group_mgr.start_with_acquired_lock(
|
||||||
stk, lock, self.set_stack_and_resource_to_failed, stk
|
stk,
|
||||||
|
lock,
|
||||||
|
self.set_stack_and_resource_to_failed,
|
||||||
|
stk
|
||||||
)
|
)
|
||||||
|
except exception.ActionInProgress:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
LOG.exception(_LE('Error while resetting stack: %s')
|
||||||
|
% stack_id)
|
||||||
|
continue
|
||||||
|
@ -125,14 +125,17 @@ class StackLock(object):
|
|||||||
'stack': self.stack_id})
|
'stack': self.stack_id})
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def thread_lock(self):
|
def thread_lock(self, retry=True):
|
||||||
"""Acquire a lock and release it only if there is an exception.
|
"""Acquire a lock and release it only if there is an exception.
|
||||||
|
|
||||||
The release method still needs to be scheduled to be run at the
|
The release method still needs to be scheduled to be run at the
|
||||||
end of the thread using the Thread.link method.
|
end of the thread using the Thread.link method.
|
||||||
|
|
||||||
|
:param retry: When True, retry if lock was released while stealing.
|
||||||
|
:type retry: boolean
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
self.acquire()
|
self.acquire(retry)
|
||||||
yield
|
yield
|
||||||
except exception.ActionInProgress:
|
except exception.ActionInProgress:
|
||||||
raise
|
raise
|
||||||
|
@ -119,7 +119,7 @@ class StackDeleteTest(common.HeatTestCase):
|
|||||||
mock_load.assert_called_with(self.ctx, stack=st)
|
mock_load.assert_called_with(self.ctx, stack=st)
|
||||||
self.assertEqual(2, len(mock_load.mock_calls))
|
self.assertEqual(2, len(mock_load.mock_calls))
|
||||||
mock_try.assert_called_once_with()
|
mock_try.assert_called_once_with()
|
||||||
mock_acquire.assert_called_once_with()
|
mock_acquire.assert_called_once_with(True)
|
||||||
mock_stop.assert_called_once_with(stack.id)
|
mock_stop.assert_called_once_with(stack.id)
|
||||||
|
|
||||||
@mock.patch.object(parser.Stack, 'load')
|
@mock.patch.object(parser.Stack, 'load')
|
||||||
@ -187,7 +187,7 @@ class StackDeleteTest(common.HeatTestCase):
|
|||||||
mock_alive.assert_called_once_with(self.ctx, OTHER_ENGINE)
|
mock_alive.assert_called_once_with(self.ctx, OTHER_ENGINE)
|
||||||
mock_call.assert_called_once_with(self.ctx, OTHER_ENGINE, "stop_stack",
|
mock_call.assert_called_once_with(self.ctx, OTHER_ENGINE, "stop_stack",
|
||||||
stack_identity=mock.ANY)
|
stack_identity=mock.ANY)
|
||||||
mock_acquire.assert_called_once_with()
|
mock_acquire.assert_called_once_with(True)
|
||||||
|
|
||||||
@mock.patch.object(parser.Stack, 'load')
|
@mock.patch.object(parser.Stack, 'load')
|
||||||
@mock.patch.object(stack_lock.StackLock, 'try_acquire')
|
@mock.patch.object(stack_lock.StackLock, 'try_acquire')
|
||||||
@ -213,5 +213,5 @@ class StackDeleteTest(common.HeatTestCase):
|
|||||||
|
|
||||||
mock_load.assert_called_with(self.ctx, stack=st)
|
mock_load.assert_called_with(self.ctx, stack=st)
|
||||||
mock_try.assert_called_once_with()
|
mock_try.assert_called_once_with()
|
||||||
mock_acquire.assert_called_once_with()
|
mock_acquire.assert_called_once_with(True)
|
||||||
mock_alive.assert_called_once_with(self.ctx, OTHER_ENGINE)
|
mock_alive.assert_called_once_with(self.ctx, OTHER_ENGINE)
|
||||||
|
@ -1142,6 +1142,7 @@ class StackServiceTest(common.HeatTestCase):
|
|||||||
@mock.patch('heat.engine.service.ThreadGroupManager',
|
@mock.patch('heat.engine.service.ThreadGroupManager',
|
||||||
return_value=mock.Mock())
|
return_value=mock.Mock())
|
||||||
@mock.patch.object(stack_object.Stack, 'get_all')
|
@mock.patch.object(stack_object.Stack, 'get_all')
|
||||||
|
@mock.patch.object(stack_object.Stack, 'get_by_id')
|
||||||
@mock.patch('heat.engine.stack_lock.StackLock',
|
@mock.patch('heat.engine.stack_lock.StackLock',
|
||||||
return_value=mock.Mock())
|
return_value=mock.Mock())
|
||||||
@mock.patch.object(parser.Stack, 'load')
|
@mock.patch.object(parser.Stack, 'load')
|
||||||
@ -1151,6 +1152,7 @@ class StackServiceTest(common.HeatTestCase):
|
|||||||
mock_admin_context,
|
mock_admin_context,
|
||||||
mock_stack_load,
|
mock_stack_load,
|
||||||
mock_stacklock,
|
mock_stacklock,
|
||||||
|
mock_get_by_id,
|
||||||
mock_get_all,
|
mock_get_all,
|
||||||
mock_thread):
|
mock_thread):
|
||||||
mock_admin_context.return_value = self.ctx
|
mock_admin_context.return_value = self.ctx
|
||||||
@ -1159,7 +1161,19 @@ class StackServiceTest(common.HeatTestCase):
|
|||||||
db_stack.id = 'foo'
|
db_stack.id = 'foo'
|
||||||
db_stack.status = 'IN_PROGRESS'
|
db_stack.status = 'IN_PROGRESS'
|
||||||
db_stack.status_reason = None
|
db_stack.status_reason = None
|
||||||
mock_get_all.return_value = [db_stack]
|
|
||||||
|
unlocked_stack = mock.MagicMock()
|
||||||
|
unlocked_stack.id = 'bar'
|
||||||
|
unlocked_stack.status = 'IN_PROGRESS'
|
||||||
|
unlocked_stack.status_reason = None
|
||||||
|
|
||||||
|
unlocked_stack_failed = mock.MagicMock()
|
||||||
|
unlocked_stack_failed.id = 'bar'
|
||||||
|
unlocked_stack_failed.status = 'FAILED'
|
||||||
|
unlocked_stack_failed.status_reason = 'because'
|
||||||
|
|
||||||
|
mock_get_all.return_value = [db_stack, unlocked_stack]
|
||||||
|
mock_get_by_id.side_effect = [db_stack, unlocked_stack_failed]
|
||||||
|
|
||||||
fake_stack = mock.MagicMock()
|
fake_stack = mock.MagicMock()
|
||||||
fake_stack.action = 'CREATE'
|
fake_stack.action = 'CREATE'
|
||||||
@ -1168,26 +1182,36 @@ class StackServiceTest(common.HeatTestCase):
|
|||||||
|
|
||||||
mock_stack_load.return_value = fake_stack
|
mock_stack_load.return_value = fake_stack
|
||||||
|
|
||||||
fake_lock = mock.MagicMock()
|
lock1 = mock.MagicMock()
|
||||||
fake_lock.get_engine_id.return_value = 'old-engine'
|
lock1.get_engine_id.return_value = 'old-engine'
|
||||||
fake_lock.acquire.return_value = None
|
lock1.acquire.return_value = None
|
||||||
mock_stacklock.return_value = fake_lock
|
lock2 = mock.MagicMock()
|
||||||
|
lock2.acquire.return_value = None
|
||||||
|
mock_stacklock.side_effect = [lock1, lock2]
|
||||||
|
|
||||||
self.eng.thread_group_mgr = mock_thread
|
self.eng.thread_group_mgr = mock_thread
|
||||||
|
|
||||||
self.eng.reset_stack_status()
|
self.eng.reset_stack_status()
|
||||||
|
|
||||||
mock_admin_context.assert_called_once_with()
|
mock_admin_context.assert_called_once_with()
|
||||||
filters = {'status': parser.Stack.IN_PROGRESS}
|
filters = {
|
||||||
|
'status': parser.Stack.IN_PROGRESS,
|
||||||
|
'convergence': False
|
||||||
|
}
|
||||||
mock_get_all.assert_called_once_with(self.ctx,
|
mock_get_all.assert_called_once_with(self.ctx,
|
||||||
filters=filters,
|
filters=filters,
|
||||||
tenant_safe=False,
|
tenant_safe=False,
|
||||||
show_nested=True)
|
show_nested=True)
|
||||||
|
mock_get_by_id.assert_has_calls([
|
||||||
|
mock.call(self.ctx, 'foo', tenant_safe=False, eager_load=True),
|
||||||
|
mock.call(self.ctx, 'bar', tenant_safe=False, eager_load=True),
|
||||||
|
])
|
||||||
mock_stack_load.assert_called_once_with(self.ctx,
|
mock_stack_load.assert_called_once_with(self.ctx,
|
||||||
stack=db_stack,
|
stack=db_stack,
|
||||||
use_stored_context=True)
|
use_stored_context=True)
|
||||||
|
self.assertTrue(lock2.release.called)
|
||||||
mock_thread.start_with_acquired_lock.assert_called_once_with(
|
mock_thread.start_with_acquired_lock.assert_called_once_with(
|
||||||
fake_stack, fake_lock,
|
fake_stack, lock1,
|
||||||
self.eng.set_stack_and_resource_to_failed, fake_stack
|
self.eng.set_stack_and_resource_to_failed, fake_stack
|
||||||
)
|
)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user