Catch exceptions when polling stack
In some clouds we can get intermittent failures polling the Heat stack. We don't really want that to fail the deployment as a whole, so catch those exceptions and keep trying. The exception message is printed so the user can decide if the problem is serious enough to stop the deployment manually.
This commit is contained in:
parent
b315f165e2
commit
3eea1a0d3a
@ -202,7 +202,18 @@ def _poll_stack(stack_name, hclient):
|
||||
done = False
|
||||
while not done:
|
||||
print '.',
|
||||
# By the time we get here we know Heat was up at one point because
|
||||
# we were able to start the stack create. Therefore, we can
|
||||
# reasonably guess that any errors from this call are going to be
|
||||
# transient.
|
||||
try:
|
||||
stack = hclient.stacks.get(stack_name, resolve_outputs=False)
|
||||
except Exception as e:
|
||||
# Print the error so the user can determine whether they need
|
||||
# to cancel the deployment, but keep trying otherwise.
|
||||
print 'WARNING: Exception occurred while polling stack: %s' % e
|
||||
time.sleep(10)
|
||||
continue
|
||||
sys.stdout.flush()
|
||||
if stack.status == 'COMPLETE':
|
||||
print 'Stack %s created successfully' % stack_name
|
||||
|
@ -348,6 +348,19 @@ class TestDeploy(testtools.TestCase):
|
||||
mock.call('foo', resolve_outputs=False)],
|
||||
hclient.stacks.get.mock_calls)
|
||||
|
||||
@mock.patch('time.sleep')
|
||||
def test_poll_retry(self, mock_sleep):
|
||||
hclient = mock.Mock()
|
||||
stacks = [mock.Mock(), Exception, mock.Mock()]
|
||||
stacks[0].status = 'IN_PROGRESS'
|
||||
stacks[2].status = 'COMPLETE'
|
||||
hclient.stacks.get.side_effect = stacks
|
||||
deploy._poll_stack('foo', hclient)
|
||||
self.assertEqual([mock.call('foo', resolve_outputs=False),
|
||||
mock.call('foo', resolve_outputs=False),
|
||||
mock.call('foo', resolve_outputs=False)],
|
||||
hclient.stacks.get.mock_calls)
|
||||
|
||||
@mock.patch('openstack_virtual_baremetal.deploy._write_role_file')
|
||||
@mock.patch('openstack_virtual_baremetal.deploy._load_role_data')
|
||||
def test_process_role(self, mock_load, mock_write):
|
||||
|
Loading…
Reference in New Issue
Block a user