Catch exceptions when polling stack

In some clouds we can get intermittent failures polling the Heat
stack.  We don't really want that to fail the deployment as a whole,
so catch those exceptions and keep trying.  The exception message is
printed so the user can decide if the problem is serious enough to
stop the deployment manually.

openstack_virtual_baremetal/deploy.py

@@ -202,7 +202,18 @@ def _poll_stack(stack_name,  hclient):
     done = False
     while not done:
         print '.',
-        stack = hclient.stacks.get(stack_name, resolve_outputs=False)
+        # By the time we get here we know Heat was up at one point because
+        # we were able to start the stack create.  Therefore, we can
+        # reasonably guess that any errors from this call are going to be
+        # transient.
+        try:
+            stack = hclient.stacks.get(stack_name, resolve_outputs=False)
+        except Exception as e:
+            # Print the error so the user can determine whether they need
+            # to cancel the deployment, but keep trying otherwise.
+            print 'WARNING: Exception occurred while polling stack: %s' % e
+            time.sleep(10)
+            continue
         sys.stdout.flush()
         if stack.status == 'COMPLETE':
             print 'Stack %s created successfully' % stack_name

openstack_virtual_baremetal/tests/test_deploy.py

@@ -348,6 +348,19 @@ class TestDeploy(testtools.TestCase):
                           mock.call('foo', resolve_outputs=False)],
                           hclient.stacks.get.mock_calls)
 
+    @mock.patch('time.sleep')
+    def test_poll_retry(self, mock_sleep):
+        hclient = mock.Mock()
+        stacks = [mock.Mock(), Exception, mock.Mock()]
+        stacks[0].status = 'IN_PROGRESS'
+        stacks[2].status = 'COMPLETE'
+        hclient.stacks.get.side_effect = stacks
+        deploy._poll_stack('foo', hclient)
+        self.assertEqual([mock.call('foo', resolve_outputs=False),
+                          mock.call('foo', resolve_outputs=False),
+                          mock.call('foo', resolve_outputs=False)],
+                          hclient.stacks.get.mock_calls)
+
     @mock.patch('openstack_virtual_baremetal.deploy._write_role_file')
     @mock.patch('openstack_virtual_baremetal.deploy._load_role_data')
     def test_process_role(self, mock_load, mock_write):