Add integration test for UPDATE_FAILED recovery

It's valid to attempt an update from UPDATE_FAILED state, so add
a functional test which proves this works.

This requires an adjustment to the common test.py _verify_status
because we now need to ensure we record the time of transition to
FAILED state as well as COMPLETE, or we'll mis-detect the failure
as a failure of the subsequent recovery update.

Co-Authored-By: Sergey Kraynev <skraynev@mirantis.com>
Change-Id: I5fc82b4c71a943ba15c04b08bcfd26fcb84dc7a4

heat_integrationtests/common/test.py

@@ -250,9 +250,9 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
 
     def _verify_status(self, stack, stack_identifier, status, fail_regexp):
         if stack.stack_status == status:
-            # Handle UPDATE_COMPLETE case: Make sure we don't
+            # Handle UPDATE_COMPLETE/FAILED case: Make sure we don't
-            # wait for a stale UPDATE_COMPLETE status.
+            # wait for a stale UPDATE_COMPLETE/FAILED status.
-            if status == 'UPDATE_COMPLETE':
+            if status in ('UPDATE_FAILED', 'UPDATE_COMPLETE'):
                 if self.updated_time.get(
                         stack_identifier) != stack.updated_time:
                     self.updated_time[stack_identifier] = stack.updated_time
@@ -263,8 +263,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
         wait_for_action = status.split('_')[0]
         if (stack.action == wait_for_action and
                 fail_regexp.search(stack.stack_status)):
-            # Handle UPDATE_FAILED case.
+            # Handle UPDATE_COMPLETE/UPDATE_FAILED case.
-            if status == 'UPDATE_FAILED':
+            if status in ('UPDATE_FAILED', 'UPDATE_COMPLETE'):
                 if self.updated_time.get(
                         stack_identifier) != stack.updated_time:
                     self.updated_time[stack_identifier] = stack.updated_time
@@ -279,7 +279,7 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
                     stack_status_reason=stack.stack_status_reason)
 
     def _wait_for_stack_status(self, stack_identifier, status,
-                               failure_pattern='^.*_FAILED$',
+                               failure_pattern=None,
                                success_on_not_found=False):
         """
         Waits for a Stack to reach a given status.
@@ -288,7 +288,13 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
         CREATE_COMPLETE, not just COMPLETE which is exposed
         via the status property of Stack in heatclient
         """
-        fail_regexp = re.compile(failure_pattern)
+        if failure_pattern:
+            fail_regexp = re.compile(failure_pattern)
+        elif 'FAILED' in status:
+            # If we're looking for e.g CREATE_FAILED, COMPLETE is unexpected.
+            fail_regexp = re.compile('^.*_COMPLETE$')
+        else:
+            fail_regexp = re.compile('^.*_FAILED$')
         build_timeout = self.conf.build_timeout
         build_interval = self.conf.build_interval
 
@@ -335,6 +341,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
         build_timeout = self.conf.build_timeout
         build_interval = self.conf.build_interval
         start = timeutils.utcnow()
+        self.updated_time[stack_identifier] = self.client.stacks.get(
+            stack_identifier).updated_time
         while timeutils.delta_seconds(start,
                                       timeutils.utcnow()) < build_timeout:
             try:

heat_integrationtests/functional/test_create_update.py

@@ -243,6 +243,29 @@ resources:
         self.assertEqual(updated_resources,
                          self.list_resources(stack_identifier))
 
+    def test_stack_update_from_failed(self):
+        # Prove it's possible to update from an UPDATE_FAILED state
+        template = _change_rsrc_properties(test_template_one_resource,
+                                           ['test1'],
+                                           {'value': 'test_update_failed'})
+        stack_identifier = self.stack_create(
+            template=template)
+        initial_resources = {'test1': 'OS::Heat::TestResource'}
+        self.assertEqual(initial_resources,
+                         self.list_resources(stack_identifier))
+
+        tmpl_update = _change_rsrc_properties(
+            test_template_one_resource, ['test1'], {'fail': True})
+        # Update with bad template, we should fail
+        self.update_stack(stack_identifier, tmpl_update,
+                          expected_status='UPDATE_FAILED')
+        # but then passing a good template should succeed
+        self.update_stack(stack_identifier, test_template_two_resource)
+        updated_resources = {'test1': 'OS::Heat::TestResource',
+                             'test2': 'OS::Heat::TestResource'}
+        self.assertEqual(updated_resources,
+                         self.list_resources(stack_identifier))
+
     def test_stack_update_provider(self):
         template = _change_rsrc_properties(
             test_template_one_resource, ['test1'],