Resolve issue with operating heat stack outputs

Sometimes, status of heat stack is not changed in DB
right after executing updating of heat stack. Because of
that sahara thinks that heat stack was successfully updated,
but it's wrong, because heat stack updating still in progress.

Because of that it's proposed to use field updated_time
for verification of successful update of heat stack.
We will store that field in variable, and will wait until that
will be changed.

Closes-bug: 1472536
Change-Id: Ifeee9a81ff1b37ff061de0ded371ae6f6bda4c95

sahara/service/heat/heat_engine.py

@@ -203,7 +203,9 @@ class HeatEngine(e.Engine):
         self._update_instance_count(stack, cluster, target_count)
         stack.instantiate(update_existing=update_stack,
                           disable_rollback=disable_rollback)
-        heat.wait_stack_completion(stack.heat_stack)
+        heat.wait_stack_completion(
+            stack.heat_stack,
+            is_update=update_stack, last_updated_time=stack.last_updated_time)
         return self._populate_cluster(cluster, stack)
 
     def _launch_instances(self, cluster, target_count, stages,

sahara/service/heat/templates.py

@@ -95,6 +95,7 @@ class ClusterStack(object):
         self.node_groups_extra = {}
         self.heat_stack = None
         self.files = {}
+        self.last_updated_time = None
 
     def add_node_group_extra(self, node_group_id, node_count,
                              gen_userdata_func):
@@ -135,6 +136,7 @@ class ClusterStack(object):
             b.execute_with_retries(heat.stacks.create, **kwargs)
         else:
             stack = h.get_stack(self.cluster.name)
+            self.last_updated_time = stack.updated_time
             LOG.debug("Updating Heat stack {stack} with args: "
                       "{args}".format(stack=stack, args=kwargs))
             b.execute_with_retries(stack.update, **kwargs)

sahara/tests/unit/utils/test_heat.py

@@ -23,16 +23,25 @@ from sahara.utils.openstack import heat as h
 class TestClusterStack(testtools.TestCase):
     @mock.patch("sahara.context.sleep", return_value=None)
     def test_wait_completion(self, _):
-        stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_COMPLETE')
+        stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_COMPLETE'])
         h.wait_stack_completion(stack)
 
-        stack = FakeHeatStack('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE')
+        stack = FakeHeatStack('UPDATE_IN_PROGRESS', ['UPDATE_COMPLETE'])
         h.wait_stack_completion(stack)
 
-        stack = FakeHeatStack('DELETE_IN_PROGRESS', 'DELETE_COMPLETE')
+        stack = FakeHeatStack('DELETE_IN_PROGRESS', ['DELETE_COMPLETE'])
         h.wait_stack_completion(stack)
 
-        stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_FAILED')
+        stack = FakeHeatStack('CREATE_COMPLETE', [
+            'CREATE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'])
+        h.wait_stack_completion(stack, is_update=True)
+
+        stack = FakeHeatStack('UPDATE_COMPLETE', [
+            'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'],
+            updated_time=-3)
+        h.wait_stack_completion(stack, is_update=True)
+
+        stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_FAILED'])
         with testtools.ExpectedException(
                 ex.HeatStackException,
                 value_re=("Heat stack failed with status "
@@ -41,13 +50,19 @@ class TestClusterStack(testtools.TestCase):
 
 
 class FakeHeatStack(object):
-    def __init__(self, stack_status=None, new_status=None, stack_name=None):
+    def __init__(self, stack_status=None, new_statuses=None, stack_name=None,
+                 updated_time=None):
         self.stack_status = stack_status or ''
-        self.new_status = new_status or ''
+        self.new_statuses = new_statuses or []
+        self.idx = 0
         self.stack_name = stack_name or ''
+        self.updated_time = updated_time
 
     def get(self):
-        self.stack_status = self.new_status
+        self.stack_status = self.new_statuses[self.idx]
+        self.idx += 1
+        if self.idx > 0 and self.stack_status == 'UPDATE_COMPLETE':
+            self.updated_time = self.idx
 
     @property
     def status(self):

sahara/utils/openstack/heat.py

@@ -66,10 +66,20 @@ def get_stack(stack_name, raise_on_missing=True):
                                _('Failed to find stack %(stack)s'))
 
 
-def wait_stack_completion(stack):
+def _verify_completion(stack, is_update=False, last_update_time=None):
     # NOTE: expected empty status because status of stack
     # maybe is not set in heat database
-    while stack.status in ['IN_PROGRESS', '']:
+    if stack.status in ['IN_PROGRESS', '']:
+        return False
+    if is_update and stack.status == 'COMPLETE':
+        if stack.updated_time == last_update_time:
+            return False
+    return True
+
+
+def wait_stack_completion(stack, is_update=False, last_updated_time=None):
+    base.execute_with_retries(stack.get)
+    while not _verify_completion(stack, is_update, last_updated_time):
         context.sleep(1)
         base.execute_with_retries(stack.get)