From b4e5e12f7b6548b8e4ba178d530a3b78e278fde4 Mon Sep 17 00:00:00 2001 From: Vitaly Gridnev Date: Mon, 14 Sep 2015 16:52:30 +0300 Subject: [PATCH] Resolve issue with operating heat stack outputs Sometimes, status of heat stack is not changed in DB right after executing updating of heat stack. Because of that sahara thinks that heat stack was successfully updated, but it's wrong, because heat stack updating still in progress. Because of that it's proposed to use field updated_time for verification of successful update of heat stack. We will store that field in variable, and will wait until that will be changed. Closes-bug: 1472536 Change-Id: Ifeee9a81ff1b37ff061de0ded371ae6f6bda4c95 --- sahara/service/heat/heat_engine.py | 4 +++- sahara/service/heat/templates.py | 2 ++ sahara/tests/unit/utils/test_heat.py | 29 +++++++++++++++++++++------- sahara/utils/openstack/heat.py | 14 ++++++++++++-- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/sahara/service/heat/heat_engine.py b/sahara/service/heat/heat_engine.py index 444e7c4320..fe7c377d45 100644 --- a/sahara/service/heat/heat_engine.py +++ b/sahara/service/heat/heat_engine.py @@ -203,7 +203,9 @@ class HeatEngine(e.Engine): self._update_instance_count(stack, cluster, target_count) stack.instantiate(update_existing=update_stack, disable_rollback=disable_rollback) - heat.wait_stack_completion(stack.heat_stack) + heat.wait_stack_completion( + stack.heat_stack, + is_update=update_stack, last_updated_time=stack.last_updated_time) return self._populate_cluster(cluster, stack) def _launch_instances(self, cluster, target_count, stages, diff --git a/sahara/service/heat/templates.py b/sahara/service/heat/templates.py index e1442e3e99..f648943e6b 100644 --- a/sahara/service/heat/templates.py +++ b/sahara/service/heat/templates.py @@ -95,6 +95,7 @@ class ClusterStack(object): self.node_groups_extra = {} self.heat_stack = None self.files = {} + self.last_updated_time = None def add_node_group_extra(self, node_group_id, node_count, gen_userdata_func): @@ -135,6 +136,7 @@ class ClusterStack(object): b.execute_with_retries(heat.stacks.create, **kwargs) else: stack = h.get_stack(self.cluster.name) + self.last_updated_time = stack.updated_time LOG.debug("Updating Heat stack {stack} with args: " "{args}".format(stack=stack, args=kwargs)) b.execute_with_retries(stack.update, **kwargs) diff --git a/sahara/tests/unit/utils/test_heat.py b/sahara/tests/unit/utils/test_heat.py index 5d11b54c10..970aefb70d 100644 --- a/sahara/tests/unit/utils/test_heat.py +++ b/sahara/tests/unit/utils/test_heat.py @@ -23,16 +23,25 @@ from sahara.utils.openstack import heat as h class TestClusterStack(testtools.TestCase): @mock.patch("sahara.context.sleep", return_value=None) def test_wait_completion(self, _): - stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_COMPLETE') + stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_COMPLETE']) h.wait_stack_completion(stack) - stack = FakeHeatStack('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE') + stack = FakeHeatStack('UPDATE_IN_PROGRESS', ['UPDATE_COMPLETE']) h.wait_stack_completion(stack) - stack = FakeHeatStack('DELETE_IN_PROGRESS', 'DELETE_COMPLETE') + stack = FakeHeatStack('DELETE_IN_PROGRESS', ['DELETE_COMPLETE']) h.wait_stack_completion(stack) - stack = FakeHeatStack('CREATE_IN_PROGRESS', 'CREATE_FAILED') + stack = FakeHeatStack('CREATE_COMPLETE', [ + 'CREATE_COMPLETE', 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE']) + h.wait_stack_completion(stack, is_update=True) + + stack = FakeHeatStack('UPDATE_COMPLETE', [ + 'UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE'], + updated_time=-3) + h.wait_stack_completion(stack, is_update=True) + + stack = FakeHeatStack('CREATE_IN_PROGRESS', ['CREATE_FAILED']) with testtools.ExpectedException( ex.HeatStackException, value_re=("Heat stack failed with status " @@ -41,13 +50,19 @@ class TestClusterStack(testtools.TestCase): class FakeHeatStack(object): - def __init__(self, stack_status=None, new_status=None, stack_name=None): + def __init__(self, stack_status=None, new_statuses=None, stack_name=None, + updated_time=None): self.stack_status = stack_status or '' - self.new_status = new_status or '' + self.new_statuses = new_statuses or [] + self.idx = 0 self.stack_name = stack_name or '' + self.updated_time = updated_time def get(self): - self.stack_status = self.new_status + self.stack_status = self.new_statuses[self.idx] + self.idx += 1 + if self.idx > 0 and self.stack_status == 'UPDATE_COMPLETE': + self.updated_time = self.idx @property def status(self): diff --git a/sahara/utils/openstack/heat.py b/sahara/utils/openstack/heat.py index 09f2511ea7..b4bfcc9036 100644 --- a/sahara/utils/openstack/heat.py +++ b/sahara/utils/openstack/heat.py @@ -66,10 +66,20 @@ def get_stack(stack_name, raise_on_missing=True): _('Failed to find stack %(stack)s')) -def wait_stack_completion(stack): +def _verify_completion(stack, is_update=False, last_update_time=None): # NOTE: expected empty status because status of stack # maybe is not set in heat database - while stack.status in ['IN_PROGRESS', '']: + if stack.status in ['IN_PROGRESS', '']: + return False + if is_update and stack.status == 'COMPLETE': + if stack.updated_time == last_update_time: + return False + return True + + +def wait_stack_completion(stack, is_update=False, last_updated_time=None): + base.execute_with_retries(stack.get) + while not _verify_completion(stack, is_update, last_updated_time): context.sleep(1) base.execute_with_retries(stack.get)