diff --git a/sahara/service/heat/heat_engine.py b/sahara/service/heat/heat_engine.py index a056e706ff..b355cb7fe3 100644 --- a/sahara/service/heat/heat_engine.py +++ b/sahara/service/heat/heat_engine.py @@ -26,6 +26,7 @@ from sahara.service.heat import templates as ht from sahara.service import volumes from sahara.utils import cluster_progress_ops as cpo from sahara.utils import general as g +from sahara.utils.openstack import base as b from sahara.utils.openstack import heat conductor = c.API @@ -39,10 +40,10 @@ class HeatEngine(e.Engine): def _add_volumes(self, ctx, cluster): for instance in g.get_instances(cluster): - res_names = heat.client().resources.get( + res_names = heat.get_resource( cluster.name, instance.instance_name).required_by for res_name in res_names: - vol_res = heat.client().resources.get(cluster.name, res_name) + vol_res = heat.get_resource(cluster.name, res_name) if vol_res.resource_type == (('OS::Cinder::' 'VolumeAttachment')): volume_id = vol_res.physical_resource_id @@ -177,7 +178,7 @@ class HeatEngine(e.Engine): def shutdown_cluster(self, cluster): """Shutdown specified cluster and all related resources.""" try: - heat.client().stacks.delete(cluster.name) + b.execute_with_retries(heat.client().stacks.delete, cluster.name) stack = heat.get_stack(cluster.name) heat.wait_stack_completion(stack) except heat_exc.HTTPNotFound: diff --git a/sahara/service/heat/templates.py b/sahara/service/heat/templates.py index 671840ffbb..f7aa42258f 100644 --- a/sahara/service/heat/templates.py +++ b/sahara/service/heat/templates.py @@ -20,6 +20,7 @@ import six import yaml from sahara.utils import general as g +from sahara.utils.openstack import base as b from sahara.utils.openstack import heat as h from sahara.utils.openstack import neutron @@ -91,11 +92,11 @@ class ClusterTemplate(object): 'template': main_tmpl} if not update_existing: - heat.stacks.create(**kwargs) + b.execute_with_retries(heat.stacks.create, **kwargs) else: - for stack in heat.stacks.list(): + for stack in b.execute_with_retries(heat.stacks.list): if stack.stack_name == self.cluster.name: - stack.update(**kwargs) + b.execute_with_retries(stack.update, **kwargs) break return ClusterStack(self, h.get_stack(self.cluster.name)) @@ -337,10 +338,9 @@ class ClusterStack(object): count = self.tmpl.node_groups_extra[node_group.id]['node_count'] - heat = h.client() for i in range(0, count): name = _get_inst_name(self.tmpl.cluster.name, node_group.name, i) - res = heat.resources.get(self.heat_stack.id, name) + res = h.get_resource(self.heat_stack.id, name) insts.append((name, res.physical_resource_id)) return insts diff --git a/sahara/utils/openstack/heat.py b/sahara/utils/openstack/heat.py index c42b2502b3..f67b7cc959 100644 --- a/sahara/utils/openstack/heat.py +++ b/sahara/utils/openstack/heat.py @@ -49,7 +49,7 @@ def client(): def get_stack(stack_name): heat = client() - for stack in heat.stacks.list(): + for stack in base.execute_with_retries(heat.stacks.list): if stack.stack_name == stack_name: return stack @@ -62,7 +62,12 @@ def wait_stack_completion(stack): # maybe is not set in heat database while stack.status in ['IN_PROGRESS', '']: context.sleep(1) - stack.get() + base.execute_with_retries(stack.get) if stack.status != 'COMPLETE': raise ex.HeatStackException(stack.stack_status) + + +def get_resource(stack, resource): + return base.execute_with_retries( + client().resources.get, stack, resource)