Adding retry ability to heatclient calls
All heatclient calls wrapped in execute_with_retry method to avoid occasional errors partially implements bp clients-calls-retry Change-Id: I835fd79b5783f4a5271cec67b013ba8aeba3091a
This commit is contained in:
parent
f96a28023f
commit
ff98dd848f
@ -26,6 +26,7 @@ from sahara.service.heat import templates as ht
|
||||
from sahara.service import volumes
|
||||
from sahara.utils import cluster_progress_ops as cpo
|
||||
from sahara.utils import general as g
|
||||
from sahara.utils.openstack import base as b
|
||||
from sahara.utils.openstack import heat
|
||||
|
||||
conductor = c.API
|
||||
@ -39,10 +40,10 @@ class HeatEngine(e.Engine):
|
||||
|
||||
def _add_volumes(self, ctx, cluster):
|
||||
for instance in g.get_instances(cluster):
|
||||
res_names = heat.client().resources.get(
|
||||
res_names = heat.get_resource(
|
||||
cluster.name, instance.instance_name).required_by
|
||||
for res_name in res_names:
|
||||
vol_res = heat.client().resources.get(cluster.name, res_name)
|
||||
vol_res = heat.get_resource(cluster.name, res_name)
|
||||
if vol_res.resource_type == (('OS::Cinder::'
|
||||
'VolumeAttachment')):
|
||||
volume_id = vol_res.physical_resource_id
|
||||
@ -177,7 +178,7 @@ class HeatEngine(e.Engine):
|
||||
def shutdown_cluster(self, cluster):
|
||||
"""Shutdown specified cluster and all related resources."""
|
||||
try:
|
||||
heat.client().stacks.delete(cluster.name)
|
||||
b.execute_with_retries(heat.client().stacks.delete, cluster.name)
|
||||
stack = heat.get_stack(cluster.name)
|
||||
heat.wait_stack_completion(stack)
|
||||
except heat_exc.HTTPNotFound:
|
||||
|
@ -20,6 +20,7 @@ import six
|
||||
import yaml
|
||||
|
||||
from sahara.utils import general as g
|
||||
from sahara.utils.openstack import base as b
|
||||
from sahara.utils.openstack import heat as h
|
||||
from sahara.utils.openstack import neutron
|
||||
|
||||
@ -91,11 +92,11 @@ class ClusterTemplate(object):
|
||||
'template': main_tmpl}
|
||||
|
||||
if not update_existing:
|
||||
heat.stacks.create(**kwargs)
|
||||
b.execute_with_retries(heat.stacks.create, **kwargs)
|
||||
else:
|
||||
for stack in heat.stacks.list():
|
||||
for stack in b.execute_with_retries(heat.stacks.list):
|
||||
if stack.stack_name == self.cluster.name:
|
||||
stack.update(**kwargs)
|
||||
b.execute_with_retries(stack.update, **kwargs)
|
||||
break
|
||||
|
||||
return ClusterStack(self, h.get_stack(self.cluster.name))
|
||||
@ -337,10 +338,9 @@ class ClusterStack(object):
|
||||
|
||||
count = self.tmpl.node_groups_extra[node_group.id]['node_count']
|
||||
|
||||
heat = h.client()
|
||||
for i in range(0, count):
|
||||
name = _get_inst_name(self.tmpl.cluster.name, node_group.name, i)
|
||||
res = heat.resources.get(self.heat_stack.id, name)
|
||||
res = h.get_resource(self.heat_stack.id, name)
|
||||
insts.append((name, res.physical_resource_id))
|
||||
|
||||
return insts
|
||||
|
@ -49,7 +49,7 @@ def client():
|
||||
|
||||
def get_stack(stack_name):
|
||||
heat = client()
|
||||
for stack in heat.stacks.list():
|
||||
for stack in base.execute_with_retries(heat.stacks.list):
|
||||
if stack.stack_name == stack_name:
|
||||
return stack
|
||||
|
||||
@ -62,7 +62,12 @@ def wait_stack_completion(stack):
|
||||
# maybe is not set in heat database
|
||||
while stack.status in ['IN_PROGRESS', '']:
|
||||
context.sleep(1)
|
||||
stack.get()
|
||||
base.execute_with_retries(stack.get)
|
||||
|
||||
if stack.status != 'COMPLETE':
|
||||
raise ex.HeatStackException(stack.stack_status)
|
||||
|
||||
|
||||
def get_resource(stack, resource):
|
||||
return base.execute_with_retries(
|
||||
client().resources.get, stack, resource)
|
||||
|
Loading…
Reference in New Issue
Block a user