Adding retry ability to novaclient calls

All novaclient calls wrapped in execute_with_retry
method to avoid occasional errors

partially implements bp clients-calls-retry

Change-Id: Iaaf3bfa5680486d08699768b612bd209ef5a86e9

sahara/service/api.py

@@ -26,6 +26,7 @@ from sahara.plugins import provisioning
 from sahara.service import quotas
 from sahara.utils import general as g
 from sahara.utils.notification import sender
+from sahara.utils.openstack import base as b
 from sahara.utils.openstack import nova
 
 
@@ -229,34 +230,36 @@ def get_images(name, tags):
 
 def get_image(**kwargs):
     if len(kwargs) == 1 and 'id' in kwargs:
-        return nova.client().images.get(kwargs['id'])
+        return b.execute_with_retries(nova.client().images.get, kwargs['id'])
     else:
-        return nova.client().images.find(**kwargs)
+        return b.execute_with_retries(nova.client().images.find, **kwargs)
 
 
 def get_registered_image(id):
-    return nova.client().images.get_registered_image(id)
+    return b.execute_with_retries(
+        nova.client().images.get_registered_image, id)
 
 
 def register_image(image_id, username, description=None):
     client = nova.client()
-    client.images.set_description(image_id, username, description)
-    return client.images.get(image_id)
+    b.execute_with_retries(
+        client.images.set_description, image_id, username, description)
+    return b.execute_with_retries(client.images.get, image_id)
 
 
 def unregister_image(image_id):
     client = nova.client()
-    client.images.unset_description(image_id)
-    return client.images.get(image_id)
+    b.execute_with_retries(client.images.unset_description, image_id)
+    return b.execute_with_retries(client.images.get, image_id)
 
 
 def add_image_tags(image_id, tags):
     client = nova.client()
-    client.images.tag(image_id, tags)
-    return client.images.get(image_id)
+    b.execute_with_retries(client.images.tag, image_id, tags)
+    return b.execute_with_retries(client.images.get, image_id)
 
 
 def remove_image_tags(image_id, tags):
     client = nova.client()
-    client.images.untag(image_id, tags)
-    return client.images.get(image_id)
+    b.execute_with_retries(client.images.untag, image_id, tags)
+    return b.execute_with_retries(client.images.get, image_id)

sahara/service/engine.py

@@ -29,6 +29,7 @@ from sahara.service import networks
 from sahara.utils import cluster_progress_ops as cpo
 from sahara.utils import edp
 from sahara.utils import general as g
+from sahara.utils.openstack import base as b
 from sahara.utils.openstack import nova
 from sahara.utils import poll_utils
 from sahara.utils import remote
@@ -64,7 +65,8 @@ class Engine(object):
 
     def get_node_group_image_username(self, node_group):
         image_id = node_group.get_image_id()
-        return nova.client().images.get(image_id).username
+        return b.execute_with_retries(
+            nova.client().images.get, image_id).username
 
     @poll_utils.poll_status('ips_assign_timeout', _("Assign IPs"), sleep=1)
     def _ips_assign(self, ips_assigned, cluster, instances):

sahara/service/networks.py

@@ -19,6 +19,7 @@ import six
 
 from sahara import conductor as c
 from sahara import context
+from sahara.utils.openstack import base as b
 from sahara.utils.openstack import neutron
 from sahara.utils.openstack import nova
 
@@ -89,11 +90,13 @@ def init_instances_ips(instance):
 
 
 def assign_floating_ip(instance_id, pool):
-    ip = nova.client().floating_ips.create(pool)
-    nova.client().servers.get(instance_id).add_floating_ip(ip)
+    ip = b.execute_with_retries(nova.client().floating_ips.create, pool)
+    server = b.execute_with_retries(nova.client().servers.get, instance_id)
+    b.execute_with_retries(server.add_floating_ip, ip)
 
 
 def delete_floating_ip(instance_id):
-    fl_ips = nova.client().floating_ips.findall(instance_id=instance_id)
+    fl_ips = b.execute_with_retries(
+        nova.client().floating_ips.findall, instance_id=instance_id)
     for fl_ip in fl_ips:
-        nova.client().floating_ips.delete(fl_ip.id)
+        b.execute_with_retries(nova.client().floating_ips.delete, fl_ip.id)

sahara/service/quotas.py

@@ -19,6 +19,7 @@ import six
 from sahara import context
 from sahara import exceptions as ex
 from sahara.i18n import _
+from sahara.utils.openstack import base as b
 from sahara.utils.openstack import cinder as cinder_client
 from sahara.utils.openstack import neutron as neutron_client
 from sahara.utils.openstack import nova as nova_client
@@ -99,7 +100,7 @@ def _update_limits_for_ng(limits, ng, count):
     sign = lambda x: (1, -1)[x < 0]
     nova = nova_client.client()
     limits['instances'] += count
-    flavor = nova.flavors.get(ng.flavor_id)
+    flavor = b.execute_with_retries(nova.flavors.get, ng.flavor_id)
     limits['ram'] += flavor.ram * count
     limits['cpu'] += flavor.vcpus * count
     if ng.floating_ip_pool:

sahara/tests/unit/service/test_instances.py

@@ -42,6 +42,8 @@ class AbstractInstanceTest(base.SaharaWithDbTestCase):
             'sahara.utils.openstack.nova.client')
         self.nova = _create_nova_mock(self.novaclient_patcher.start())
         self.nova.server_groups.findall.return_value = []
+        self.nova.floating_ips.findall.__name__ = 'findall'
+        self.nova.floating_ips.delete.__name__ = 'delete'
 
         self.get_userdata_patcher = mock.patch(
             'sahara.utils.remote.get_userdata_template')

sahara/tests/unit/service/validation/utils.py

@@ -155,6 +155,7 @@ def start_patch(patch_templates=True):
     nova().security_groups.list.side_effect = _get_security_groups_list
     nova().keypairs.get.side_effect = _get_keypair
     nova().networks.find.side_effect = _get_network
+    nova().networks.find.__name__ = 'find'
     nova().floating_ip_pools.list.side_effect = _get_fl_ip_pool_list
     nova().availability_zones.list.side_effect = _get_availability_zone_list
 

sahara/topology/topology_helper.py

@@ -22,6 +22,7 @@ from sahara import context
 from sahara import exceptions as ex
 from sahara.i18n import _
 from sahara.i18n import _LW
+from sahara.utils.openstack import base as b
 from sahara.utils.openstack import nova
 from sahara.utils import xmlutils as x
 
@@ -116,7 +117,7 @@ def generate_topology_map(cluster, is_node_awareness):
     for ng in cluster.node_groups:
         for i in ng.instances:
             # TODO(alazarev) get all servers info with one request
-            ni = nova_client.servers.get(i.instance_id)
+            ni = b.execute_with_retries(nova_client.servers.get, i.instance_id)
             hostId = ni.hostId
             if hostId not in mapping:
                 raise ex.NotFoundException(

sahara/utils/openstack/nova.py

@@ -58,15 +58,16 @@ def client():
 
 
 def get_flavor(**kwargs):
-    return client().flavors.find(**kwargs)
+    return base.execute_with_retries(client().flavors.find, **kwargs)
 
 
 def get_instance_info(instance):
-    return client().servers.get(instance.instance_id)
+    return base.execute_with_retries(
+        client().servers.get, instance.instance_id)
 
 
 def get_network(**kwargs):
     try:
-        return client().networks.find(**kwargs)
+        return base.execute_with_retries(client().networks.find, **kwargs)
     except nova_ex.NotFound:
         return None