Add retry for getting amphora VM

An issue has been observed several times when amphora VM failed
to get, although its creation was successful.
Add retry to avoid stop loadbalancer creation.

Change-Id: Ic94d226ad6e25323b94bc85f3ad847937b3be218
Story: 2007637
Task: 39690

octavia/compute/drivers/nova_driver.py

@@ -19,6 +19,7 @@ from novaclient import exceptions as nova_exceptions
 from oslo_config import cfg
 from oslo_log import log as logging
 from stevedore import driver as stevedore_driver
+import tenacity
 
 from octavia.common import clients
 from octavia.common import constants
@@ -179,6 +180,14 @@ class VirtualMachineManager(compute_base.ComputeBase):
             raise exceptions.ComputeStatusException() from e
         return constants.DOWN
 
+    def _raise_compute_exception(self):
+        LOG.exception("Error retrieving nova virtual machine.")
+        raise exceptions.ComputeGetException()
+
+    @tenacity.retry(retry=tenacity.retry_if_exception_type(),
+                    stop=tenacity.stop_after_attempt(CONF.compute.max_retries),
+                    retry_error_callback=_raise_compute_exception,
+                    wait=tenacity.wait_fixed(CONF.compute.retry_interval))
     def get_amphora(self, compute_id, management_network_id=None):
         '''Retrieve the information in nova of a virtual machine.
 
@@ -188,11 +197,7 @@ class VirtualMachineManager(compute_base.ComputeBase):
         :returns: fault message or None
         '''
         # utilize nova client ServerManager 'get' method to retrieve info
-        try:
         amphora = self.manager.get(compute_id)
-        except Exception as e:
-            LOG.exception("Error retrieving nova virtual machine.")
-            raise exceptions.ComputeGetException() from e
         return self._translate_amphora(amphora, management_network_id)
 
     def _translate_amphora(self, nova_response, management_network_id=None):

octavia/tests/unit/compute/drivers/test_nova_driver.py

@@ -298,6 +298,13 @@ class TestNovaClient(base.TestCase):
         self.assertRaises(exceptions.ComputeGetException,
                           self.manager.get_amphora, self.amphora.id)
 
+    def test_get_amphora_retried(self):
+        self.manager.manager.get.side_effect = [Exception, self.nova_response]
+        amphora, fault = self.manager.get_amphora(self.amphora.compute_id)
+        self.assertEqual(self.amphora, amphora)
+        self.assertEqual(self.nova_response.fault, fault)
+        self.manager.manager.get.called_with(server=amphora.id)
+
     def test_translate_amphora(self):
         amphora, fault = self.manager._translate_amphora(self.nova_response)
         self.assertEqual(self.amphora, amphora)

releasenotes/notes/add-retry-for-getting-amphora-7f96ec666403ea49.yaml

@@ -0,0 +1,4 @@
+fixes:
+  - |
+    Fix an issue when load balancer creation was aborted due to en error on get
+    of amphora VM.