Fix Octavia service unavailable error

This patch fixes the Octavia unavailable service error, as it forces
the check_members_balanced method to wait until Octavia service is
ready and only then it allows the method to send traffic and to assert.

Change-Id: Icfc8c95a5836dcf88636cfa1bdc5e5d68a6f51b4

tobiko/openstack/octavia/__init__.py

@@ -38,6 +38,7 @@ wait_for_active_and_functional_members_and_lb = (
     _waiters.wait_for_active_and_functional_members_and_lb)
 wait_for_lb_to_be_updated_and_active = (
         _waiters.wait_for_lb_to_be_updated_and_active)
+wait_for_octavia_service = _waiters.wait_for_octavia_service
 
 # Validators
 check_members_balanced = _validators.check_members_balanced
@@ -45,6 +46,7 @@ check_members_balanced = _validators.check_members_balanced
 # Exceptions
 RequestException = _exceptions.RequestException
 TimeoutException = _exceptions.TimeoutException
+OctaviaClientException = _exceptions.OctaviaClientException
 
 # Constants
 PROVISIONING_STATUS = _constants.PROVISIONING_STATUS

tobiko/openstack/octavia/_exceptions.py

@@ -14,6 +14,8 @@
 #    under the License.
 from __future__ import absolute_import
 
+from octaviaclient.api import exceptions
+
 import tobiko
 
 
@@ -24,3 +26,8 @@ class RequestException(tobiko.TobikoException):
 
 class TimeoutException(tobiko.TobikoException):
     message = "Timeout exception: {reason}"
+
+
+class OctaviaClientException(tobiko.TobikoException,
+                             exceptions.OctaviaClientException):
+    message = "Octavia client error: {reason}"

tobiko/openstack/octavia/_waiters.py

@@ -141,3 +141,27 @@ def wait_for_lb_to_be_updated_and_active(loadbalancer_id):
                             status=octavia.ACTIVE,
                             get_client=octavia.get_loadbalancer,
                             object_id=loadbalancer_id)
+
+
+def wait_for_octavia_service(loadbalancer_id: str,
+                             waiting_msg='Waiting for the LB to become '
+                                         'functional again...',
+                             interval: tobiko.Seconds = None,
+                             timeout: tobiko.Seconds = None,
+                             count: int = 10):
+
+    LOG.info(waiting_msg)
+    for attempt in tobiko.retry(timeout=timeout,
+                                interval=interval,
+                                count=count):
+        try:
+            octavia.list_amphorae(loadbalancer_id=loadbalancer_id)
+            LOG.info('Octavia service is available!')
+            return
+        except Exception as ex:
+            if issubclass(octavia.OctaviaClientException, ex.__class__):
+                LOG.info(waiting_msg)
+                if attempt.is_last:
+                    raise
+            else:
+                raise ex

tobiko/tests/faults/octavia/test_faults.py

@@ -71,6 +71,9 @@ class OctaviaBasicFaultTest(testtools.TestCase):
                 pool_id=self.pool_stack.pool_id,
                 loadbalancer_id=self.loadbalancer_stack.loadbalancer_id)
 
+        octavia.wait_for_octavia_service(
+            loadbalancer_id=self.loadbalancer_stack.loadbalancer_id)
+
         # Send traffic
         octavia.check_members_balanced(
             pool_id=self.pool_stack.pool_id,

tobiko/tests/faults/octavia/test_services.py

@@ -87,6 +87,9 @@ class OctaviaServicesFaultTest(testtools.TestCase):
             pool_id=self.pool_stack.pool_id,
             loadbalancer_id=self.loadbalancer_stack.loadbalancer_id)
 
+        octavia.wait_for_octavia_service(
+            loadbalancer_id=self.loadbalancer_stack.loadbalancer_id)
+
         # Sending initial traffic before we stop octavia services
         octavia.check_members_balanced(
             pool_id=self.pool_stack.pool_id,

tobiko/tests/scenario/octavia/test_traffic.py

@@ -60,6 +60,9 @@ class OctaviaBasicTrafficScenarioTest(testtools.TestCase):
             lb_port=self.listener_stack.lb_port,
             loadbalancer_id=self.loadbalancer_stack.loadbalancer_id)
 
+        octavia.wait_for_octavia_service(
+            loadbalancer_id=self.loadbalancer_stack.loadbalancer_id)
+
     @pytest.mark.flaky(reruns=3)
     def test_traffic(self):
         octavia.check_members_balanced(