Fix traffic test - add curl timeout error

This patch fixes the timeout error we get at the traffic test, which
happens when we send the traffic to the loadbalancer when it is not
ready/stable.

Change-Id: Ia3eaa4f9dfddf29828a40f19b46127fd7e7a8e05

tobiko/openstack/octavia/__init__.py

@@ -45,6 +45,7 @@ RequestException = _exceptions.RequestException
 TimeoutException = _exceptions.TimeoutException
 OctaviaClientException = _exceptions.OctaviaClientException
 RoundRobinException = _exceptions.RoundRobinException
+TrafficTimeoutError = _exceptions.TrafficTimeoutError
 
 # Constants
 PROVISIONING_STATUS = _constants.PROVISIONING_STATUS

tobiko/openstack/octavia/_exceptions.py

@@ -33,3 +33,7 @@ OctaviaClientException = exceptions.OctaviaClientException
 
 class RoundRobinException(tobiko.TobikoException):
     message = "Round robin exception: {reason}"
+
+
+class TrafficTimeoutError(tobiko.TobikoException):
+    message = "Traffic timeout error: {reason}"

tobiko/openstack/octavia/_validators.py

@@ -23,6 +23,7 @@ import tobiko
 from tobiko.openstack import octavia
 from tobiko.shell import curl
 from tobiko.shell import ssh
+from tobiko.shell import sh
 
 
 LOG = log.getLogger(__name__)
@@ -57,12 +58,18 @@ def check_members_balanced(ip_address: str,
     replies: typing.Dict[str, int] = collections.defaultdict(lambda: 0)
     for attempt in tobiko.retry(count=members_count * requests_count,
                                 interval=interval):
-        content = curl.execute_curl(hostname=ip_address,
-                                    scheme=protocol,
-                                    port=port,
-                                    path='id',
-                                    connect_timeout=connect_timeout,
-                                    ssh_client=ssh_client).strip()
+        try:
+            content = curl.execute_curl(hostname=ip_address,
+                                        scheme=protocol,
+                                        port=port,
+                                        path='id',
+                                        connect_timeout=connect_timeout,
+                                        ssh_client=ssh_client).strip()
+        except sh.ShellCommandFailed as ex:
+            if ex.exit_status == 28:
+                raise octavia.TrafficTimeoutError(
+                    reason=str(ex.stderr)) from ex
+
         replies[content] += 1
 
         if last_content is not None and lb_algorithm == 'ROUND_ROBIN':

tobiko/tests/scenario/octavia/test_traffic.py

@@ -86,7 +86,8 @@ class OctaviaBasicTrafficScenarioTest(testtools.TestCase):
 
                 break
 
-            except octavia.RoundRobinException as e:
+            except (octavia.RoundRobinException,
+                    octavia.TrafficTimeoutError) as e:
                 if attempt.is_last:
                     raise e