From aa65dfb5265536eca40dbaf9b1826f8bf5148f80 Mon Sep 17 00:00:00 2001
From: Rodolfo Alonso Hernandez <ralonsoh@redhat.com>
Date: Wed, 18 Sep 2019 11:30:04 +0000
Subject: [PATCH] Add retry decorator to SSH "execute" method

In case of SSH timeout (TimeoutException, TimeoutError), the
tenacity.retry decorator retries the execution of the SSH
"execute" method up to 10 times.

Some SSH execute calls, related to QoS scenario tests, have been
enhanced by setting a relatively small timeout value. The commands
executed should be quick enough to be executed in this amount of time.
In case of timeout (due to communication problems), the retry decorator
will send again the command to be executed.

Change-Id: Idc0d55b776f499a4bc5d8c9d9a549f0af8f3fac0
Closes-Bug: #1844516
---
 neutron_tempest_plugin/common/ssh.py        | 18 ++++++++++++++++++
 neutron_tempest_plugin/scenario/test_qos.py |  8 ++++----
 requirements.txt                            |  1 +
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/neutron_tempest_plugin/common/ssh.py b/neutron_tempest_plugin/common/ssh.py
index ea30a282..96f0ef9c 100644
--- a/neutron_tempest_plugin/common/ssh.py
+++ b/neutron_tempest_plugin/common/ssh.py
@@ -14,12 +14,15 @@
 
 import locale
 import os
+import socket
 import time
 
 from oslo_log import log
 import paramiko
+import six
 from tempest.lib.common import ssh
 from tempest.lib import exceptions
+import tenacity
 
 from neutron_tempest_plugin import config
 from neutron_tempest_plugin import exceptions as exc
@@ -29,6 +32,16 @@ CONF = config.CONF
 LOG = log.getLogger(__name__)
 
 
+RETRY_EXCEPTIONS = (exceptions.TimeoutException, paramiko.SSHException,
+                    socket.error)
+if six.PY2:
+    # NOTE(ralonsoh): TimeoutError was added in 3.3 and corresponds to
+    # OSError(errno.ETIMEDOUT)
+    RETRY_EXCEPTIONS += (OSError, )
+else:
+    RETRY_EXCEPTIONS += (TimeoutError, )
+
+
 class Client(ssh.Client):
 
     default_ssh_lang = 'en_US.UTF-8'
@@ -179,6 +192,11 @@ class Client(ssh.Client):
                                         user=self.username,
                                         password=self.password)
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(10),
+        wait=tenacity.wait_fixed(1),
+        retry=tenacity.retry_if_exception_type(RETRY_EXCEPTIONS),
+        reraise=True)
     def exec_command(self, cmd, encoding="utf-8", timeout=None):
         if timeout:
             original_timeout = self.timeout
diff --git a/neutron_tempest_plugin/scenario/test_qos.py b/neutron_tempest_plugin/scenario/test_qos.py
index 82a5391c..ba8cc884 100644
--- a/neutron_tempest_plugin/scenario/test_qos.py
+++ b/neutron_tempest_plugin/scenario/test_qos.py
@@ -85,9 +85,9 @@ class QoSTestMixin(object):
         cmd = ("(dd if=/dev/zero bs=%(bs)d count=%(count)d of=%(file_path)s) "
                % {'bs': self.BUFFER_SIZE, 'count': self.COUNT,
                'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
         cmd = "stat -c %%s %s" % self.FILE_PATH
-        filesize = ssh_client.exec_command(cmd)
+        filesize = ssh_client.exec_command(cmd, timeout=5)
         if int(filesize.strip()) != self.FILE_SIZE:
             raise sc_exceptions.FileCreationFailedException(
                 file=self.FILE_PATH)
@@ -96,7 +96,7 @@ class QoSTestMixin(object):
     def _kill_nc_process(ssh_client):
         cmd = "killall -q nc"
         try:
-            ssh_client.exec_command(cmd)
+            ssh_client.exec_command(cmd, timeout=5)
         except exceptions.SSHExecCommandFailed:
             pass
 
@@ -104,7 +104,7 @@ class QoSTestMixin(object):
         self._kill_nc_process(ssh_client)
         cmd = ("(nc -ll -p %(port)d < %(file_path)s > /dev/null &)" % {
                 'port': port, 'file_path': self.FILE_PATH})
-        ssh_client.exec_command(cmd)
+        ssh_client.exec_command(cmd, timeout=5)
 
         # Open TCP socket to remote VM and download big file
         start_time = time.time()
diff --git a/requirements.txt b/requirements.txt
index bb836d17..2febb7e5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,7 @@ oslo.utils>=3.33.0 # Apache-2.0
 paramiko>=2.0.0 # LGPLv2.1+
 six>=1.10.0 # MIT
 tempest>=17.1.0 # Apache-2.0
+tenacity>=3.2.1 # Apache-2.0
 ddt>=1.0.1 # MIT
 testtools>=2.2.0 # MIT
 testscenarios>=0.4 # Apache-2.0/BSD