Adjust ssh timeouts

Generic driver jobs are failing because of timeouts when
establishing the initial ssh connection from manila-share
to the service VM.

Bump up the default value of the connection timeout for paramiko
client and also set the banner timeout since the failure occurred
during banner exchange.  Set the two timeouts to the same value
for now.  This ensures that the connection timeout is at least as
long as the banner timeout and there is no current need in manila
to control these independently.

This is more of a workaround than a real fix since a real fix
would remove the delay during banner exchange.  I suspect that
the real fix will need to be in neutron/ovs though.

Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39
Partial-bug: #1807216
(cherry picked from commit 7548706b09)
(cherry picked from commit 8919db0186)
(cherry picked from commit 2abde73d1e)

devstack/plugin.sh

@@ -100,6 +100,10 @@ function configure_default_backends {
         if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then
             iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD
         fi
+
+        if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then
+            iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT
+        fi
     done
 }
 

devstack/settings

@@ -183,6 +183,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn
 MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"}
 MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"}
 
+# SSH TIMEOUT
+MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180}
+
 # Admin Network setup
 MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24}
 

manila/tests/share/drivers/hitachi/hnas/test_ssh.py

@@ -1491,7 +1491,8 @@ class HNASSSHTestCase(test.TestCase):
                                                       look_for_keys=False,
                                                       timeout=None,
                                                       password=self.password,
-                                                      port=self.port)
+                                                      port=self.port,
+                                                      banner_timeout=None)
         self.assertIn('Request submitted successfully.', output)
 
     def test__execute_ssh_exception(self):

manila/tests/test_utils.py

@@ -225,7 +225,8 @@ class FakeSSHClient(object):
         pass
 
     def connect(self, ip, port=22, username=None, password=None,
-                key_filename=None, look_for_keys=None, timeout=10):
+                key_filename=None, look_for_keys=None, timeout=10,
+                banner_timeout=10):
         pass
 
     def get_transport(self):
@@ -284,7 +285,7 @@ class SSHPoolTestCase(test.TestCase):
             fake_ssh_client.connect.assert_called_once_with(
                 "127.0.0.1", port=22, username="test",
                 password="test", key_filename=None, look_for_keys=False,
-                timeout=10)
+                timeout=10, banner_timeout=10)
 
     def test_create_ssh_with_key(self):
         path_to_private_key = "/fakepath/to/privatekey"
@@ -297,7 +298,7 @@ class SSHPoolTestCase(test.TestCase):
             fake_ssh_client.connect.assert_called_once_with(
                 "127.0.0.1", port=22, username="test", password=None,
                 key_filename=path_to_private_key, look_for_keys=False,
-                timeout=10)
+                timeout=10, banner_timeout=10)
 
     def test_create_ssh_with_nothing(self):
         fake_ssh_client = mock.Mock()
@@ -308,7 +309,7 @@ class SSHPoolTestCase(test.TestCase):
             fake_ssh_client.connect.assert_called_once_with(
                 "127.0.0.1", port=22, username="test", password=None,
                 key_filename=None, look_for_keys=True,
-                timeout=10)
+                timeout=10, banner_timeout=10)
 
     def test_create_ssh_error_connecting(self):
         attrs = {'connect.side_effect': paramiko.SSHException, }
@@ -320,7 +321,7 @@ class SSHPoolTestCase(test.TestCase):
             fake_ssh_client.connect.assert_called_once_with(
                 "127.0.0.1", port=22, username="test", password=None,
                 key_filename=None, look_for_keys=True,
-                timeout=10)
+                timeout=10, banner_timeout=10)
 
     def test_closed_reopend_ssh_connections(self):
         with mock.patch.object(paramiko, "SSHClient",

manila/utils.py

@@ -127,13 +127,25 @@ class SSHPool(pools.Pool):
         elif self.password:
             look_for_keys = False
         try:
+            LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, "
+                      "password: %s, key_filename: %s, look_for_keys: %s, "
+                      "timeout: %s, banner_timeout: %s",
+                      self.ip,
+                      self.port,
+                      self.login,
+                      self.password,
+                      self.path_to_private_key,
+                      look_for_keys,
+                      self.conn_timeout,
+                      self.conn_timeout)
             ssh.connect(self.ip,
                         port=self.port,
                         username=self.login,
                         password=self.password,
                         key_filename=self.path_to_private_key,
                         look_for_keys=look_for_keys,
-                        timeout=self.conn_timeout)
+                        timeout=self.conn_timeout,
+                        banner_timeout=self.conn_timeout)
             # Paramiko by default sets the socket timeout to 0.1 seconds,
             # ignoring what we set through the sshclient. This doesn't help for
             # keeping long lived connections. Hence we have to bypass it, by