From 7548706b0928f2c23bf27e10d116ceca585660e4 Mon Sep 17 00:00:00 2001 From: Tom Barron Date: Sat, 29 Dec 2018 20:41:14 -0600 Subject: [PATCH] Adjust ssh timeouts Generic driver jobs are failing because of timeouts when establishing the initial ssh connection from manila-share to the service VM. Bump up the default value of the connection timeout for paramiko client and also set the banner timeout since the failure occurred during banner exchange. Set the two timeouts to the same value for now. This ensures that the connection timeout is at least as long as the banner timeout and there is no current need in manila to control these independently. This is more of a workaround than a real fix since a real fix would remove the delay during banner exchange. I suspect that the real fix will need to be in neutron/ovs though. Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39 Partial-bug: #1807216 --- devstack/plugin.sh | 4 ++++ devstack/settings | 3 +++ .../tests/share/drivers/hitachi/hnas/test_ssh.py | 3 ++- manila/tests/test_utils.py | 11 ++++++----- manila/utils.py | 14 +++++++++++++- 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index c78756e38a..696fdea385 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -102,6 +102,10 @@ function configure_default_backends { if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD fi + + if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then + iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT + fi done } diff --git a/devstack/settings b/devstack/settings index 2d7bff4408..d41cbeb805 100644 --- a/devstack/settings +++ b/devstack/settings @@ -187,6 +187,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"} MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"} +# SSH TIMEOUT +MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180} + # Admin Network setup MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24} diff --git a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py index 6261d418d7..6889dc06ff 100644 --- a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py +++ b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py @@ -1491,7 +1491,8 @@ class HNASSSHTestCase(test.TestCase): look_for_keys=False, timeout=None, password=self.password, - port=self.port) + port=self.port, + banner_timeout=None) self.assertIn('Request submitted successfully.', output) def test__execute_ssh_exception(self): diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py index b076933dde..530fb26e4b 100644 --- a/manila/tests/test_utils.py +++ b/manila/tests/test_utils.py @@ -225,7 +225,8 @@ class FakeSSHClient(object): pass def connect(self, ip, port=22, username=None, password=None, - key_filename=None, look_for_keys=None, timeout=10): + key_filename=None, look_for_keys=None, timeout=10, + banner_timeout=10): pass def get_transport(self): @@ -284,7 +285,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password="test", key_filename=None, look_for_keys=False, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_with_key(self): path_to_private_key = "/fakepath/to/privatekey" @@ -297,7 +298,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=path_to_private_key, look_for_keys=False, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_with_nothing(self): fake_ssh_client = mock.Mock() @@ -308,7 +309,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=None, look_for_keys=True, - timeout=10) + timeout=10, banner_timeout=10) def test_create_ssh_error_connecting(self): attrs = {'connect.side_effect': paramiko.SSHException, } @@ -320,7 +321,7 @@ class SSHPoolTestCase(test.TestCase): fake_ssh_client.connect.assert_called_once_with( "127.0.0.1", port=22, username="test", password=None, key_filename=None, look_for_keys=True, - timeout=10) + timeout=10, banner_timeout=10) def test_closed_reopend_ssh_connections(self): with mock.patch.object(paramiko, "SSHClient", diff --git a/manila/utils.py b/manila/utils.py index a55d1b61a5..bb417ec6b5 100644 --- a/manila/utils.py +++ b/manila/utils.py @@ -128,13 +128,25 @@ class SSHPool(pools.Pool): elif self.password: look_for_keys = False try: + LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, " + "password: %s, key_filename: %s, look_for_keys: %s, " + "timeout: %s, banner_timeout: %s", + self.ip, + self.port, + self.login, + self.password, + self.path_to_private_key, + look_for_keys, + self.conn_timeout, + self.conn_timeout) ssh.connect(self.ip, port=self.port, username=self.login, password=self.password, key_filename=self.path_to_private_key, look_for_keys=look_for_keys, - timeout=self.conn_timeout) + timeout=self.conn_timeout, + banner_timeout=self.conn_timeout) # Paramiko by default sets the socket timeout to 0.1 seconds, # ignoring what we set through the sshclient. This doesn't help for # keeping long lived connections. Hence we have to bypass it, by