Adjust ssh timeouts

Generic driver jobs are failing because of timeouts when
establishing the initial ssh connection from manila-share
to the service VM.

Bump up the default value of the connection timeout for paramiko
client and also set the banner timeout since the failure occurred
during banner exchange.  Set the two timeouts to the same value
for now.  This ensures that the connection timeout is at least as
long as the banner timeout and there is no current need in manila
to control these independently.

This is more of a workaround than a real fix since a real fix
would remove the delay during banner exchange.  I suspect that
the real fix will need to be in neutron/ovs though.

Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39
Partial-bug: #1807216
This commit is contained in:
Tom Barron 2018-12-29 20:41:14 -06:00
parent cf41d1867c
commit 7548706b09
5 changed files with 28 additions and 7 deletions

View File

@ -102,6 +102,10 @@ function configure_default_backends {
if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then
iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD
fi fi
if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then
iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT
fi
done done
} }

View File

@ -187,6 +187,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn
MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"} MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"}
MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"} MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"}
# SSH TIMEOUT
MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180}
# Admin Network setup # Admin Network setup
MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24} MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24}

View File

@ -1491,7 +1491,8 @@ class HNASSSHTestCase(test.TestCase):
look_for_keys=False, look_for_keys=False,
timeout=None, timeout=None,
password=self.password, password=self.password,
port=self.port) port=self.port,
banner_timeout=None)
self.assertIn('Request submitted successfully.', output) self.assertIn('Request submitted successfully.', output)
def test__execute_ssh_exception(self): def test__execute_ssh_exception(self):

View File

@ -225,7 +225,8 @@ class FakeSSHClient(object):
pass pass
def connect(self, ip, port=22, username=None, password=None, def connect(self, ip, port=22, username=None, password=None,
key_filename=None, look_for_keys=None, timeout=10): key_filename=None, look_for_keys=None, timeout=10,
banner_timeout=10):
pass pass
def get_transport(self): def get_transport(self):
@ -284,7 +285,7 @@ class SSHPoolTestCase(test.TestCase):
fake_ssh_client.connect.assert_called_once_with( fake_ssh_client.connect.assert_called_once_with(
"127.0.0.1", port=22, username="test", "127.0.0.1", port=22, username="test",
password="test", key_filename=None, look_for_keys=False, password="test", key_filename=None, look_for_keys=False,
timeout=10) timeout=10, banner_timeout=10)
def test_create_ssh_with_key(self): def test_create_ssh_with_key(self):
path_to_private_key = "/fakepath/to/privatekey" path_to_private_key = "/fakepath/to/privatekey"
@ -297,7 +298,7 @@ class SSHPoolTestCase(test.TestCase):
fake_ssh_client.connect.assert_called_once_with( fake_ssh_client.connect.assert_called_once_with(
"127.0.0.1", port=22, username="test", password=None, "127.0.0.1", port=22, username="test", password=None,
key_filename=path_to_private_key, look_for_keys=False, key_filename=path_to_private_key, look_for_keys=False,
timeout=10) timeout=10, banner_timeout=10)
def test_create_ssh_with_nothing(self): def test_create_ssh_with_nothing(self):
fake_ssh_client = mock.Mock() fake_ssh_client = mock.Mock()
@ -308,7 +309,7 @@ class SSHPoolTestCase(test.TestCase):
fake_ssh_client.connect.assert_called_once_with( fake_ssh_client.connect.assert_called_once_with(
"127.0.0.1", port=22, username="test", password=None, "127.0.0.1", port=22, username="test", password=None,
key_filename=None, look_for_keys=True, key_filename=None, look_for_keys=True,
timeout=10) timeout=10, banner_timeout=10)
def test_create_ssh_error_connecting(self): def test_create_ssh_error_connecting(self):
attrs = {'connect.side_effect': paramiko.SSHException, } attrs = {'connect.side_effect': paramiko.SSHException, }
@ -320,7 +321,7 @@ class SSHPoolTestCase(test.TestCase):
fake_ssh_client.connect.assert_called_once_with( fake_ssh_client.connect.assert_called_once_with(
"127.0.0.1", port=22, username="test", password=None, "127.0.0.1", port=22, username="test", password=None,
key_filename=None, look_for_keys=True, key_filename=None, look_for_keys=True,
timeout=10) timeout=10, banner_timeout=10)
def test_closed_reopend_ssh_connections(self): def test_closed_reopend_ssh_connections(self):
with mock.patch.object(paramiko, "SSHClient", with mock.patch.object(paramiko, "SSHClient",

View File

@ -128,13 +128,25 @@ class SSHPool(pools.Pool):
elif self.password: elif self.password:
look_for_keys = False look_for_keys = False
try: try:
LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, "
"password: %s, key_filename: %s, look_for_keys: %s, "
"timeout: %s, banner_timeout: %s",
self.ip,
self.port,
self.login,
self.password,
self.path_to_private_key,
look_for_keys,
self.conn_timeout,
self.conn_timeout)
ssh.connect(self.ip, ssh.connect(self.ip,
port=self.port, port=self.port,
username=self.login, username=self.login,
password=self.password, password=self.password,
key_filename=self.path_to_private_key, key_filename=self.path_to_private_key,
look_for_keys=look_for_keys, look_for_keys=look_for_keys,
timeout=self.conn_timeout) timeout=self.conn_timeout,
banner_timeout=self.conn_timeout)
# Paramiko by default sets the socket timeout to 0.1 seconds, # Paramiko by default sets the socket timeout to 0.1 seconds,
# ignoring what we set through the sshclient. This doesn't help for # ignoring what we set through the sshclient. This doesn't help for
# keeping long lived connections. Hence we have to bypass it, by # keeping long lived connections. Hence we have to bypass it, by