Browse Source

Adjust ssh timeouts

Generic driver jobs are failing because of timeouts when
establishing the initial ssh connection from manila-share
to the service VM.

Bump up the default value of the connection timeout for paramiko
client and also set the banner timeout since the failure occurred
during banner exchange.  Set the two timeouts to the same value
for now.  This ensures that the connection timeout is at least as
long as the banner timeout and there is no current need in manila
to control these independently.

This is more of a workaround than a real fix since a real fix
would remove the delay during banner exchange.  I suspect that
the real fix will need to be in neutron/ovs though.

Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39
Partial-bug: #1807216
Tom Barron 3 months ago
parent
commit
7548706b09

+ 4
- 0
devstack/plugin.sh View File

@@ -102,6 +102,10 @@ function configure_default_backends {
102 102
         if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then
103 103
             iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD
104 104
         fi
105
+
106
+        if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then
107
+            iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT
108
+        fi
105 109
     done
106 110
 }
107 111
 

+ 3
- 0
devstack/settings View File

@@ -187,6 +187,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn
187 187
 MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"}
188 188
 MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"}
189 189
 
190
+# SSH TIMEOUT
191
+MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180}
192
+
190 193
 # Admin Network setup
191 194
 MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24}
192 195
 

+ 2
- 1
manila/tests/share/drivers/hitachi/hnas/test_ssh.py View File

@@ -1491,7 +1491,8 @@ class HNASSSHTestCase(test.TestCase):
1491 1491
                                                       look_for_keys=False,
1492 1492
                                                       timeout=None,
1493 1493
                                                       password=self.password,
1494
-                                                      port=self.port)
1494
+                                                      port=self.port,
1495
+                                                      banner_timeout=None)
1495 1496
         self.assertIn('Request submitted successfully.', output)
1496 1497
 
1497 1498
     def test__execute_ssh_exception(self):

+ 6
- 5
manila/tests/test_utils.py View File

@@ -225,7 +225,8 @@ class FakeSSHClient(object):
225 225
         pass
226 226
 
227 227
     def connect(self, ip, port=22, username=None, password=None,
228
-                key_filename=None, look_for_keys=None, timeout=10):
228
+                key_filename=None, look_for_keys=None, timeout=10,
229
+                banner_timeout=10):
229 230
         pass
230 231
 
231 232
     def get_transport(self):
@@ -284,7 +285,7 @@ class SSHPoolTestCase(test.TestCase):
284 285
             fake_ssh_client.connect.assert_called_once_with(
285 286
                 "127.0.0.1", port=22, username="test",
286 287
                 password="test", key_filename=None, look_for_keys=False,
287
-                timeout=10)
288
+                timeout=10, banner_timeout=10)
288 289
 
289 290
     def test_create_ssh_with_key(self):
290 291
         path_to_private_key = "/fakepath/to/privatekey"
@@ -297,7 +298,7 @@ class SSHPoolTestCase(test.TestCase):
297 298
             fake_ssh_client.connect.assert_called_once_with(
298 299
                 "127.0.0.1", port=22, username="test", password=None,
299 300
                 key_filename=path_to_private_key, look_for_keys=False,
300
-                timeout=10)
301
+                timeout=10, banner_timeout=10)
301 302
 
302 303
     def test_create_ssh_with_nothing(self):
303 304
         fake_ssh_client = mock.Mock()
@@ -308,7 +309,7 @@ class SSHPoolTestCase(test.TestCase):
308 309
             fake_ssh_client.connect.assert_called_once_with(
309 310
                 "127.0.0.1", port=22, username="test", password=None,
310 311
                 key_filename=None, look_for_keys=True,
311
-                timeout=10)
312
+                timeout=10, banner_timeout=10)
312 313
 
313 314
     def test_create_ssh_error_connecting(self):
314 315
         attrs = {'connect.side_effect': paramiko.SSHException, }
@@ -320,7 +321,7 @@ class SSHPoolTestCase(test.TestCase):
320 321
             fake_ssh_client.connect.assert_called_once_with(
321 322
                 "127.0.0.1", port=22, username="test", password=None,
322 323
                 key_filename=None, look_for_keys=True,
323
-                timeout=10)
324
+                timeout=10, banner_timeout=10)
324 325
 
325 326
     def test_closed_reopend_ssh_connections(self):
326 327
         with mock.patch.object(paramiko, "SSHClient",

+ 13
- 1
manila/utils.py View File

@@ -128,13 +128,25 @@ class SSHPool(pools.Pool):
128 128
         elif self.password:
129 129
             look_for_keys = False
130 130
         try:
131
+            LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, "
132
+                      "password: %s, key_filename: %s, look_for_keys: %s, "
133
+                      "timeout: %s, banner_timeout: %s",
134
+                      self.ip,
135
+                      self.port,
136
+                      self.login,
137
+                      self.password,
138
+                      self.path_to_private_key,
139
+                      look_for_keys,
140
+                      self.conn_timeout,
141
+                      self.conn_timeout)
131 142
             ssh.connect(self.ip,
132 143
                         port=self.port,
133 144
                         username=self.login,
134 145
                         password=self.password,
135 146
                         key_filename=self.path_to_private_key,
136 147
                         look_for_keys=look_for_keys,
137
-                        timeout=self.conn_timeout)
148
+                        timeout=self.conn_timeout,
149
+                        banner_timeout=self.conn_timeout)
138 150
             # Paramiko by default sets the socket timeout to 0.1 seconds,
139 151
             # ignoring what we set through the sshclient. This doesn't help for
140 152
             # keeping long lived connections. Hence we have to bypass it, by

Loading…
Cancel
Save