Browse Source

Adjust ssh timeouts

Generic driver jobs are failing because of timeouts when
establishing the initial ssh connection from manila-share
to the service VM.

Bump up the default value of the connection timeout for paramiko
client and also set the banner timeout since the failure occurred
during banner exchange.  Set the two timeouts to the same value
for now.  This ensures that the connection timeout is at least as
long as the banner timeout and there is no current need in manila
to control these independently.

This is more of a workaround than a real fix since a real fix
would remove the delay during banner exchange.  I suspect that
the real fix will need to be in neutron/ovs though.

Change-Id: Ib5e59faaf9667b9cb5e7d4072531b7d6c3d4da39
Partial-bug: #1807216
(cherry picked from commit 7548706b09)
(cherry picked from commit 8919db0186)
(cherry picked from commit 2abde73d1e)
(cherry picked from commit 2793bd5e83)
changes/65/630965/3
Tom Barron 4 months ago
parent
commit
0f061ac34b

+ 4
- 0
devstack/plugin.sh View File

@@ -84,6 +84,10 @@ function configure_default_backends {
84 84
         if [ $(trueorfalse False MANILA_USE_SERVICE_INSTANCE_PASSWORD) == True ]; then
85 85
             iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD
86 86
         fi
87
+
88
+        if [ "$SHARE_DRIVER" == "manila.share.drivers.generic.GenericShareDriver" ]; then
89
+            iniset $MANILA_CONF $group_name ssh_conn_timeout $MANILA_SSH_TIMEOUT
90
+        fi
87 91
     done
88 92
 }
89 93
 

+ 3
- 0
devstack/settings View File

@@ -177,6 +177,9 @@ MANILA_DOCKER_IMAGE_URL=${MANILA_DOCKER_IMAGE_URL:-"https://github.com/a-ovchinn
177 177
 MANILA_NETWORK_API_CLASS=${MANILA_NETWORK_API_CLASS:-"manila.network.neutron.neutron_network_plugin.NeutronBindNetworkPlugin"}
178 178
 MANILA_NEUTRON_VNIC_TYPE=${MANILA_NEUTRON_VNIC_TYPE:-"normal"}
179 179
 
180
+# SSH TIMEOUT
181
+MANILA_SSH_TIMEOUT=${MANILA_SSH_TIMEOUT:-180}
182
+
180 183
 # Admin Network setup
181 184
 MANILA_ADMIN_NET_RANGE=${MANILA_ADMIN_NET_RANGE:=10.2.5.0/24}
182 185
 

+ 2
- 1
manila/tests/share/drivers/hitachi/hnas/test_ssh.py View File

@@ -1381,7 +1381,8 @@ class HNASSSHTestCase(test.TestCase):
1381 1381
                                                       look_for_keys=False,
1382 1382
                                                       timeout=None,
1383 1383
                                                       password=self.password,
1384
-                                                      port=self.port)
1384
+                                                      port=self.port,
1385
+                                                      banner_timeout=None)
1385 1386
         self.assertIn('Request submitted successfully.', output)
1386 1387
 
1387 1388
     def test__execute_ssh_exception(self):

+ 6
- 5
manila/tests/test_utils.py View File

@@ -223,7 +223,8 @@ class FakeSSHClient(object):
223 223
         pass
224 224
 
225 225
     def connect(self, ip, port=22, username=None, password=None,
226
-                key_filename=None, look_for_keys=None, timeout=10):
226
+                key_filename=None, look_for_keys=None, timeout=10,
227
+                banner_timeout=10):
227 228
         pass
228 229
 
229 230
     def get_transport(self):
@@ -282,7 +283,7 @@ class SSHPoolTestCase(test.TestCase):
282 283
             fake_ssh_client.connect.assert_called_once_with(
283 284
                 "127.0.0.1", port=22, username="test",
284 285
                 password="test", key_filename=None, look_for_keys=False,
285
-                timeout=10)
286
+                timeout=10, banner_timeout=10)
286 287
 
287 288
     def test_create_ssh_with_key(self):
288 289
         path_to_private_key = "/fakepath/to/privatekey"
@@ -295,7 +296,7 @@ class SSHPoolTestCase(test.TestCase):
295 296
             fake_ssh_client.connect.assert_called_once_with(
296 297
                 "127.0.0.1", port=22, username="test", password=None,
297 298
                 key_filename=path_to_private_key, look_for_keys=False,
298
-                timeout=10)
299
+                timeout=10, banner_timeout=10)
299 300
 
300 301
     def test_create_ssh_with_nothing(self):
301 302
         fake_ssh_client = mock.Mock()
@@ -306,7 +307,7 @@ class SSHPoolTestCase(test.TestCase):
306 307
             fake_ssh_client.connect.assert_called_once_with(
307 308
                 "127.0.0.1", port=22, username="test", password=None,
308 309
                 key_filename=None, look_for_keys=True,
309
-                timeout=10)
310
+                timeout=10, banner_timeout=10)
310 311
 
311 312
     def test_create_ssh_error_connecting(self):
312 313
         attrs = {'connect.side_effect': paramiko.SSHException, }
@@ -318,7 +319,7 @@ class SSHPoolTestCase(test.TestCase):
318 319
             fake_ssh_client.connect.assert_called_once_with(
319 320
                 "127.0.0.1", port=22, username="test", password=None,
320 321
                 key_filename=None, look_for_keys=True,
321
-                timeout=10)
322
+                timeout=10, banner_timeout=10)
322 323
 
323 324
     def test_closed_reopend_ssh_connections(self):
324 325
         with mock.patch.object(paramiko, "SSHClient",

+ 13
- 1
manila/utils.py View File

@@ -103,13 +103,25 @@ class SSHPool(pools.Pool):
103 103
         elif self.password:
104 104
             look_for_keys = False
105 105
         try:
106
+            LOG.debug("ssh.connect: ip: %s, port: %s, username: %s, "
107
+                      "password: %s, key_filename: %s, look_for_keys: %s, "
108
+                      "timeout: %s, banner_timeout: %s",
109
+                      self.ip,
110
+                      self.port,
111
+                      self.login,
112
+                      self.password,
113
+                      self.path_to_private_key,
114
+                      look_for_keys,
115
+                      self.conn_timeout,
116
+                      self.conn_timeout)
106 117
             ssh.connect(self.ip,
107 118
                         port=self.port,
108 119
                         username=self.login,
109 120
                         password=self.password,
110 121
                         key_filename=self.path_to_private_key,
111 122
                         look_for_keys=look_for_keys,
112
-                        timeout=self.conn_timeout)
123
+                        timeout=self.conn_timeout,
124
+                        banner_timeout=self.conn_timeout)
113 125
             if self.conn_timeout:
114 126
                 transport = ssh.get_transport()
115 127
                 transport.set_keepalive(self.conn_timeout)

Loading…
Cancel
Save