Browse Source

Merge "Improve failover of SSH connections"

Jenkins 2 years ago
parent
commit
d8c5b6b8b7

+ 2
- 2
fuelweb_test/helpers/checkers.py View File

@@ -1466,8 +1466,8 @@ def check_package_version(ip, package_name, expected_version, condition='ge'):
1466 1466
     :param condition: predicate can be on of eq, ne, lt, le, ge, gt
1467 1467
     :return None: or raise UnexpectedExitCode
1468 1468
     """
1469
-    cmd = "dpkg -s {0} " \
1470
-          "| awk -F': ' '/Version/ {{print \$2}}'".format(package_name)
1469
+    cmd = ("dpkg -s {0} "
1470
+           "| awk -F': ' '/Version/ {{print $2}}'".format(package_name))
1471 1471
     logger.debug(cmd)
1472 1472
     result = ssh_manager.execute_on_remote(
1473 1473
         ip,

+ 101
- 63
fuelweb_test/helpers/ssh_manager.py View File

@@ -19,9 +19,10 @@ import traceback
19 19
 from warnings import warn
20 20
 
21 21
 from devops.helpers.metaclasses import SingletonMeta
22
+from devops.helpers.ssh_client import SSHAuth
22 23
 from devops.helpers.ssh_client import SSHClient
23 24
 from paramiko import RSAKey
24
-from paramiko.ssh_exception import AuthenticationException
25
+from paramiko import SSHException
25 26
 import six
26 27
 
27 28
 from fuelweb_test import logger
@@ -67,8 +68,15 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
67 68
         self.slave_login = slave_login
68 69
         self.__slave_password = slave_password
69 70
 
70
-    @staticmethod
71
-    def _connect(remote):
71
+    def _get_keys(self):
72
+        keys = []
73
+        admin_remote = self.get_remote(self.admin_ip)
74
+        key_string = '/root/.ssh/id_rsa'
75
+        with admin_remote.open(key_string) as f:
76
+            keys.append(RSAKey.from_private_key(f))
77
+        return keys
78
+
79
+    def connect(self, remote):
72 80
         """ Check if connection is stable and return this one
73 81
 
74 82
         :param remote:
@@ -80,99 +88,129 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
80 88
                     seconds=5,
81 89
                     error_message="Socket timeout! Forcing reconnection"):
82 90
                 remote.check_call("cd ~")
83
-        except:
91
+        except Exception:
84 92
             logger.debug(traceback.format_exc())
85
-            logger.info('SSHManager: Check for current connection fails. '
86
-                        'Trying to reconnect')
87
-            remote.reconnect()
93
+            logger.debug('SSHManager: Check for current connection fails. '
94
+                         'Trying to reconnect')
95
+            remote = self.reconnect(remote)
88 96
         return remote
89 97
 
90
-    def _get_keys(self):
91
-        keys = []
92
-        admin_remote = self.get_remote(self.admin_ip)
93
-        key_string = '/root/.ssh/id_rsa'
94
-        with admin_remote.open(key_string) as f:
95
-            keys.append(RSAKey.from_private_key(f))
96
-        return keys
98
+    def reconnect(self, remote):
99
+        """ Reconnect to remote or update connection
97 100
 
98
-    def get_remote(self, ip, port=22):
99
-        """ Function returns remote SSH connection to node by ip address
101
+        :param remote:
102
+        :return:
103
+        """
104
+        ip = remote.hostname
105
+        port = remote.port
106
+        try:
107
+            remote.reconnect()
108
+        except SSHException:
109
+            self.update_connection(ip=ip, port=port)
110
+        return self.connections[(ip, port)]
111
+
112
+    def init_remote(self, ip, port=22, custom_creds=None):
113
+        """ Initialise connection to remote
100 114
 
101 115
         :param ip: IP of host
102 116
         :type ip: str
103 117
         :param port: port for SSH
104 118
         :type port: int
105
-        :rtype: SSHClient
119
+        :param custom_creds: custom creds
120
+        :type custom_creds: dict
106 121
         """
107
-        if (ip, port) not in self.connections:
108
-            logger.debug('SSH_MANAGER: Create new connection for '
109
-                         '{ip}:{port}'.format(ip=ip, port=port))
122
+        logger.debug('SSH_MANAGER: Create new connection for '
123
+                     '{ip}:{port}'.format(ip=ip, port=port))
110 124
 
111
-            keys = self._get_keys() if ip != self.admin_ip else []
112
-            if ip == self.admin_ip:
125
+        keys = self._get_keys() if ip != self.admin_ip else []
126
+        if ip == self.admin_ip:
127
+            ssh_client = SSHClient(
128
+                host=ip,
129
+                port=port,
130
+                auth=SSHAuth(
131
+                    username=self.admin_login,
132
+                    password=self.__admin_password,
133
+                    keys=keys)
134
+            )
135
+            ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo']
136
+        elif custom_creds:
137
+            ssh_client = SSHClient(
138
+                host=ip,
139
+                port=port,
140
+                auth=SSHAuth(**custom_creds))
141
+        else:
142
+            try:
113 143
                 ssh_client = SSHClient(
114 144
                     host=ip,
115 145
                     port=port,
116
-                    username=self.admin_login,
117
-                    password=self.__admin_password,
118
-                    private_keys=keys
119
-                )
120
-                ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo']
121
-            else:
122
-                try:
123
-                    ssh_client = SSHClient(
124
-                        host=ip,
125
-                        port=port,
146
+                    auth=SSHAuth(
126 147
                         username=self.slave_login,
127 148
                         password=self.__slave_password,
128
-                        private_keys=keys
129
-                    )
130
-                except AuthenticationException:
131
-                    ssh_client = SSHClient(
132
-                        host=ip,
133
-                        port=port,
149
+                        keys=keys)
150
+                )
151
+            except SSHException:
152
+                ssh_client = SSHClient(
153
+                    host=ip,
154
+                    port=port,
155
+                    auth=SSHAuth(
134 156
                         username=self.slave_fallback_login,
135 157
                         password=self.__slave_password,
136
-                        private_keys=keys
137
-                    )
138
-                ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo']
139
-            self.connections[(ip, port)] = ssh_client
140
-        logger.debug('SSH_MANAGER: Return existed connection for '
141
-                     '{ip}:{port}'.format(ip=ip, port=port))
158
+                        keys=keys)
159
+                )
160
+            ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo']
161
+
162
+        self.connections[(ip, port)] = ssh_client
163
+        logger.debug('SSH_MANAGER: New connection for '
164
+                     '{ip}:{port} is created'.format(ip=ip, port=port))
165
+
166
+    def get_remote(self, ip, port=22):
167
+        """ Function returns remote SSH connection to node by ip address
168
+
169
+        :param ip: IP of host
170
+        :type ip: str
171
+        :param port: port for SSH
172
+        :type port: int
173
+        :rtype: SSHClient
174
+        """
175
+        if (ip, port) in self.connections:
176
+            logger.debug('SSH_MANAGER: Return existed connection for '
177
+                         '{ip}:{port}'.format(ip=ip, port=port))
178
+        else:
179
+            self.init_remote(ip=ip, port=port)
142 180
         logger.debug('SSH_MANAGER: Connections {0}'.format(self.connections))
143
-        return self._connect(self.connections[(ip, port)])
181
+        return self.connect(self.connections[(ip, port)])
144 182
 
145
-    def update_connection(self, ip, login=None, password=None,
146
-                          keys=None, port=22):
183
+    def update_connection(self, ip, port=22, login=None, password=None,
184
+                          keys=None):
147 185
         """Update existed connection
148 186
 
149 187
         :param ip: host ip string
188
+        :param port: ssh port int
150 189
         :param login: login string
151 190
         :param password: password string
152 191
         :param keys: list of keys
153
-        :param port: ssh port int
154 192
         :return: None
155 193
         """
156 194
         if (ip, port) in self.connections:
157
-            logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format(
158
-                ip=ip, port=port))
159
-            self.connections[(ip, port)].clear()
160
-            logger.info('SSH_MANAGER: Create new connection for '
161
-                        '{ip}:{port}'.format(ip=ip, port=port))
162
-
163
-        self.connections[(ip, port)] = SSHClient(
164
-            host=ip,
165
-            port=port,
166
-            username=login,
167
-            password=password,
168
-            private_keys=keys if keys is not None else []
169
-        )
195
+            logger.debug('SSH_MANAGER: Close connection for {ip}:{port}'
196
+                         .format(ip=ip, port=port))
197
+            ssh_client = self.connections.pop((ip, port))
198
+            ssh_client.close()
199
+        if login and (password or keys):
200
+            custom_creds = {
201
+                'username': login,
202
+                'password': password,
203
+                'keys': keys
204
+            }
205
+        else:
206
+            custom_creds = None
207
+        self.init_remote(ip=ip, port=port, custom_creds=custom_creds)
170 208
 
171 209
     def clean_all_connections(self):
172 210
         for (ip, port), connection in self.connections.items():
173 211
             connection.clear()
174
-            logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format(
175
-                ip=ip, port=port))
212
+            logger.debug('SSH_MANAGER: Close connection for {ip}:{port}'
213
+                         .format(ip=ip, port=port))
176 214
 
177 215
     def execute(self, ip, cmd, port=22, sudo=None):
178 216
         remote = self.get_remote(ip=ip, port=port)

+ 1
- 0
fuelweb_test/models/environment.py View File

@@ -377,6 +377,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
377 377
             )
378 378
             self.ssh_manager.update_connection(
379 379
                 ip=self.ssh_manager.admin_ip,
380
+                port=22,
380 381
                 login=new_login,
381 382
                 password=new_password
382 383
             )

+ 2
- 7
fuelweb_test/tests/test_multipath_devices.py View File

@@ -25,7 +25,6 @@ from fuelweb_test.settings import MULTIPATH
25 25
 from fuelweb_test.settings import MULTIPATH_TEMPLATE
26 26
 from fuelweb_test.settings import NEUTRON_SEGMENT
27 27
 from fuelweb_test.settings import SLAVE_MULTIPATH_DISKS_COUNT
28
-from fuelweb_test.settings import SSH_FUEL_CREDENTIALS
29 28
 from fuelweb_test.settings import REPLACE_DEFAULT_REPOS
30 29
 from fuelweb_test.settings import REPLACE_DEFAULT_REPOS_ONLY_ONCE
31 30
 from fuelweb_test.tests import base_test_case
@@ -53,9 +52,7 @@ class TestMultipath(base_test_case.TestBasic):
53 52
         """
54 53
         cmd = "multipath -l -v2"
55 54
 
56
-        ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'],
57
-                                      SSH_FUEL_CREDENTIALS['password'],
58
-                                      keys=ssh_manager._get_keys())
55
+        ssh_manager.update_connection(ip)
59 56
         ssh_manager.get_remote(ip)
60 57
         result = ssh_manager.execute_on_remote(
61 58
             ip=ip,
@@ -106,9 +103,7 @@ class TestMultipath(base_test_case.TestBasic):
106 103
         """
107 104
         cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep -c lvm"
108 105
 
109
-        ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'],
110
-                                      SSH_FUEL_CREDENTIALS['password'],
111
-                                      keys=ssh_manager._get_keys())
106
+        ssh_manager.update_connection(ip)
112 107
         ssh_manager.get_remote(ip)
113 108
         result = ssh_manager.execute_on_remote(
114 109
             ip=ip,

Loading…
Cancel
Save