Merge "Improve failover of SSH connections"

fuelweb_test/helpers/checkers.py

@@ -1466,8 +1466,8 @@ def check_package_version(ip, package_name, expected_version, condition='ge'):
     :param condition: predicate can be on of eq, ne, lt, le, ge, gt
     :return None: or raise UnexpectedExitCode
     """
-    cmd = "dpkg -s {0} " \
+    cmd = ("dpkg -s {0} "
-          "| awk -F': ' '/Version/ {{print \$2}}'".format(package_name)
+           "| awk -F': ' '/Version/ {{print $2}}'".format(package_name))
     logger.debug(cmd)
     result = ssh_manager.execute_on_remote(
         ip,

fuelweb_test/helpers/ssh_manager.py

@@ -19,9 +19,10 @@ import traceback
 from warnings import warn
 
 from devops.helpers.metaclasses import SingletonMeta
+from devops.helpers.ssh_client import SSHAuth
 from devops.helpers.ssh_client import SSHClient
 from paramiko import RSAKey
-from paramiko.ssh_exception import AuthenticationException
+from paramiko import SSHException
 import six
 
 from fuelweb_test import logger
@@ -67,8 +68,15 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
         self.slave_login = slave_login
         self.__slave_password = slave_password
 
-    @staticmethod
+    def _get_keys(self):
-    def _connect(remote):
+        keys = []
+        admin_remote = self.get_remote(self.admin_ip)
+        key_string = '/root/.ssh/id_rsa'
+        with admin_remote.open(key_string) as f:
+            keys.append(RSAKey.from_private_key(f))
+        return keys
+
+    def connect(self, remote):
         """ Check if connection is stable and return this one
 
         :param remote:
@@ -80,20 +88,80 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
                     seconds=5,
                     error_message="Socket timeout! Forcing reconnection"):
                 remote.check_call("cd ~")
-        except:
+        except Exception:
             logger.debug(traceback.format_exc())
-            logger.info('SSHManager: Check for current connection fails. '
+            logger.debug('SSHManager: Check for current connection fails. '
-                        'Trying to reconnect')
+                         'Trying to reconnect')
-            remote.reconnect()
+            remote = self.reconnect(remote)
         return remote
 
-    def _get_keys(self):
+    def reconnect(self, remote):
-        keys = []
+        """ Reconnect to remote or update connection
-        admin_remote = self.get_remote(self.admin_ip)
+
-        key_string = '/root/.ssh/id_rsa'
+        :param remote:
-        with admin_remote.open(key_string) as f:
+        :return:
-            keys.append(RSAKey.from_private_key(f))
+        """
-        return keys
+        ip = remote.hostname
+        port = remote.port
+        try:
+            remote.reconnect()
+        except SSHException:
+            self.update_connection(ip=ip, port=port)
+        return self.connections[(ip, port)]
+
+    def init_remote(self, ip, port=22, custom_creds=None):
+        """ Initialise connection to remote
+
+        :param ip: IP of host
+        :type ip: str
+        :param port: port for SSH
+        :type port: int
+        :param custom_creds: custom creds
+        :type custom_creds: dict
+        """
+        logger.debug('SSH_MANAGER: Create new connection for '
+                     '{ip}:{port}'.format(ip=ip, port=port))
+
+        keys = self._get_keys() if ip != self.admin_ip else []
+        if ip == self.admin_ip:
+            ssh_client = SSHClient(
+                host=ip,
+                port=port,
+                auth=SSHAuth(
+                    username=self.admin_login,
+                    password=self.__admin_password,
+                    keys=keys)
+            )
+            ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo']
+        elif custom_creds:
+            ssh_client = SSHClient(
+                host=ip,
+                port=port,
+                auth=SSHAuth(**custom_creds))
+        else:
+            try:
+                ssh_client = SSHClient(
+                    host=ip,
+                    port=port,
+                    auth=SSHAuth(
+                        username=self.slave_login,
+                        password=self.__slave_password,
+                        keys=keys)
+                )
+            except SSHException:
+                ssh_client = SSHClient(
+                    host=ip,
+                    port=port,
+                    auth=SSHAuth(
+                        username=self.slave_fallback_login,
+                        password=self.__slave_password,
+                        keys=keys)
+                )
+            ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo']
+
+        self.connections[(ip, port)] = ssh_client
+        logger.debug('SSH_MANAGER: New connection for '
+                     '{ip}:{port} is created'.format(ip=ip, port=port))
 
     def get_remote(self, ip, port=22):
         """ Function returns remote SSH connection to node by ip address
@@ -104,75 +172,45 @@ class SSHManager(six.with_metaclass(SingletonMeta, object)):
         :type port: int
         :rtype: SSHClient
         """
-        if (ip, port) not in self.connections:
+        if (ip, port) in self.connections:
-            logger.debug('SSH_MANAGER: Create new connection for '
+            logger.debug('SSH_MANAGER: Return existed connection for '
                          '{ip}:{port}'.format(ip=ip, port=port))
-
+        else:
-            keys = self._get_keys() if ip != self.admin_ip else []
+            self.init_remote(ip=ip, port=port)
-            if ip == self.admin_ip:
-                ssh_client = SSHClient(
-                    host=ip,
-                    port=port,
-                    username=self.admin_login,
-                    password=self.__admin_password,
-                    private_keys=keys
-                )
-                ssh_client.sudo_mode = SSH_FUEL_CREDENTIALS['sudo']
-            else:
-                try:
-                    ssh_client = SSHClient(
-                        host=ip,
-                        port=port,
-                        username=self.slave_login,
-                        password=self.__slave_password,
-                        private_keys=keys
-                    )
-                except AuthenticationException:
-                    ssh_client = SSHClient(
-                        host=ip,
-                        port=port,
-                        username=self.slave_fallback_login,
-                        password=self.__slave_password,
-                        private_keys=keys
-                    )
-                ssh_client.sudo_mode = SSH_SLAVE_CREDENTIALS['sudo']
-            self.connections[(ip, port)] = ssh_client
-        logger.debug('SSH_MANAGER: Return existed connection for '
-                     '{ip}:{port}'.format(ip=ip, port=port))
         logger.debug('SSH_MANAGER: Connections {0}'.format(self.connections))
-        return self._connect(self.connections[(ip, port)])
+        return self.connect(self.connections[(ip, port)])
 
-    def update_connection(self, ip, login=None, password=None,
+    def update_connection(self, ip, port=22, login=None, password=None,
-                          keys=None, port=22):
+                          keys=None):
         """Update existed connection
 
         :param ip: host ip string
+        :param port: ssh port int
         :param login: login string
         :param password: password string
         :param keys: list of keys
-        :param port: ssh port int
         :return: None
         """
         if (ip, port) in self.connections:
-            logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format(
+            logger.debug('SSH_MANAGER: Close connection for {ip}:{port}'
-                ip=ip, port=port))
+                         .format(ip=ip, port=port))
-            self.connections[(ip, port)].clear()
+            ssh_client = self.connections.pop((ip, port))
-            logger.info('SSH_MANAGER: Create new connection for '
+            ssh_client.close()
-                        '{ip}:{port}'.format(ip=ip, port=port))
+        if login and (password or keys):
-
+            custom_creds = {
-        self.connections[(ip, port)] = SSHClient(
+                'username': login,
-            host=ip,
+                'password': password,
-            port=port,
+                'keys': keys
-            username=login,
+            }
-            password=password,
+        else:
-            private_keys=keys if keys is not None else []
+            custom_creds = None
-        )
+        self.init_remote(ip=ip, port=port, custom_creds=custom_creds)
 
     def clean_all_connections(self):
         for (ip, port), connection in self.connections.items():
             connection.clear()
-            logger.info('SSH_MANAGER: Close connection for {ip}:{port}'.format(
+            logger.debug('SSH_MANAGER: Close connection for {ip}:{port}'
-                ip=ip, port=port))
+                         .format(ip=ip, port=port))
 
     def execute(self, ip, cmd, port=22, sudo=None):
         remote = self.get_remote(ip=ip, port=port)

fuelweb_test/models/environment.py

@@ -377,6 +377,7 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)):
             )
             self.ssh_manager.update_connection(
                 ip=self.ssh_manager.admin_ip,
+                port=22,
                 login=new_login,
                 password=new_password
             )

fuelweb_test/tests/test_multipath_devices.py

@@ -25,7 +25,6 @@ from fuelweb_test.settings import MULTIPATH
 from fuelweb_test.settings import MULTIPATH_TEMPLATE
 from fuelweb_test.settings import NEUTRON_SEGMENT
 from fuelweb_test.settings import SLAVE_MULTIPATH_DISKS_COUNT
-from fuelweb_test.settings import SSH_FUEL_CREDENTIALS
 from fuelweb_test.settings import REPLACE_DEFAULT_REPOS
 from fuelweb_test.settings import REPLACE_DEFAULT_REPOS_ONLY_ONCE
 from fuelweb_test.tests import base_test_case
@@ -53,9 +52,7 @@ class TestMultipath(base_test_case.TestBasic):
         """
         cmd = "multipath -l -v2"
 
-        ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'],
+        ssh_manager.update_connection(ip)
-                                      SSH_FUEL_CREDENTIALS['password'],
-                                      keys=ssh_manager._get_keys())
         ssh_manager.get_remote(ip)
         result = ssh_manager.execute_on_remote(
             ip=ip,
@@ -106,9 +103,7 @@ class TestMultipath(base_test_case.TestBasic):
         """
         cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep -c lvm"
 
-        ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'],
+        ssh_manager.update_connection(ip)
-                                      SSH_FUEL_CREDENTIALS['password'],
-                                      keys=ssh_manager._get_keys())
         ssh_manager.get_remote(ip)
         result = ssh_manager.execute_on_remote(
             ip=ip,