From 899c83af044b33b7f5ee5f4bd666d453a64e8104 Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Thu, 23 May 2019 09:42:49 +0200 Subject: [PATCH] Fix SSH connection procedure when server instance is booting. When connecting to a server instance that still hasn't networking setup complete (for example because it is still booting), the first SSH connection attemp fails. Then Tobiko it is no more able to recover from the first failure. This should fix it. Change-Id: I8ce3e213cf641fa5cbe06cdb56d60eda539b018a --- tobiko/openstack/nova/config.py | 5 +- tobiko/shell/paramiko/__init__.py | 5 ++ tobiko/shell/paramiko/_client.py | 124 +++++++++++------------------- tobiko/shell/paramiko/config.py | 2 +- 4 files changed, 56 insertions(+), 80 deletions(-) diff --git a/tobiko/openstack/nova/config.py b/tobiko/openstack/nova/config.py index 63a93427c..d41e99497 100644 --- a/tobiko/openstack/nova/config.py +++ b/tobiko/openstack/nova/config.py @@ -24,4 +24,7 @@ def register_tobiko_options(conf): cfg.StrOpt('flavor', help="Default flavor for new server instances"), cfg.StrOpt('key_file', default='~/.ssh/id_rsa', - help="ssh key file for new server instances")]) + help="Default SSH key to login to server instances"), + cfg.StrOpt('username', default='cirros', + help="Default username to login to server instances") + ]) diff --git a/tobiko/shell/paramiko/__init__.py b/tobiko/shell/paramiko/__init__.py index 691f148d3..cc05c2cae 100644 --- a/tobiko/shell/paramiko/__init__.py +++ b/tobiko/shell/paramiko/__init__.py @@ -15,11 +15,16 @@ # under the License. from __future__ import absolute_import +import paramiko + from tobiko.shell.paramiko import _config from tobiko.shell.paramiko import _client +SSHException = paramiko.SSHException + SSHHostConfig = _config.SSHHostConfig SSHClientFixture = _client.SSHClientFixture ssh_client = _client.ssh_client ssh_proxy_client = _client.ssh_proxy_client +SSHConnectFailure = _client.SSHConnectFailure diff --git a/tobiko/shell/paramiko/_client.py b/tobiko/shell/paramiko/_client.py index d5fed1adb..e61b1e40d 100644 --- a/tobiko/shell/paramiko/_client.py +++ b/tobiko/shell/paramiko/_client.py @@ -75,8 +75,8 @@ SSH_CONNECT_PARAMETERS = { #: The targets name in the kerberos database. default: hostname 'gss_host': str, - #:Indicates whether or not the DNS is trusted to securely canonicalize the - # name of the host being connected to (default True). + #: Indicates whether or not the DNS is trusted to securely canonicalize the + # name of the host being connected to (default True). 'gss_trust_dns': bool, #: An optional timeout (in seconds) to wait for the SSH banner to be @@ -88,6 +88,10 @@ SSH_CONNECT_PARAMETERS = { } +class SSHConnectFailure(tobiko.TobikoException): + message = "Failed to login to {login}\n{cause}" + + class SSHClientFixture(tobiko.SharedFixture): host = None @@ -102,9 +106,6 @@ class SSHClientFixture(tobiko.SharedFixture): proxy_client = None proxy_command = None - #: An open socket or socket-like object (such as a Channel) to use for - # communication to the target host - proxy_sock = None connect_parameters = None @@ -127,7 +128,6 @@ class SSHClientFixture(tobiko.SharedFixture): def setup_fixture(self): self.setup_host_config() self.setup_connect_parameters() - self.setup_proxy_sock() self.setup_ssh_client() def setup_host_config(self): @@ -193,11 +193,41 @@ class SSHClientFixture(tobiko.SharedFixture): message = "Invalid timeout: {!r}".format(port) raise ValueError(message) - def setup_proxy_sock(self): - if self.proxy_sock: - # Proxy sock already set up - return + def setup_ssh_client(self): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.load_system_host_keys() + now = time.time() + parameters = dict(self.connect_parameters) + deadline = now + parameters.pop('timeout') + sleep_time = self.connect_sleep_time + login = self.connect_login + while True: + timeout = deadline - now + LOG.debug("Logging in to %r... (time left %d seconds)", login, + timeout) + try: + sock = self._open_proxy_sock() + client.connect(sock=sock, timeout=timeout, **parameters) + except (EOFError, socket.error, socket.timeout, + paramiko.SSHException) as ex: + now = time.time() + if now + sleep_time >= deadline: + raise SSHConnectFailure(login=login, cause=ex) + + LOG.debug("Error logging in to %s (%s); retrying in %d " + "seconds...", login, ex, sleep_time) + time.sleep(sleep_time) + sleep_time += self.connect_sleep_time_increment + + else: + self.client = client + self.addCleanup(client.close) + LOG.info("Successfully logged it to %s", login) + break + + def _open_proxy_sock(self): proxy_command = self.host_config.proxy_command or self.proxy_command proxy_client = self.proxy_client if proxy_client: @@ -205,7 +235,7 @@ class SSHClientFixture(tobiko.SharedFixture): proxy_command = proxy_command or 'nc {hostname!r} {port!r}' elif not proxy_command: # Proxy sock is not required - return + return None # Apply connect parameters to proxy command parameters = self.connect_parameters @@ -222,76 +252,14 @@ class SSHClientFixture(tobiko.SharedFixture): # Open proxy channel LOG.debug("Execute proxy command with proxy client %r: %r", proxy_client, proxy_command) - self.proxy_sock = proxy_client.get_transport().open_session() - self.addCleanup(self.cleanup_proxy_sock) - self.proxy_sock.exec_command(proxy_command) + proxy_sock = proxy_client.get_transport().open_session() + proxy_sock.exec_command(proxy_command) else: LOG.debug("Execute proxy command on local host: %r", proxy_command) - self.proxy_sock = paramiko.ProxyCommand(proxy_command) - self.addCleanup(self.cleanup_proxy_sock) + proxy_sock = paramiko.ProxyCommand(proxy_command) - def setup_ssh_client(self): - client = paramiko.SSHClient() - client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - client.load_system_host_keys() - - now = time.time() - parameters = dict(self.connect_parameters) - deadline = now + parameters.pop('timeout') - sleep_time = self.connect_sleep_time - while True: - timeout = deadline - now - message = "time left {!s} seconds".format(timeout) - try: - self._connect_client(client, - message=message, - timeout=timeout, - **parameters) - break - except (EOFError, socket.error, socket.timeout, - paramiko.SSHException): - now = time.time() - if now + sleep_time >= sleep_time: - raise - - LOG.debug('Retry to connect to %r in %d seconds', - self.connect_login, sleep_time) - time.sleep(sleep_time) - now = time.time() - if now >= deadline: - raise - sleep_time += self.connect_sleep_time_increment - - def _connect_client(self, client, message=None, **parameters): - """Returns an ssh connection to the specified host.""" - extra_info = '' - if message: - extra_info = ' (' + message + ')' - LOG.info("Creating SSH connection to %r%s...", self.connect_login, - extra_info) - - try: - client.connect(sock=self.proxy_sock, **parameters) - except Exception as ex: - LOG.debug("Error connecting to %s%s: %s", self.connect_login, - extra_info, ex) - raise - else: - self.client = client - self.addCleanup(self.cleanup_ssh_client, client) - LOG.info("SSH connection to %s successfully created", - self.connect_login) - - def cleanup_ssh_client(self): - if self.client: - self.client = None - self.client.close() - - def cleanup_proxy_sock(self): - proxy_sock = self.proxy_sock - if proxy_sock: - self.proxy_sock = None - proxy_sock.close() + self.addCleanup(proxy_sock.close) + return proxy_sock @property def connect_sleep_time(self): diff --git a/tobiko/shell/paramiko/config.py b/tobiko/shell/paramiko/config.py index 9431f0b57..8afe96ddc 100644 --- a/tobiko/shell/paramiko/config.py +++ b/tobiko/shell/paramiko/config.py @@ -65,4 +65,4 @@ def setup_tobiko_config(conf): paramiko_logger.logger.setLevel(log.DEBUG) elif paramiko_logger.isEnabledFor(log.DEBUG): # Silence paramiko debugging messages - paramiko_logger.logger.setLevel(log.INFO) + paramiko_logger.logger.setLevel(log.WARNING)