Connect to TripleO cloud via its hypervisor host

This allows to connect to remote TripleO hypervisor host by setting
a simple option into tobiko.conf as below:

 [ssh]
 proxy_jump = root@<my-remote-host-name>

The patch requires you to add your ~/.ssh/id_rsa SSH key to the
hypervisor host befor executing test cases.

Tobiko will automatically download the ~/.ssh/id_rsa file from the
remote hypervisor host and use it to connect to remote undercloud-0
host.

The patch also fixes the problem of having to manually delete overcloud
SSH key files when changing to a new TripleO cloud.

Change-Id: I0037d09f0f5285dcc861ba5c286adfb14364e868

tobiko/shell/ssh/_client.py

@@ -486,8 +486,8 @@ class SSHClientManager(object):
             return proxy_jump
         host_config = host_config or _config.ssh_host_config(
             host=host, config_files=config_files)
-        proxy_host = host_config.proxy_jump
+        proxy_jump = host_config.proxy_jump
-        return proxy_host and self.get_client(proxy_host) or None
+        return proxy_jump and self.get_client(proxy_jump) or None
 
 
 CLIENTS = SSHClientManager()
@@ -505,11 +505,19 @@ def ssh_client(host, port=None, username=None, proxy_jump=None,
 
 def ssh_connect(hostname, username=None, port=None, connection_interval=None,
                 connection_attempts=None, connection_timeout=None,
-                proxy_command=None, proxy_client=None, **parameters):
+                proxy_command=None, proxy_client=None, key_filename=None,
+                **parameters):
     client = paramiko.SSHClient()
     client.set_missing_host_key_policy(paramiko.WarningPolicy())
     login = _command.ssh_login(hostname=hostname, username=username, port=port)
 
+    if key_filename:
+        # Ensures we try enough times to try all keys
+        tobiko.check_valid_type(key_filename, list)
+        connection_attempts = max(connection_attempts or 1,
+                                  len(key_filename),
+                                  1)
+
     for attempt in tobiko.retry(count=connection_attempts,
                                 timeout=connection_timeout,
                                 interval=connection_interval,
@@ -533,7 +541,17 @@ def ssh_connect(hostname, username=None, port=None, connection_interval=None,
                            username=username,
                            port=port,
                            sock=proxy_sock,
+                           key_filename=key_filename,
                            **parameters)
+        except ValueError as ex:
+            if (str(ex) == 'q must be exactly 160, 224, or 256 bits long' and
+                    key_filename):
+                # Must try without the first key
+                LOG.debug("Retry connecting with the next key")
+                key_filename = key_filename[1:] + [key_filename[0]]
+                continue
+            else:
+                raise
         except (EOFError, socket.error, socket.timeout,
                 paramiko.SSHException) as ex:
             attempt.check_limits()

tobiko/shell/ssh/_config.py

@@ -17,6 +17,8 @@ from __future__ import absolute_import
 
 import collections
 import os
+import typing  # noqa
+import urllib
 
 from oslo_log import log
 import paramiko
@@ -77,13 +79,35 @@ class SSHConfigFixture(tobiko.SharedFixture):
                     self.config.parse(f)
                 LOG.debug("File %r parsed.", config_file)
 
-    def lookup(self, host=None):
+    def lookup(self,
-        host_config = host and self.config.lookup(host) or {}
+               host: typing.Optional[str] = None,
+               hostname: typing.Optional[str] = None,
+               username: typing.Optional[str] = None,
+               port: typing.Optional[int] = None):
+        if host and ('@' in host or ':' in host):
+            host_url = urllib.parse.urlparse(f"ssh://{host}")
+            hostname = hostname or host_url.hostname or None
+            username = username or host_url.username or None
+            port = port or host_url.port or None
+        else:
+            hostname = hostname or host
+
+        if hostname and self.config:
+            host_config: dict = self.config.lookup(hostname)
+        else:
+            host_config = {}
+
         # remove unsupported directive
         include_files = host_config.pop('include', None)
         if include_files:
             LOG.warning('Ignoring unsupported directive: Include %s',
                         include_files)
+        if hostname:
+            host_config.setdefault('hostname', hostname)
+        if username:
+            host_config.setdefault('user', username)
+        if port:
+            host_config.setdefault('port', port)
         return SSHHostConfig(host=host,
                              ssh_config=self,
                              host_config=host_config,
@@ -117,8 +141,20 @@ class SSHHostConfig(collections.namedtuple('SSHHostConfig', ['host',
 
     @property
     def key_filename(self):
-        return (self.host_config.get('identityfile') or
+        key_filename = []
-                self.default.key_file)
+        host_config_key_files = self.host_config.get('identityfile')
+        if host_config_key_files:
+            for filename in host_config_key_files:
+                if filename:
+                    key_filename.append(tobiko.tobiko_config_path(filename))
+
+        default_key_files = self.default.key_file
+        if default_key_files:
+            for filename in default_key_files:
+                if filename:
+                    key_filename.append(tobiko.tobiko_config_path(filename))
+
+        return key_filename
 
     @property
     def proxy_jump(self):

tobiko/shell/ssh/_ssh_key_file.py

@@ -0,0 +1,83 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import typing  # noqa
+
+from oslo_log import log
+
+import tobiko
+from tobiko.shell.ssh import _client
+
+
+LOG = log.getLogger(__name__)
+
+
+DEFAULT_SSH_KEY_FILE = "~/.ssh/id_rsa"
+
+
+def get_key_file(ssh_client: _client.SSHClientFixture,
+                 key_file: str = DEFAULT_SSH_KEY_FILE):
+    return tobiko.setup_fixture(
+        GetSSHKeyFileFixture(ssh_client=ssh_client,
+                             remote_key_file=key_file)).key_file
+
+
+class GetSSHKeyFileFixture(tobiko.SharedFixture):
+
+    key_file = None
+
+    def __init__(self, ssh_client: _client.SSHClientFixture,
+                 remote_key_file: str = DEFAULT_SSH_KEY_FILE):
+        super(GetSSHKeyFileFixture, self).__init__()
+        self.ssh_client = ssh_client
+        self.remote_key_file = remote_key_file
+
+    def setup_fixture(self):
+        client = self.ssh_client.connect()
+        _, stdout, stderr = client.exec_command('hostname')
+        remote_hostname = stdout.read().strip().decode()
+        if not remote_hostname:
+            error = stderr.read()
+            raise RuntimeError(
+                "Unable to get hostname from proxy jump server:\n"
+                f"{error}")
+
+        _, stdout, stderr = client.exec_command(
+            f"cat {self.remote_key_file}")
+        private_key = stdout.read()
+        if not private_key:
+            error = stderr.read()
+            LOG.error("Unable to get SSH private key from proxy jump "
+                      f"server:\n{error}")
+            return
+
+        _, stdout, stderr = client.exec_command(
+            f"cat {self.remote_key_file}.pub")
+        public_key = stdout.read()
+        if not public_key:
+            error = stderr.read()
+            LOG.error("Unable to get SSH public key from proxy jump "
+                      f"server:\n{error}")
+            return
+
+        key_file = tobiko.tobiko_config_path(
+            f"~/.ssh/id_rsa-{remote_hostname}")
+        with tobiko.open_output_file(key_file) as fd:
+            fd.write(private_key.decode())
+        with tobiko.open_output_file(key_file + '.pub') as fd:
+            fd.write(public_key.decode())
+        self.key_file = key_file

tobiko/shell/ssh/config.py

@@ -14,10 +14,13 @@
 from __future__ import absolute_import
 
 import itertools
+import os
 
 from oslo_config import cfg
 from oslo_log import log
 
+LOG = log.getLogger(__name__)
+
 GROUP_NAME = 'ssh'
 OPTIONS = [
     cfg.BoolOpt('debug',
@@ -36,9 +39,9 @@ OPTIONS = [
     cfg.ListOpt('config_files',
                 default=['ssh_config'],
                 help="Default user SSH configuration files"),
-    cfg.StrOpt('key_file',
+    cfg.ListOpt('key_file',
-               default='~/.ssh/id_rsa',
+                default=['~/.ssh/id_rsa'],
-               help="Default SSH private key file"),
+                help="Default SSH private key file(s)"),
     cfg.BoolOpt('allow_agent',
                 default=False,
                 help=("Set to False to disable connecting to the "
@@ -79,6 +82,9 @@ def list_options():
 
 
 def setup_tobiko_config(conf):
+    from tobiko.shell.ssh import _client
+    from tobiko.shell.ssh import _ssh_key_file
+
     paramiko_logger = log.getLogger('paramiko')
     if conf.ssh.debug:
         if not paramiko_logger.isEnabledFor(log.DEBUG):
@@ -88,3 +94,10 @@ def setup_tobiko_config(conf):
         if paramiko_logger.isEnabledFor(log.ERROR):
             # Silence paramiko debugging messages
             paramiko_logger.logger.setLevel(log.FATAL)
+
+    ssh_proxy_client = _client.ssh_proxy_client()
+    if ssh_proxy_client:
+        key_file = _ssh_key_file.get_key_file(ssh_client=ssh_proxy_client)
+        if key_file and os.path.isfile(key_file):
+            LOG.info(f"Use SSH proxy server keyfile: {key_file}")
+            conf.ssh.key_file.append(key_file)

tobiko/tripleo/_overcloud.py

@@ -104,10 +104,7 @@ class OvercloudSshKeyFileFixture(tobiko.SharedFixture):
             CONF.tobiko.tripleo.overcloud_ssh_key_filename)
 
     def setup_fixture(self):
-        key_filename = self.key_filename
-        if not os.path.isfile(key_filename):
         self.setup_key_file()
-            assert os.path.isfile(key_filename)
 
     def setup_key_file(self):
         key_filename = self.key_filename