diff --git a/tobiko/shell/ssh/__init__.py b/tobiko/shell/ssh/__init__.py index 60ec2f296..2d30699bc 100644 --- a/tobiko/shell/ssh/__init__.py +++ b/tobiko/shell/ssh/__init__.py @@ -19,6 +19,7 @@ from tobiko.shell.ssh import _config from tobiko.shell.ssh import _client from tobiko.shell.ssh import _command from tobiko.shell.ssh import _forward +from tobiko.shell.ssh import _key_file from tobiko.shell.ssh import _skip @@ -42,5 +43,8 @@ get_forward_port_address = _forward.get_forward_port_address SSHTunnelForwarderFixture = _forward.SSHTunnelForwarderFixture SSHTunnelForwarder = _forward.SSHTunnelForwarder +list_key_filenames = _key_file.list_key_filenames +list_proxy_jump_key_filenames = _key_file.list_proxy_jump_key_filenames + has_ssh_proxy_jump = _skip.has_ssh_proxy_jump skip_unless_has_ssh_proxy_jump = _skip.skip_unless_has_ssh_proxy_jump diff --git a/tobiko/shell/ssh/_client.py b/tobiko/shell/ssh/_client.py index 0f1015671..b627a1f22 100644 --- a/tobiko/shell/ssh/_client.py +++ b/tobiko/shell/ssh/_client.py @@ -558,7 +558,10 @@ class SSHClientManager(object): **connect_parameters) return new_client - def get_proxy_client(self, host=None, proxy_jump=None, host_config=None, + def get_proxy_client(self, + host=None, + proxy_jump=None, + host_config=None, config_files=None): if isinstance(proxy_jump, SSHClientFixture): return proxy_jump @@ -730,7 +733,9 @@ def ssh_proxy_sock(hostname=None, port=None, command=None, client=None, return sock -def ssh_proxy_client(manager=None, host=None, host_config=None, +def ssh_proxy_client(manager=None, + host=None, + host_config=None, config_files=None): manager = manager or CLIENTS return manager.get_proxy_client(host=host, diff --git a/tobiko/shell/ssh/_config.py b/tobiko/shell/ssh/_config.py index b822e59d6..61c332c1a 100644 --- a/tobiko/shell/ssh/_config.py +++ b/tobiko/shell/ssh/_config.py @@ -165,15 +165,11 @@ class SSHHostConfig(collections.namedtuple('SSHHostConfig', ['host', host_config_key_files = self.host_config.get('identityfile') if host_config_key_files: for filename in host_config_key_files: - if filename: - key_filename.append(tobiko.tobiko_config_path(filename)) - - default_key_files = self.default.key_file - if default_key_files: - for filename in default_key_files: - if filename: + if filename and os.path.isfile(filename): key_filename.append(tobiko.tobiko_config_path(filename)) + from tobiko.shell.ssh import _key_file + key_filename.extend(_key_file.list_key_filenames()) return key_filename @property @@ -184,7 +180,7 @@ class SSHHostConfig(collections.namedtuple('SSHHostConfig', ['host', return None proxy_hostname = self.ssh_config.lookup(proxy_jump).hostname - if ({proxy_jump, proxy_hostname} & {self.host, self.hostname}): + if {proxy_jump, proxy_hostname} & {self.host, self.hostname}: # Avoid recursive proxy jump definition return None diff --git a/tobiko/shell/ssh/_ssh_key_file.py b/tobiko/shell/ssh/_key_file.py similarity index 63% rename from tobiko/shell/ssh/_ssh_key_file.py rename to tobiko/shell/ssh/_key_file.py index 859c58af6..9431df4db 100644 --- a/tobiko/shell/ssh/_ssh_key_file.py +++ b/tobiko/shell/ssh/_key_file.py @@ -15,8 +15,8 @@ # under the License. from __future__ import absolute_import -import os.path -import typing # noqa +import os +import typing from oslo_log import log @@ -83,3 +83,47 @@ class GetSSHKeyFileFixture(tobiko.SharedFixture): with tobiko.open_output_file(key_file + '.pub') as fd: fd.write(public_key.decode()) self.key_file = key_file + + +def list_key_filenames() -> typing.List[str]: + return tobiko.setup_fixture(KeyFilenamesFixture).key_filenames + + +def list_proxy_jump_key_filenames() -> typing.List[str]: + return tobiko.setup_fixture(ProxyJumpKeyFilenamesFixture).key_filenames + + +class KeyFilenamesFixture(tobiko.SharedFixture): + + def __init__(self): + super().__init__() + self.key_filenames: typing.List[str] = [] + + def setup_fixture(self): + conf = tobiko.tobiko_config() + for key_file in tobiko.select_uniques(conf.ssh.key_file): + if key_file: + key_file = tobiko.tobiko_config_path(key_file) + if (key_file not in self.key_filenames and + os.path.isfile(key_file)): + LOG.info(f"Use local keyfile: {key_file}") + self.key_filenames.append(key_file) + + +class ProxyJumpKeyFilenamesFixture(KeyFilenamesFixture): + + def setup_fixture(self): + ssh_proxy_client = _client.ssh_proxy_client() + if ssh_proxy_client is None: + return + + conf = tobiko.tobiko_config() + remote_key_files = tobiko.select_uniques(conf.ssh.key_file) + for remote_key_file in remote_key_files: + key_file = get_key_file(ssh_client=ssh_proxy_client, + key_file=remote_key_file) + if (key_file and + key_file not in self.key_filenames and + os.path.isfile(key_file)): + LOG.info(f"Use SSH proxy server keyfile: {key_file}") + self.key_filenames.append(key_file) diff --git a/tobiko/shell/ssh/config.py b/tobiko/shell/ssh/config.py index adef6e29b..93c6c3c44 100644 --- a/tobiko/shell/ssh/config.py +++ b/tobiko/shell/ssh/config.py @@ -14,7 +14,6 @@ from __future__ import absolute_import import itertools -import os from oslo_config import cfg from oslo_log import log @@ -86,9 +85,6 @@ def list_options(): def setup_tobiko_config(conf): - from tobiko.shell.ssh import _client - from tobiko.shell.ssh import _ssh_key_file - paramiko_logger = log.getLogger('paramiko') if conf.ssh.debug: if not paramiko_logger.isEnabledFor(log.DEBUG): @@ -98,13 +94,3 @@ def setup_tobiko_config(conf): if paramiko_logger.isEnabledFor(log.ERROR): # Silence paramiko debugging messages paramiko_logger.logger.setLevel(log.FATAL) - - ssh_proxy_client = _client.ssh_proxy_client() - if ssh_proxy_client: - key_file: str - for remote_key_file in conf.ssh.key_file: - key_file = _ssh_key_file.get_key_file(ssh_client=ssh_proxy_client, - key_file=remote_key_file) - if key_file and os.path.isfile(key_file): - LOG.info(f"Use SSH proxy server keyfile: {key_file}") - conf.ssh.key_file.append(key_file) diff --git a/tobiko/tests/functional/tripleo/test_overcloud.py b/tobiko/tests/functional/tripleo/test_overcloud.py index dbd584256..730c22cc2 100644 --- a/tobiko/tests/functional/tripleo/test_overcloud.py +++ b/tobiko/tests/functional/tripleo/test_overcloud.py @@ -30,7 +30,7 @@ import tobiko CONF = config.CONF -@tripleo.skip_if_missing_overcloud +@tripleo.skip_if_missing_undercloud class OvercloudKeystoneCredentialsTest(testtools.TestCase): def test_fetch_overcloud_credentials(self): diff --git a/tobiko/tests/unit/test_select.py b/tobiko/tests/unit/test_select.py index 906ff7b51..a1603a830 100644 --- a/tobiko/tests/unit/test_select.py +++ b/tobiko/tests/unit/test_select.py @@ -151,6 +151,13 @@ class SelectionTest(unit.TobikoUnitTest): selection = self.create_selection([a, b, c]) self.assertEqual([a], selection.unselect(lambda obj: obj.number == 1)) + def test_select_uniques(self): + a = Obj(0, 'a') + b = Obj(1, 'b') + c = Obj(1, 'c') + selection = self.create_selection([a, b, c, a, b, c]) + self.assertEqual([a, b, c], selection.select_uniques()) + class SelectTest(SelectionTest): diff --git a/tobiko/tests/unit/tripleo/test_config.py b/tobiko/tests/unit/tripleo/test_config.py index 71d5b052f..a4f9ae62e 100644 --- a/tobiko/tests/unit/tripleo/test_config.py +++ b/tobiko/tests/unit/tripleo/test_config.py @@ -24,7 +24,9 @@ TIPLEO_CONF = CONF.tobiko.tripleo class TripleoConfigTest(unit.TobikoUnitTest): def test_ssh_key_filename(self): - self.assertIsInstance(TIPLEO_CONF.undercloud_ssh_key_filename, str) + value = TIPLEO_CONF.undercloud_ssh_key_filename + if value is not None: + self.assertIsInstance(value, str) class UndercloudConfigTest(unit.TobikoUnitTest): diff --git a/tobiko/tripleo/__init__.py b/tobiko/tripleo/__init__.py index ee4b8453e..7621ac465 100644 --- a/tobiko/tripleo/__init__.py +++ b/tobiko/tripleo/__init__.py @@ -31,6 +31,8 @@ skip_unless_undercloud_has_ansible = \ _ansible.skip_unless_undercloud_has_ansible run_playbook_from_undercloud = _ansible.run_playbook_from_undercloud +OvercloudKeystoneCredentialsFixture = \ + overcloud.OvercloudKeystoneCredentialsFixture find_overcloud_node = overcloud.find_overcloud_node list_overcloud_nodes = overcloud.list_overcloud_nodes load_overcloud_rcfile = overcloud.load_overcloud_rcfile diff --git a/tobiko/tripleo/_overcloud.py b/tobiko/tripleo/_overcloud.py index d89a3d5f6..d8939f5f7 100644 --- a/tobiko/tripleo/_overcloud.py +++ b/tobiko/tripleo/_overcloud.py @@ -39,7 +39,7 @@ def has_overcloud(): return _undercloud.has_undercloud() -def load_overcloud_rcfile(): +def load_overcloud_rcfile() -> typing.Dict[str, str]: return _undercloud.fetch_os_env(*CONF.tobiko.tripleo.overcloud_rcfile) @@ -50,10 +50,13 @@ skip_if_missing_overcloud = tobiko.skip_unless( class OvercloudKeystoneCredentialsFixture( keystone.EnvironKeystoneCredentialsFixture): - def get_environ(self): - if has_overcloud(): + def get_environ(self) -> typing.Dict[str, str]: + LOG.debug('Looking for credentials from TripleO undercloud host...') + if _undercloud.has_undercloud(): return load_overcloud_rcfile() else: + LOG.debug("TripleO undercloud host not available for fetching " + 'credentials files.') return {} diff --git a/tobiko/tripleo/_undercloud.py b/tobiko/tripleo/_undercloud.py index e8be95780..ce53e9b29 100644 --- a/tobiko/tripleo/_undercloud.py +++ b/tobiko/tripleo/_undercloud.py @@ -13,6 +13,8 @@ # under the License. from __future__ import absolute_import +import json +import os import typing from oslo_log import log @@ -32,7 +34,8 @@ def undercloud_ssh_client() -> ssh.SSHClientFixture: host_config = undercloud_host_config() if not host_config.hostname: raise NoSuchUndercloudHostname('No such undercloud hostname') - return ssh.ssh_client(host=host_config.hostname, host_config=host_config) + return ssh.ssh_client(host=host_config.hostname, + **host_config.connect_parameters) class NoSuchUndercloudHostname(tobiko.TobikoException): @@ -45,8 +48,11 @@ class InvalidRCFile(tobiko.TobikoException): def fetch_os_env(rcfile, *rcfiles) -> typing.Dict[str, str]: rcfiles = (rcfile,) + rcfiles + LOG.debug('Fetching OS environment variables from TripleO undercloud ' + f'host files: {",".join(rcfiles)}') errors = [] for rcfile in rcfiles: + LOG.debug(f'Reading rcfile: {rcfile}...') try: result = sh.execute(f". {rcfile}; env | grep '^OS_'", ssh_client=undercloud_ssh_client()) @@ -55,11 +61,15 @@ def fetch_os_env(rcfile, *rcfiles) -> typing.Dict[str, str]: f"({ex})") errors.append(tobiko.exc_info) else: + LOG.debug(f'Parsing environment variables from: {rcfile}...') env = {} for line in result.stdout.splitlines(): name, value = line.split('=') env[name] = value if env: + env_dump = json.dumps(env, sort_keys=True, indent=4) + LOG.debug(f'Environment variables read from: {rcfile}:\n' + f'{env_dump}') return env for error in errors: LOG.exception(f"Unable to get overcloud RC file '{rcfile}' " @@ -102,15 +112,18 @@ def check_undercloud() -> bool: try: ssh_client = undercloud_ssh_client() except NoSuchUndercloudHostname: + LOG.debug('TripleO undercloud hostname not found') return False try: ssh_client.connect(retry_count=1, connection_attempts=1, timeout=15.) except Exception as ex: - LOG.debug('Unable to connect to undercloud host: %s', ex, + LOG.debug(f'Unable to connect to TripleO undercloud host: {ex}', exc_info=1) return False + + LOG.debug('TripleO undercloud host found') return True @@ -127,17 +140,30 @@ class UndecloudHostConfig(tobiko.SharedFixture): hostname: typing.Optional[str] = None port: typing.Optional[int] = None username: typing.Optional[str] = None - key_filename: typing.Optional[str] = None def __init__(self, **kwargs): super(UndecloudHostConfig, self).__init__() self._connect_parameters = ssh.gather_ssh_connect_parameters(**kwargs) + self.key_filename: typing.List[str] = [] def setup_fixture(self): self.hostname = CONF.tobiko.tripleo.undercloud_ssh_hostname.strip() self.port = CONF.tobiko.tripleo.undercloud_ssh_port self.username = CONF.tobiko.tripleo.undercloud_ssh_username - self.key_filename = CONF.tobiko.tripleo.undercloud_ssh_key_filename + self.key_filename = self.get_key_filenames() + + @staticmethod + def get_key_filenames() -> typing.List[str]: + key_filenames: typing.List[str] = [] + conf = tobiko.tobiko_config() + key_filename = conf.tripleo.undercloud_ssh_key_filename + if key_filename: + key_filename = tobiko.tobiko_config_path(key_filename) + if os.path.isfile(key_filename): + key_filenames.append(key_filename) + key_filenames.extend(ssh.list_proxy_jump_key_filenames()) + key_filenames.extend(ssh.list_key_filenames()) + return tobiko.select_uniques(key_filenames) @property def connect_parameters(self): diff --git a/tobiko/tripleo/config.py b/tobiko/tripleo/config.py index a62b35581..519154f09 100644 --- a/tobiko/tripleo/config.py +++ b/tobiko/tripleo/config.py @@ -32,7 +32,7 @@ OPTIONS = [ default='stack', help="Username with access to stackrc and overcloudrc files"), cfg.StrOpt('undercloud_ssh_key_filename', - default='~/.ssh/id_rsa', + default=None, help="SSH key filename used to login to Undercloud node"), cfg.ListOpt('undercloud_rcfile', default=['~/stackrc'],