Add SSH connectivity to overcloud nodes.

Change-Id: Ib87d31a47860d1fa9589d5fe3b08010f99e5f338

.tobiko-env/lib/python2.7/site-packages/neutron_lib/tests/etc/dummy_policy.json

@@ -0,0 +1,4 @@
+{
+    "context_is_admin":  "role:dummy",
+    "context_is_advsvc":  "role:dummy"
+}

.tobiko-env/lib/python2.7/site-packages/neutron_lib/tests/etc/neutron_lib.conf

@@ -0,0 +1,8 @@
+[DEFAULT]
+# Show debugging output in logs (sets DEBUG log level output)
+debug = False
+
+lock_path = $state_path/lock
+
+[database]
+connection = 'sqlite://'

.tobiko-env/lib/python2.7/site-packages/neutron_lib/tests/etc/no_policy.json

@@ -0,0 +1,2 @@
+{
+}

.tobiko-env/lib/python2.7/site-packages/neutron_lib/tests/etc/policy.json

@@ -0,0 +1,5 @@
+{
+    "context_is_admin":  "role:admin",
+    "context_is_advsvc":  "role:advsvc",
+    "default": "rule:admin_or_owner"
+}

tobiko/shell/ssh/__init__.py

@@ -26,3 +26,4 @@ ssh_client = _client.ssh_client
 ssh_command = _command.ssh_command
 ssh_proxy_client = _client.ssh_proxy_client
 SSHConnectFailure = _client.SSHConnectFailure
+gather_ssh_connect_parameters = _client.gather_ssh_connect_parameters

tobiko/shell/ssh/_client.py

@@ -15,6 +15,8 @@
 #    under the License.
 from __future__ import absolute_import
 
+import collections
+
 import getpass
 import os
 import socket
@@ -31,12 +33,48 @@ from tobiko.shell.ssh import _command
 LOG = log.getLogger(__name__)
 
 
+def valid_hostname(value):
+    hostname = str(value)
+    if not hostname:
+        message = "Invalid hostname: {!r}".format(hostname)
+        raise ValueError(message)
+    return hostname
+
+
+def valid_port(value):
+    port = int(value)
+    if port <= 0 or port > 65535:
+        message = "Invalid port number: {!r}".format(port)
+        raise ValueError(message)
+    return port
+
+
+def valid_path(value):
+    return os.path.abspath(os.path.expanduser(value))
+
+
+def positive_float(value):
+    value = float(value)
+    if value <= 0.:
+        message = "{!r} is not positive".format(value)
+        raise ValueError(message)
+    return value
+
+
+def positive_int(value):
+    value = int(value)
+    if value <= 0:
+        message = "{!r} is not positive".format(value)
+        raise ValueError(message)
+    return value
+
+
 SSH_CONNECT_PARAMETERS = {
     #: The server to connect to
-    'hostname': str,
+    'hostname': valid_hostname,
 
     #: The server port to connect to
-    'port': int,
+    'port': valid_port,
 
     #: The username to authenticate as (defaults to the current local username)
     'username': str,
@@ -50,10 +88,10 @@ SSH_CONNECT_PARAMETERS = {
 
     #: The filename, or list of filenames, of optional private key(s) and/or
     #: certs to try for authentication
-    'key_filename': str,
+    'key_filename': os.path.expanduser,
 
     #: An optional timeout (in seconds) for the TCP connect
-    'timeout': float,
+    'timeout': positive_float,
 
     #: Set to False to disable connecting to the SSH agent
     'allow_agent': bool,
@@ -83,22 +121,77 @@ SSH_CONNECT_PARAMETERS = {
 
     #: An optional timeout (in seconds) to wait for the SSH banner to be
     #: presented.
-    'banner_timeout': float,
+    'banner_timeout': positive_float,
 
     #: An optional timeout (in seconds) to wait for an authentication response
-    'auth_timeout': float,
+    'auth_timeout': positive_float,
 
     #: Number of connection attempts to be tried before timeout
-    'connection_attempts': int,
+    'connection_attempts': positive_int,
 
     #: Minimum amount of time to wait between two connection attempts
-    'connection_interval': float,
+    'connection_interval': positive_float,
 
     #: Command to be executed to open proxy sock
     'proxy_command': str,
 }
 
 
+def gather_ssh_connect_parameters(source=None, destination=None, schema=None,
+                                  remove_from_schema=False, **kwargs):
+    if schema is None:
+        assert not remove_from_schema
+        schema = SSH_CONNECT_PARAMETERS
+    parameters = {}
+
+    if source:
+        # gather from object
+        if isinstance(source, collections.Mapping):
+            parameters.update(_items_from_mapping(mapping=source,
+                                                  schema=schema))
+        else:
+            parameters.update(_items_from_object(obj=source,
+                                                 schema=schema))
+
+    if kwargs:
+        # gather from kwargs
+        parameters.update(_items_from_mapping(mapping=kwargs, schema=schema))
+        kwargs = exclude_mapping_items(kwargs, schema)
+        if kwargs:
+            message = "Invalid SSH connection parameters: {!r}".format(kwargs)
+            raise ValueError(message)
+
+    if remove_from_schema and parameters:
+        # update schema
+        for name in parameters:
+            del schema[name]
+
+    if destination is not None:
+        destination.update(parameters)
+    return parameters
+
+
+def _items_from_mapping(mapping, schema):
+    for name, init in schema.items():
+        value = mapping.get(name)
+        if value is not None:
+            yield name, init(value)
+
+
+def _items_from_object(obj, schema):
+    for name, init in schema.items():
+        value = getattr(obj, name, None)
+        if value is not None:
+            yield name, init(value)
+
+
+def exclude_mapping_items(mapping, exclude):
+        # Exclude parameters that are already in target dictionary
+    return {key: value
+            for key, value in mapping.items()
+            if key not in exclude}
+
+
 class SSHConnectFailure(tobiko.TobikoException):
     message = "Failed to login to {login}\n{cause}"
 
@@ -118,9 +211,10 @@ class SSHClientFixture(tobiko.SharedFixture):
     proxy_client = None
     proxy_sock = None
     connect_parameters = None
+    schema = SSH_CONNECT_PARAMETERS
 
     def __init__(self, host=None, proxy_client=None, host_config=None,
-                 config_files=None, **connect_parameters):
+                 config_files=None, schema=None, **kwargs):
         super(SSHClientFixture, self).__init__()
         if host:
             self.host = host
@@ -130,14 +224,10 @@ class SSHClientFixture(tobiko.SharedFixture):
             self.host_config = host_config
         if config_files:
             self.config_files = config_files
-        invalid_parameters = sorted([name
+
-                                     for name in connect_parameters
+        self.schema = schema = dict(schema or self.schema)
-                                     if name not in SSH_CONNECT_PARAMETERS])
+        self._connect_parameters = gather_ssh_connect_parameters(
-        if invalid_parameters:
+            schema=schema, **kwargs)
-            message = "Invalid SSH connection parameters: {!s}".format(
-                ', '.join(invalid_parameters))
-            raise ValueError(message)
-        self._connect_parameters = connect_parameters
 
     def setup_fixture(self):
         self.setup_host_config()
@@ -157,71 +247,49 @@ class SSHClientFixture(tobiko.SharedFixture):
         - parameters got from ~/.ssh/config and tobiko.conf
         - parameters got from fixture object attributes
         """
+        self.connect_parameters = self.get_connect_parameters()
 
-        # Get default parameter values from self object
+    def get_connect_parameters(self, schema=None):
-        self.connect_parameters = parameters = items_from_object(
+        schema = dict(schema or self.schema)
-            schema=SSH_CONNECT_PARAMETERS, obj=self)
+        parameters = {}
-        LOG.debug('Default parameters for host %r:\n%r', self.host,
+        for gather_parameters in [self.gather_initial_connect_parameters,
-                  parameters)
+                                  self.gather_host_config_connect_parameters,
+                                  self.gather_default_connect_parameters]:
+            gather_parameters(destination=parameters,
+                              schema=schema,
+                              remove_from_schema=True)
+        if parameters:
+            LOG.debug('SSH connect parameters for host %r:\n%r', self.host,
+                      parameters)
+        return parameters
 
-        # Override parameters from host configuration files
+    def gather_initial_connect_parameters(self, **kwargs):
-        parameters.update(
+        parameters = gather_ssh_connect_parameters(
-            items_from_mapping(schema=SSH_CONNECT_PARAMETERS,
+            source=self._connect_parameters, **kwargs)
-                               mapping=self.host_config.connect_parameters))
+        if parameters:
-        LOG.debug('Configured connect parameters for host %r:\n%r',
+            LOG.debug('Initial SSH connect parameters for host %r:\n'
-                  self.host, parameters)
+                      '%r', self.host, parameters)
+        return parameters
 
-        # Override parameters with __init__ parameters
+    def gather_host_config_connect_parameters(self, **kwargs):
-        parameters.update(
+        parameters = gather_ssh_connect_parameters(
-            items_from_mapping(schema=SSH_CONNECT_PARAMETERS,
+            source=self.host_config.connect_parameters, **kwargs)
-                               mapping=self._connect_parameters))
+        if parameters:
-        LOG.debug('Resulting connect parameters for host %r:\n%r', self.host,
+            LOG.debug('Host configured SSH connect parameters for host %r:\n'
-                  parameters)
+                      '%r', self.host, parameters)
+        return parameters
 
-        # Validate hostname
+    def gather_default_connect_parameters(self, **kwargs):
-        hostname = parameters.get('hostname')
+        parameters = gather_ssh_connect_parameters(source=self, **kwargs)
-        if not hostname:
+        if parameters:
-            message = "Invalid hostname: {!r}".format(hostname)
+            LOG.debug('Default SSH connect parameters for host %r:\n'
-            raise ValueError(message)
+                      '%r', self.host, parameters)
-
+        return parameters
-        # Expand key_filename
-        key_filename = parameters.get('key_filename')
-        if key_filename:
-            key_filename = os.path.expanduser(key_filename)
-            if not os.path.exists(key_filename):
-                message = "key_filename {!r} doesn't exist".format(hostname)
-                raise ValueError(message)
-            parameters['key_filename'] = key_filename
-
-        # Validate connection timeout
-        timeout = parameters.get('timeout')
-        if not timeout or timeout < 0.:
-            message = "Invalid timeout: {!r}".format(timeout)
-            raise ValueError(message)
-
-        # Validate connection attempts
-        connection_attempts = parameters.get('connection_attempts')
-        if not connection_attempts or connection_attempts < 0:
-            message = "Invalid connection attempts: {!r}".format(
-                connection_attempts)
-            raise ValueError(message)
-
-        # Validate connection attempts
-        connection_interval = parameters.get('connection_interval')
-        if not connection_interval or connection_interval < 0.:
-            message = "Invalid connection interval: {!r}".format(
-                connection_interval)
-            raise ValueError(message)
-
-        # Validate connection port
-        port = parameters.get('port')
-        if not port or port < 1 or port > 65535:
-            message = "Invalid port: {!r}".format(port)
-            raise ValueError(message)
 
     def setup_ssh_client(self):
         self.client, self.proxy_sock = ssh_connect(
-            proxy_client=self.proxy_client, **self.connect_parameters)
+            proxy_client=self.proxy_client,
+            **self.connect_parameters)
         self.addCleanup(self.client.close)
         if self.proxy_sock:
             self.addCleanup(self.proxy_sock.close)
@@ -230,18 +298,6 @@ class SSHClientFixture(tobiko.SharedFixture):
         return tobiko.setup_fixture(self).client
 
 
-def items_from_mapping(schema, mapping):
-    return ((key, init(mapping.get(key)))
-            for key, init in schema.items()
-            if mapping.get(key) is not None)
-
-
-def items_from_object(schema, obj):
-    return {key: init(getattr(obj, key, None))
-            for key, init in schema.items()
-            if getattr(obj, key, None) is not None}
-
-
 UNDEFINED_CLIENT = 'UNDEFINED_CLIENT'
 
 
@@ -269,7 +325,8 @@ class SSHClientManager(object):
                                                  config_files=config_files)
             self.clients[host_key] = client = SSHClientFixture(
                 host=host, hostname=hostname, port=port, username=username,
-                proxy_client=proxy_client, **connect_parameters)
+                proxy_client=proxy_client, host_config=host_config,
+                **connect_parameters)
         return client
 
     def get_proxy_client(self, host=None, proxy_jump=None, host_config=None,

tobiko/tests/functional/tripleo/test_overcloud.py

@@ -13,6 +13,8 @@
 #    under the License.
 from __future__ import absolute_import
 
+import os
+
 import netaddr
 import testtools
 
@@ -61,13 +63,22 @@ class OvercloudNovaApiTest(testtools.TestCase):
         self.assertIsInstance(overcloud_node_ip, netaddr.IPAddress)
 
     def test_overcloud_host_config(self):
+        hostname = overcloud.find_overcloud_node().name
         host_config = tobiko.setup_fixture(
-            overcloud.overcloud_host_config(hostname='controller-0'))
+            overcloud.overcloud_host_config(hostname=hostname))
-        self.assertEqual('controller-0', host_config.host)
+        self.assertEqual(hostname, host_config.host)
         self.assertIsInstance(host_config.hostname, netaddr.IPAddress)
         self.assertEqual(CONF.tobiko.tripleo.overcloud_ssh_port,
                          host_config.port)
         self.assertEqual(CONF.tobiko.tripleo.overcloud_ssh_username,
                          host_config.username)
-        self.assertEqual(CONF.tobiko.tripleo.ssh_key_filename,
+        key_filename = os.path.expanduser(
-                         host_config.key_filename)
+            CONF.tobiko.tripleo.overcloud_ssh_key_filename)
+        self.assertEqual(key_filename, host_config.key_filename)
+        self.assertTrue(os.path.isfile(key_filename))
+        self.assertTrue(os.path.isfile(key_filename + '.pub'))
+
+    def test_overcloud_ssh_client_connection(self):
+        hostname = overcloud.find_overcloud_node().name
+        ssh_client = overcloud.overcloud_ssh_client(hostname=hostname)
+        ssh_client.connect()

tobiko/tests/unit/tripleo/test_config.py

@@ -26,7 +26,7 @@ TIPLEO_CONF = CONF.tobiko.tripleo
 class TripleoConfigTest(unit.TobikoUnitTest):
 
     def test_ssh_key_filename(self):
-        self.assertIsInstance(TIPLEO_CONF.ssh_key_filename,
+        self.assertIsInstance(TIPLEO_CONF.undercloud_ssh_key_filename,
                               six.string_types)
 
 

tobiko/tripleo/config.py

@@ -19,11 +19,6 @@ from oslo_config import cfg
 
 GROUP_NAME = 'tripleo'
 OPTIONS = [
-    # TripleO options
-    cfg.StrOpt('ssh_key_filename',
-               default='~/.ssh/id_rsa',
-               help="SSH key filename used to login to TripleO nodes"),
-
     # Undercloud options
     cfg.StrOpt('undercloud_ssh_hostname',
                default=None,
@@ -35,6 +30,9 @@ OPTIONS = [
     cfg.StrOpt('undercloud_ssh_username',
                default='stack',
                help="Username with access to stackrc and overcloudrc files"),
+    cfg.StrOpt('undercloud_ssh_key_filename',
+               default='~/.ssh/id_rsa',
+               help="SSH key filename used to login to Undercloud node"),
     cfg.StrOpt('undercloud_rcfile',
                default='~/stackrc',
                help="Undercloud RC filename"),
@@ -46,6 +44,9 @@ OPTIONS = [
     cfg.StrOpt('overcloud_ssh_username',
                default='heat-admin',
                help="Default username used to connect to overcloud nodes"),
+    cfg.StrOpt('overcloud_ssh_key_filename',
+               default='~/.ssh/id_overcloud',
+               help="SSH key filename used to login to Overcloud nodes"),
     cfg.StrOpt('overcloud_rcfile',
                default='~/overcloudrc',
                help="Overcloud RC filename"),

tobiko/tripleo/overcloud.py

@@ -13,12 +13,17 @@
 #    under the License.
 from __future__ import absolute_import
 
+import io
+import os
+
 import six
 
 import tobiko
 from tobiko import config
 from tobiko.openstack import keystone
 from tobiko.openstack import nova
+from tobiko.shell import sh
+from tobiko.shell import ssh
 from tobiko.tripleo import undercloud
 
 
@@ -56,6 +61,13 @@ def find_overcloud_node(**params):
     return nova.find_server(client=client, **params)
 
 
+def overcloud_ssh_client(hostname, ip_version=None, network_name=None):
+    host_config = overcloud_host_config(hostname=hostname,
+                                        ip_version=ip_version,
+                                        network_name=network_name)
+    return ssh.ssh_client(host=hostname, host_config=host_config)
+
+
 def overcloud_host_config(hostname, ip_version=None, network_name=None):
     host_config = OvercloudHostConfig(host=hostname,
                                       ip_version=ip_version,
@@ -72,15 +84,52 @@ def overcloud_node_ip_address(ip_version=None, network_name=None,
                                        network_name=network_name)
 
 
+class OvercloudSshKeyFileFixture(tobiko.SharedFixture):
+
+    key_filename = os.path.expanduser(
+        CONF.tobiko.tripleo.overcloud_ssh_key_filename)
+
+    def setup_fixture(self):
+        key_filename = self.key_filename
+        if not os.path.isfile(key_filename):
+            self.setup_key_file()
+            assert os.path.isfile(key_filename)
+
+    def setup_key_file(self):
+        key_filename = self.key_filename
+        key_dirname = os.path.dirname(key_filename)
+        tobiko.makedirs(key_dirname, mode=0o700)
+
+        ssh_client = undercloud.undercloud_ssh_client()
+        _get_undercloud_file(ssh_client=ssh_client,
+                             source='~/.ssh/id_rsa',
+                             destination=key_filename,
+                             mode=0o600)
+        _get_undercloud_file(ssh_client=ssh_client,
+                             source='~/.ssh/id_rsa.pub',
+                             destination=key_filename + '.pub',
+                             mode=0o600)
+
+
+def _get_undercloud_file(ssh_client, source, destination, mode):
+    content = sh.execute(['cat', source],
+                         ssh_client=ssh_client).stdout
+    with io.open(destination, 'wb') as fd:
+        fd.write(content.encode())
+    os.chmod(destination, mode)
+
+
 class OvercloudHostConfig(tobiko.SharedFixture):
     hostname = None
     port = None
     username = None
-    key_filename = None
+    key_file = tobiko.required_setup_fixture(OvercloudSshKeyFileFixture)
     ip_version = None
     network_name = None
+    key_filename = None
 
-    def __init__(self, host, ip_version=None, network_name=None):
+    def __init__(self, host, ip_version=None, network_name=None,
+                 **kwargs):
         super(OvercloudHostConfig, self).__init__()
         tobiko.check_valid_type(host, six.string_types)
         self.host = host
@@ -88,11 +137,19 @@ class OvercloudHostConfig(tobiko.SharedFixture):
             self.ip_version = ip_version
         if network_name:
             self.network_name = network_name
+        self._connect_parameters = ssh.gather_ssh_connect_parameters(**kwargs)
 
     def setup_fixture(self):
-        self.hostname = overcloud_node_ip_address(
+        self.hostname = self.hostname or overcloud_node_ip_address(
             name=self.host, ip_version=self.ip_version,
             network_name=self.network_name)
-        self.port = CONF.tobiko.tripleo.overcloud_ssh_port
+        self.port = self.port or CONF.tobiko.tripleo.overcloud_ssh_port
-        self.username = CONF.tobiko.tripleo.overcloud_ssh_username
+        self.username = (self.username or
-        self.key_filename = CONF.tobiko.tripleo.ssh_key_filename
+                         CONF.tobiko.tripleo.overcloud_ssh_username)
+        self.key_filename = self.key_filename or self.key_file.key_filename
+
+    @property
+    def connect_parameters(self):
+        parameters = ssh.gather_ssh_connect_parameters(self)
+        parameters.update(self._connect_parameters)
+        return parameters

tobiko/tripleo/undercloud.py

@@ -68,11 +68,21 @@ class UndecloudHostConfig(tobiko.SharedFixture):
     username = None
     key_filename = None
 
+    def __init__(self, **kwargs):
+        super(UndecloudHostConfig, self).__init__()
+        self._connect_parameters = ssh.gather_ssh_connect_parameters(**kwargs)
+
     def setup_fixture(self):
         self.hostname = CONF.tobiko.tripleo.undercloud_ssh_hostname
         self.port = CONF.tobiko.tripleo.undercloud_ssh_port
         self.username = CONF.tobiko.tripleo.undercloud_ssh_username
-        self.key_filename = CONF.tobiko.tripleo.ssh_key_filename
+        self.key_filename = CONF.tobiko.tripleo.undercloud_ssh_key_filename
+
+    @property
+    def connect_parameters(self):
+        parameters = ssh.gather_ssh_connect_parameters(self)
+        parameters.update(self._connect_parameters)
+        return parameters
 
 
 def undercloud_keystone_client():