Fix os-faults config file generation

Change-Id: I32b5b0aa7e6fbd54190fadec89bb0c072df9c258

tobiko/openstack/os_faults/__init__.py

@@ -13,7 +13,6 @@
 #    under the License.
 from __future__ import absolute_import
 
-
 from tobiko.openstack.os_faults import _config_file
 from tobiko.openstack.os_faults import _cloud
 from tobiko.openstack.os_faults import _execute

tobiko/openstack/os_faults/_config_file.py

@@ -14,12 +14,15 @@
 from __future__ import absolute_import
 
 import os
+import typing  # noqa
 
 import jinja2
+import six
 
 from oslo_log import log
 
 import tobiko
+from tobiko.openstack.os_faults import _exception
 from tobiko.openstack import topology
 from tobiko.shell import ssh
 
@@ -37,9 +40,10 @@ class OsFaultsConfigFileFixture(tobiko.SharedFixture):
     config = None
     config_filename = None
     template_filename = None
+    topo = None
 
     def __init__(self, config=None, config_filename=None,
-                 template_filename=None):
+                 template_filename=None, topo=None):
         super(OsFaultsConfigFileFixture, self).__init__()
         self.templates_dir = os.path.join(os.path.dirname(__file__),
                                           'templates')
@@ -49,6 +53,8 @@ class OsFaultsConfigFileFixture(tobiko.SharedFixture):
             self.config_filename = config_filename
         if template_filename is not None:
             self.template_filename = template_filename
+        if topo:
+            self.topo = topo
 
     def setup_fixture(self):
         _config = self.config
@@ -128,61 +134,104 @@ class OsFaultsConfigFileFixture(tobiko.SharedFixture):
                  template_filename, config_filename)
         tobiko.makedirs(config_dirname)
 
-        template_dirname = os.path.dirname(template_filename)
-        j2_env = jinja2.Environment(
-            loader=jinja2.FileSystemLoader(template_dirname),
-            trim_blocks=True)
-        template = j2_env.get_template(template_basename)
-        config_content = template.render(
-            nodes=self.list_nodes(),
-            services=self.list_services(),
-            containers=self.list_containers())
-        with tobiko.open_output_file(config_filename) as f:
-            f.write(config_content)
+        make_os_faults_config_file(config_filename=config_filename,
+                                   template_filename=template_filename,
+                                   topo=self.topo)
         return config_filename
 
-    def list_services(self):
-        return self.config.services
 
-    def list_containers(self):
-        return self.config.containers
+def make_os_faults_config_file(config_filename, template_filename, topo=None):
+    # type: (str, str, topology.OpenStackTopology) -> int
+    template = get_os_faults_config_template(template_filename)
+    config_content = get_os_faults_config_content(template=template, topo=topo)
+    with tobiko.open_output_file(config_filename) as stream:
+        LOG.debug('Write os-foults config file to %r:\n%s', config_filename,
+                  config_content)
+        return stream.write(config_content)
 
-    def list_nodes(self):
-        """Returns a list of dictionaries with nodes name and address."""
-        return [self._node_from_topology(node)
-                for node in topology.list_openstack_nodes()]
 
-    def _node_from_topology(self, node):
-        auth = self._node_auth_from_topology(node)
-        return dict(fqdn=node.name,
-                    ip=str(node.public_ip),
-                    auth=auth)
+def get_os_faults_config_template(filename):
+    # type: (str) -> jinja2.Template
+    basename = os.path.basename(filename)
+    dirname = os.path.dirname(filename)
+    loader = jinja2.FileSystemLoader(dirname)
+    environment = jinja2.Environment(loader=loader, trim_blocks=True)
+    return environment.get_template(basename)
 
-    def _node_auth_from_topology(self, node):
-        jump = self._node_auth_jump_from_topology(node)
-        ssh_parameters = node.ssh_parameters
-        return dict(username=ssh_parameters['username'],
-                    private_key_file=os.path.expanduser(
-                        ssh_parameters['key_filename']),
-                    jump=jump)
 
-    def _node_auth_jump_from_topology(self, node):
-        host_config = ssh.ssh_host_config(str(node.public_ip))
-        if host_config.proxy_jump:
-            proxy_config = ssh.ssh_host_config(host_config.proxy_jump)
-            return dict(username=proxy_config.username,
-                        host=proxy_config.hostname,
-                        private_key_file=os.path.expanduser(
-                            proxy_config.key_filename))
+def get_os_faults_config_content(template, topo=None):
+    # type: (jinja2.Template, topology.OpenStackTopology) -> typing.Text
+    topo = topo or topology.get_openstack_topology()
+    nodes = [get_os_faults_node_from_topology(node) for node in topo.nodes]
+    # TODO: get services and containers from OpenStack topology
+    services = []  # type: typing.List[str]
+    containers = []  # type: typing.List[str]
+    return template.render(nodes=nodes,
+                           services=services,
+                           containers=containers)
+
+
+def get_os_faults_node_from_topology(node):
+    # type: (topology.OpenStackTopologyNode) -> typing.Dict
+    return {'fqdn': node.name,
+            'ip': str(node.public_ip),
+            'auth': get_os_faults_node_auth_from_topology(node)}
+
+
+def get_os_faults_node_auth_from_topology(node):
+    # type: (topology.OpenStackTopologyNode) -> typing.Dict
+    private_key_file = get_os_faults_private_key_file(
+        key_filename=node.ssh_parameters['key_filename'])
+    port = int(node.ssh_parameters.get('port') or 22)
+    if port != 22:
+        raise ValueError('Invalid port value: ' + repr(port))
+    auth = {'username': node.ssh_parameters['username'],
+            'private_key_file': private_key_file}
+    jump = get_os_faults_node_auth_jump_from_topology(node)
+    if jump:
+        auth['jump'] = jump
+    return auth
+
+
+def get_os_faults_node_auth_jump_from_topology(node):
+    # type: (topology.OpenStackTopologyNode) -> typing.Optional[typing.Dict]
+    host_config = ssh.ssh_host_config(str(node.public_ip))
+    if host_config.proxy_jump:
+        config = ssh.ssh_host_config(host_config.proxy_jump)
+        port = int(config.port or 22)
+        if port != 22:
+            raise ValueError('Invalid port value: ' + repr(port))
+        private_key_file = get_os_faults_private_key_file(
+            key_filename=config.key_filename)
+        return {'host': config.hostname,
+                'username': config.username,
+                'private_key_file': private_key_file}
+    else:
+        return None
+
+
+def get_os_faults_private_key_file(key_filename):
+    # type: (typing.Union[str, typing.Sequence]) -> str
+
+    if isinstance(key_filename, six.string_types):
+        key_filename = [key_filename]
+    else:
+        key_filename = list(key_filename)
+    for filename in key_filename:
+        filename = os.path.expanduser(filename)
+        if os.path.exists(filename):
+            return os.path.expanduser(filename)
         else:
-            return None
+            LOG.warning('Private key file not found: %r', filename)
+    raise _exception.NoSuchPrivateKeyFilename(
+        key_filename=', '.join(key_filename))
 
 
 def parse_config_node(node):
+    # type: (str) -> typing.Dict
     fields = node.split('.')
     if len(fields) != 2:
         message = ("Invalid cloud node format: {!r} "
                    "(expected '<name>:<address>')").format(node)
         raise ValueError(message)
-    return {'name': fields[0],
-            'address': fields[1]}
+    return {'name': fields[0], 'address': fields[1]}

tobiko/openstack/os_faults/_exception.py

@@ -0,0 +1,20 @@
+# Copyright 2019 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import tobiko
+
+
+class NoSuchPrivateKeyFilename(tobiko.TobikoException):
+    message = "No such private key filename(s): {key_filename}'"

tobiko/openstack/os_faults/templates/os-faults.yaml.j2

@@ -18,6 +18,7 @@ node_discover:
 {% endif %}
 {% endfor %}
 
+{% if services %}
 services:
 {% for service in services %}
   {{ service }}:
@@ -26,7 +27,9 @@ services:
       service_name: {{ service }}
       grep: {{ service }}
 {% endfor %}
+{% endif %}
 
+{% if containers %}
 containers:
 {% for container in containers %}
   {{ container }}:
@@ -34,3 +37,4 @@ containers:
     args:
       container_name: {{ container }}
 {% endfor %}
+{% endif %}

tobiko/openstack/topology/__init__.py

@@ -27,5 +27,6 @@ get_default_openstack_topology_class = (
     _topology.get_default_openstack_topology_class)
 list_openstack_nodes = _topology.list_openstack_nodes
 OpenStackTopology = _topology.OpenStackTopology
+OpenStackTopologyNode = _topology.OpenStackTopologyNode
 set_default_openstack_topology_class = (
     _topology.set_default_openstack_topology_class)

tobiko/openstack/topology/_topology.py

@@ -14,8 +14,10 @@
 from __future__ import absolute_import
 
 import collections
-import weakref
 import socket
+import typing  # noqa
+import weakref
+
 
 import netaddr
 from oslo_log import log
@@ -41,6 +43,7 @@ DEFAULT_TOPOLOGY_CLASS = (
 
 
 def get_openstack_topology(topology_class=None):
+    # type: (typing.Any) -> OpenStackTopology
     topology_class = topology_class or get_default_openstack_topology_class()
     return tobiko.setup_fixture(topology_class)
 
@@ -81,6 +84,7 @@ def list_openstack_node_groups(topology=None):
 
 
 def get_default_openstack_topology_class():
+    # type: () -> typing.Any
     return DEFAULT_TOPOLOGY_CLASS
 
 

tobiko/tests/functional/openstack/test_os_faults.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2019 Red Hat
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+
+import testtools
+from tobiko.openstack import os_faults
+
+
+class OsFaultsTest(testtools.TestCase):
+
+    def test_nodes_connection(self):
+        cloud_management = os_faults.get_os_fault_cloud_managenemt()
+        cloud_management.verify()

tox.ini

@@ -70,7 +70,7 @@ envdir = {toxworkdir}/pep8
 deps = {[testenv:pep8]deps}
 commands =
     pylint -j0 --max-line-length=80 -E -e W,E \
-        -d unused-import,broad-except tobiko
+        -d unused-import,broad-except,fixme tobiko
 
 
 [flake8]