Refactor OS-faults integration

- make it configurable from tobiko.conf
- move it to tobiko.openstack.os_faults package
- sequentially execute os-faults based test cases via
  command line in a separate environment ('tox -e os-faults')
- integrate with tripleo overcloud

Change-Id: Iad3683abe755d263437aefd74b0ba10b4c31b48c

tobiko/cmd/fault.py

@@ -17,7 +17,7 @@ import argparse
 import logging
 import sys
 
-from tobiko.fault import executor
+from tobiko.openstack import os_faults
 
 
 LOG = logging.getLogger(__name__)
@@ -38,16 +38,14 @@ class FaultCMD(object):
 
     def run(self):
         """Run faults."""
-        fault_exec = executor.FaultExecutor()
-        fault_exec.execute(self.args.fault)
+        os_faults.os_faults_execute(self.args.fault)
 
 
 def setup_logging(debug=None):
     """Sets the logging."""
     # pylint: disable=W0622
-    format = '%(message)s'
     level = logging.DEBUG if debug else logging.INFO
-    logging.basicConfig(level=level, format=format)
+    logging.basicConfig(level=level, format='%(message)s')
 
 
 def main():

tobiko/config.py

@@ -32,6 +32,7 @@ CONFIG_MODULES = ['tobiko.openstack.glance.config',
                   'tobiko.openstack.keystone.config',
                   'tobiko.openstack.neutron.config',
                   'tobiko.openstack.nova.config',
+                  'tobiko.openstack.os_faults.config',
                   'tobiko.shell.ssh.config',
                   'tobiko.shell.ping.config',
                   'tobiko.shell.sh.config',

tobiko/fault/config.py

@@ -1,74 +0,0 @@
-# Copyright 2019 Red Hat
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-from __future__ import absolute_import
-
-import os
-
-import jinja2
-
-from oslo_log import log
-
-import tobiko
-from tobiko.fault import constants as fault_const
-from tobiko.openstack import nova
-
-
-LOG = log.getLogger(__name__)
-
-
-class FaultConfig(object):
-    """Responsible for managing faults configuration."""
-
-    DEFAULT_CONF_PATH = os.path.expanduser('~/.config/openstack')
-    DEFAULT_CONF_NAME = "os-faults.yml"
-    DEFAULT_CONF_FILE = os.path.join(DEFAULT_CONF_PATH, DEFAULT_CONF_NAME)
-
-    def __init__(self, conf_file):
-        self.templates_dir = os.path.join(os.path.dirname(__file__),
-                                          'templates')
-        if conf_file:
-            self.conf_file = conf_file
-        else:
-            conf_file = self.DEFAULT_CONF_FILE
-            if os.path.isfile(conf_file):
-                self.conf_file = conf_file
-            else:
-                self.conf_file = self.generate_config_file()
-
-    def generate_config_file(self):
-        """Generates os-faults configuration file."""
-        LOG.info("Generating os-fault configuration file.")
-        tobiko.makedirs(self.DEFAULT_CONF_PATH)
-        rendered_conf = self.get_rendered_configuration()
-        with open(self.DEFAULT_CONF_FILE, "w") as f:
-            f.write(rendered_conf)
-        return self.DEFAULT_CONF_FILE
-
-    def get_rendered_configuration(self):
-        """Returns rendered os-fault configuration file."""
-        j2_env = jinja2.Environment(
-            loader=jinja2.FileSystemLoader(self.templates_dir),
-            trim_blocks=True)
-        template = j2_env.get_template('os-faults.yml.j2')
-        nodes = self.get_nodes()
-        return template.render(nodes=nodes,
-                               services=fault_const.SERVICES,
-                               containers=fault_const.CONTAINERS)
-
-    def get_nodes(self):
-        """Returns a list of dictionaries with nodes name and address."""
-        client = nova.get_nova_client()
-        return [{'name': server.name,
-                 'address': server.addresses['ctlplane'][0]['addr']}
-                for server in client.servers.list()]

tobiko/fault/executor.py

@@ -1,52 +0,0 @@
-# Copyright 2019 Red Hat
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-from __future__ import absolute_import
-
-import sys
-
-from oslo_log import log
-
-import jsonschema
-import os_faults
-
-from tobiko.fault.config import FaultConfig
-
-LOG = log.getLogger(__name__)
-
-
-class FaultExecutor(object):
-    """Responsible for executing faults."""
-
-    def __init__(self, conf_file=None, cloud=None):
-        self.config = FaultConfig(conf_file=conf_file)
-        self.cloud = cloud
-
-    def connect(self):
-        """Connect to the cloud using os-faults."""
-        try:
-            self.cloud = os_faults.connect(
-                config_filename=self.config.conf_file)
-            self.cloud.verify()
-        except os_faults.ansible.executor.AnsibleExecutionUnreachable:
-            LOG.warning("Couldn't verify connectivity to the"
-                        " cloud with os-faults configuration")
-        except jsonschema.exceptions.ValidationError:
-            LOG.error("Wrong os-fault configuration format. Exiting...")
-            sys.exit(2)
-
-    def execute(self, fault):
-        """Executes given fault using os-faults human API."""
-        LOG.info("Using %s" % self.config.conf_file)
-        self.connect()
-        os_faults.human_api(self.cloud, fault)

tobiko/openstack/os_faults/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2019 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+
+from tobiko.openstack.os_faults import _config_file
+from tobiko.openstack.os_faults import _cloud
+from tobiko.openstack.os_faults import _execute
+
+get_os_fault_cloud_managenemt = _cloud.get_os_fault_cloud_managenemt
+OsFaultsCloudManagementFixture = _cloud.OsFaultsCloudManagementFixture
+
+get_os_fault_config_filename = _config_file.get_os_fault_config_filename
+
+os_faults_execute = _execute.os_faults_execute

tobiko/openstack/os_faults/_cloud.py

@@ -0,0 +1,60 @@
+# Copyright 2019 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+from oslo_log import log
+
+import os_faults
+
+
+import tobiko
+from tobiko.openstack.os_faults import _config_file
+
+LOG = log.getLogger(__name__)
+
+
+def get_os_fault_cloud_managenemt(config_filename=None):
+    fixture = OsFaultsCloudManagementFixture(config_filename=config_filename)
+    return tobiko.setup_fixture(fixture).cloud_management
+
+
+class OsFaultsCloudManagementFixture(tobiko.SharedFixture):
+    """Responsible for executing faults."""
+
+    config_filename = None
+    cloud_management = None
+
+    def __init__(self, config_filename=None, cloud_management=None):
+        super(OsFaultsCloudManagementFixture, self).__init__()
+        if config_filename:
+            self.config_filename = config_filename
+        if cloud_management:
+            self.cloud_management = cloud_management
+
+    def setup_fixture(self):
+        self.connect()
+
+    def connect(self):
+        """Connect to the cloud using os-faults."""
+        cloud_management = self.cloud_management
+        if cloud_management is None:
+            config_filename = self.config_filename
+            if config_filename is None:
+                self.config_filename = config_filename = (
+                    _config_file.get_os_fault_config_filename())
+            LOG.info("OS-Faults: connecting with config filename %s",
+                     config_filename)
+            self.cloud_management = cloud_management = os_faults.connect(
+                config_filename=config_filename)
+        return cloud_management

tobiko/openstack/os_faults/_config_file.py

@@ -0,0 +1,180 @@
+# Copyright 2019 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import os
+
+import jinja2
+
+from oslo_log import log
+
+import tobiko
+from tobiko.tripleo import overcloud
+
+
+LOG = log.getLogger(__name__)
+
+
+def get_os_fault_config_filename():
+    return tobiko.setup_fixture(OsFaultsConfigFileFixture).config_filename
+
+
+class OsFaultsConfigFileFixture(tobiko.SharedFixture):
+    """Responsible for managing faults configuration."""
+
+    config = None
+    config_filename = None
+    template_filename = None
+
+    def __init__(self, config=None, config_filename=None,
+                 template_filename=None):
+        super(OsFaultsConfigFileFixture, self).__init__()
+        self.templates_dir = os.path.join(os.path.dirname(__file__),
+                                          'templates')
+        if config is not None:
+            config = config
+        if config_filename is not None:
+            self.config_filename = config_filename
+        if template_filename is not None:
+            self.template_filename = template_filename
+
+    def setup_fixture(self):
+        _config = self.config
+        if not _config:
+            from tobiko import config
+            CONF = config.CONF
+            self.config = _config = CONF.tobiko.os_faults
+        self.config_filename = config_filename = self.get_config_filename()
+        if config_filename is None:
+            self.config_filename = self.generate_config_file(
+                config_filename=config_filename)
+
+    def get_config_filename(self):
+        config_filename = self.config_filename
+        if config_filename is None:
+            config_filename = os.environ.get('OS_FAULTS_CONFIG') or None
+
+        if config_filename is None:
+            config_dirnames = self.config.config_dirnames
+            config_filenames = self.config.config_filenames
+            for dirname in config_dirnames:
+                dirname = os.path.realpath(os.path.expanduser(dirname))
+                for filename in config_filenames:
+                    filename = os.path.join(dirname, filename)
+                    if os.path.isfile(filename):
+                        config_filename = filename
+                        break
+
+        if config_filename is None:
+            LOG.warning("Unable to find any of 'os_faults' files (%s) in "
+                        "any directory (%s",
+                        ', '.join(config_filenames),
+                        ', '.join(config_dirnames))
+        return config_filename
+
+    def get_template_filename(self):
+        template_filename = self.template_filename
+        if template_filename is None:
+            template_filename = os.environ.get('OS_FAULTS_TEMPLATE') or None
+
+        if template_filename is None:
+            template_dirnames = self.config.template_dirnames
+            config_filenames = self.config.config_filenames
+            template_filenames = [filename + '.j2'
+                                  for filename in config_filenames]
+            for dirname in template_dirnames:
+                dirname = os.path.realpath(os.path.expanduser(dirname))
+                for filename in template_filenames:
+                    filename = os.path.join(dirname, filename)
+                    if os.path.isfile(filename):
+                        template_filename = filename
+                        break
+
+        if template_filename is None:
+            LOG.warning("Unable to find any of 'os_faults' template file "
+                        "(%s) in any directory (%s").format(
+                            ', '.join(template_filenames),
+                            ', '.join(template_dirnames))
+        return template_filename
+
+    def generate_config_file(self, config_filename):
+        """Generates os-faults configuration file."""
+
+        self.template_filename = template_filename = (
+            self.get_template_filename())
+        template_basename = os.path.basename(template_filename)
+        if config_filename is None:
+            config_dirname = os.path.realpath(
+                os.path.expanduser(self.config.generate_config_dirname))
+            config_basename, template_ext = os.path.splitext(template_basename)
+            assert template_ext == '.j2'
+            config_filename = os.path.join(config_dirname, config_basename)
+        else:
+            config_dirname = os.path.dirname(config_filename)
+
+        LOG.info("Generating os-fault config file from template %r to %r.",
+                 template_filename, config_filename)
+        tobiko.makedirs(config_dirname)
+
+        template_dirname = os.path.dirname(template_filename)
+        j2_env = jinja2.Environment(
+            loader=jinja2.FileSystemLoader(template_dirname),
+            trim_blocks=True)
+        template = j2_env.get_template(template_basename)
+        config_content = template.render(
+            nodes=self.list_nodes(),
+            services=self.list_services(),
+            containers=self.list_containers(),
+            proxy=None)
+        with open(config_filename, "w") as f:
+            f.write(config_content)
+        return config_filename
+
+    def list_services(self):
+        return self.config.services
+
+    def list_containers(self):
+        return self.config.containers
+
+    def list_nodes(self):
+        """Returns a list of dictionaries with nodes name and address."""
+        nodes = self.config.nodes
+        if nodes:
+            return [parse_config_node(node_string)
+                    for node_string in nodes]
+        elif overcloud.has_overcloud():
+            nodes = []
+            overcloud_nodes = overcloud.list_overcloud_nodes()
+            for overcloud_node in overcloud_nodes:
+                host_config = overcloud.overcloud_host_config(
+                    overcloud_node.name)
+                os_faults_node = dict(
+                    name=overcloud_node.name,
+                    username=host_config.username,
+                    address=host_config.hostname,
+                    private_key_file=host_config.key_filename)
+                nodes.append(os_faults_node)
+            return nodes
+
+        raise NotImplementedError("Cloud node listing not configured.")
+
+
+def parse_config_node(node):
+    fields = node.split('.')
+    if len(fields) != 2:
+        message = ("Invalid cloud node format: {!r} "
+                   "(expected '<name>:<address>')").format(node)
+        raise ValueError(message)
+    return {'name': fields[0],
+            'address': fields[1]}

tobiko/openstack/os_faults/_execute.py

@@ -1,6 +1,4 @@
-# Copyright (c) 2019 Red Hat, Inc.
-#
-# All Rights Reserved.
+# Copyright 2019 Red Hat
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -15,19 +13,20 @@
 #    under the License.
 from __future__ import absolute_import
 
-from tobiko.tests import unit
-from tobiko.fault import executor
+from oslo_log import log
+
+from tobiko.openstack.os_faults import _cloud
 
 
-class FaultTest(unit.TobikoUnitTest):
+LOG = log.getLogger(__name__)
 
-    conf_file = "/some/conf/file"
-    fault = "some_fault"
 
-    def setUp(self):
-        super(FaultTest, self).setUp()
-        self.fault_exec = executor.FaultExecutor(conf_file=self.conf_file)
-
-    def test_init(self):
-        self.assertEqual(self.fault_exec.config.conf_file, self.conf_file)
-        self.assertEqual(self.fault_exec.cloud, None)
+def os_faults_execute(command, cloud_management=None, config_filename=None,
+                      **kwargs):
+    cloud_management = (
+        cloud_management or
+        _cloud.get_os_fault_cloud_managenemt(
+            config_filename=config_filename))
+    if kwargs:
+        command = command.format(**command)
+    return cloud_management.execute(command)

tobiko/openstack/os_faults/config.py

@@ -0,0 +1,81 @@
+# Copyright 2019 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+from __future__ import absolute_import
+
+import os
+import itertools
+
+from oslo_config import cfg
+
+
+OS_FAULTS_SERVICES = ['openvswitch',
+                      'tripleo_cinder_api',
+                      'tripleo_cinder_api_cron',
+                      'tripleo_cinder_scheduler',
+                      'tripleo_clustercheck',
+                      'tripleo_glance_api',
+                      'tripleo_horizon']
+
+OS_FAULTS_CONTAINERS = ['neutron_ovs_agent',
+                        'neutron_metadata_agent',
+                        'neutron_api']
+
+OS_FAULTS_CONFIG_DIRNAMES = ['.',
+                             '~/.config/os-faults',
+                             '/etc/openstack']
+
+OS_FAULTS_CONFIG_FILENAMES = ['os-faults.json',
+                              'os-faults.yaml',
+                              'os-faults.yml']
+
+OS_FAULTS_TEMPLATE_DIRNAMES = ['.',
+                               os.path.join(os.path.dirname(__file__),
+                                            'templates')]
+
+OS_FAULTS_GENERATE_CONFIG_DIRNAME = '~/.tobiko/os-faults'
+
+
+GROUP_NAME = 'os_faults'
+OPTIONS = [
+    cfg.ListOpt('config_dirnames',
+                default=OS_FAULTS_CONFIG_DIRNAMES,
+                help="Directories where to look for os-faults config file"),
+    cfg.ListOpt('config_filenames',
+                default=OS_FAULTS_CONFIG_FILENAMES,
+                help="Base file names used to look for os-faults config file"),
+    cfg.ListOpt('template_dirnames',
+                default=OS_FAULTS_TEMPLATE_DIRNAMES,
+                help=("location where to look for a template file to be used "
+                      "to generate os-faults config file")),
+    cfg.StrOpt('generate_config_dirname',
+               default=OS_FAULTS_GENERATE_CONFIG_DIRNAME,
+               help=("location where to generate config file from template")),
+    cfg.ListOpt('services',
+                default=OS_FAULTS_SERVICES,
+                help="List of services to be handler with os-faults"),
+    cfg.ListOpt('containers',
+                default=OS_FAULTS_CONTAINERS,
+                help="List of containers to be handler with os-faults"),
+    cfg.ListOpt('nodes',
+                default=None,
+                help="List of cloud nodes to be handled with os-faults")
+    ]
+
+
+def register_tobiko_options(conf):
+    conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS)
+
+
+def list_options():
+    return [(GROUP_NAME, itertools.chain(OPTIONS))]

tobiko/openstack/os_faults/templates/os-faults.yaml.j2

@@ -8,8 +8,14 @@ node_discover:
     - fqdn: {{ node['name'] }}
       ip: {{ node['address'] }}
       auth:
-        username: heat-admin
-        private_key_file: /home/stack/.ssh/id_rsa
+        username: {{ node['username'] }}
+        private_key_file: {{ node['private_key_file'] }}
+{% if proxy %}
+        jump:
+            host: {{ proxy['host'] }}
+            username: {{ proxy['username'] }}
+            private_key_file: {{ proxy['private_key_file'] }}
+{% endif %}
 {% endfor %}
 
 services:

tobiko/tests/os_faults/test_cloud.py

@@ -1,4 +1,5 @@
-# Copyright 2019 Red Hat
+# Copyright (c) 2019 Red Hat
+# All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -13,7 +14,13 @@
 #    under the License.
 from __future__ import absolute_import
 
-SERVICES = ['openvswitch', 'tripleo_cinder_api', 'tripleo_cinder_api_cron',
-            'tripleo_cinder_scheduler', 'tripleo_clustercheck',
-            'tripleo_glance_api', 'tripleo_horizon']
-CONTAINERS = ['neutron_ovs_agent', 'neutron_metadata_agent', 'neutron_api']
+
+import testtools
+from tobiko.openstack import os_faults
+
+
+class CloudManagementTest(testtools.TestCase):
+
+    def test_connect(self):
+        cloud_management = os_faults.get_os_fault_cloud_managenemt()
+        cloud_management.verify()

tobiko/tripleo/overcloud.py

@@ -75,9 +75,9 @@ def overcloud_host_config(hostname, ip_version=None, network_name=None):
     return tobiko.setup_fixture(host_config)
 
 
-def overcloud_node_ip_address(ip_version=None, network_name=None,
+def overcloud_node_ip_address(ip_version=None, network_name=None, server=None,
                               **params):
-    server = find_overcloud_node(**params)
+    server = server or find_overcloud_node(**params)
     ip_version = ip_version or CONF.tobiko.tripleo.overcloud_ip_version
     network_name = network_name or CONF.tobiko.tripleo.overcloud_network_name
     return nova.find_server_ip_address(server=server, ip_version=ip_version,
@@ -86,8 +86,10 @@ def overcloud_node_ip_address(ip_version=None, network_name=None,
 
 class OvercloudSshKeyFileFixture(tobiko.SharedFixture):
 
-    key_filename = os.path.expanduser(
-        CONF.tobiko.tripleo.overcloud_ssh_key_filename)
+    @property
+    def key_filename(self):
+        return os.path.expanduser(
+            CONF.tobiko.tripleo.overcloud_ssh_key_filename)
 
     def setup_fixture(self):
         key_filename = self.key_filename

tox.ini

@@ -125,6 +125,17 @@ setenv =
     OS_TEST_PATH={toxinidir}/tobiko/tests/scenario/neutron
 
 
+[testenv:os-faults]
+
+envdir = {toxworkdir}/scenario
+deps = {[testenv:scenario]deps}
+passenv = {[testenv:scenario]passenv}
+setenv =
+    {[testenv:scenario]setenv}
+    OS_TEST_PATH={toxinidir}/tobiko/tests/os_faults
+commands = stestr run --serial {posargs}
+
+
 [testenv:venv]
 
 envdir = {toxworkdir}/scenario