Allow run tests for both tempest plugin and regression tests

Regression tests will be maintained in-tree. See categries list:
https://etherpad.openstack.org/p/heat-integration-test-categories

We will seperate tests to tempest plugin or in-tree base on conditions
of each tests. So we need to rework on framework for integration tests.
This patch propose following changes:
* This partially reverts commit fff6518e69
* Allow running heat tempest plugin and heat regression tests.
* iniset configs to both heat_integrationtests.conf and tempest.conf

Change-Id: Ief31dc961bc108e2863119598dfb16581a38e9cf
Depends-On: I5e9325766ce166e62c731330c462c030cb1e11fc
Co-Authored-By: Zane Bitter <zbitter@redhat.com>

.gitignore

@@ -25,6 +25,7 @@ etc/heat/heat.conf.sample
 .idea
 # integration tests requirements are auto-generated from stub file
 heat_integrationtests/requirements.txt
+heat_integrationtests/heat_integrationtests.conf.sample
 
 # generated policy file
 etc/heat/policy.yaml.sample

heat_integrationtests/README.rst

@@ -2,28 +2,24 @@
 Heat integration tests
 ======================
 
-These tests can be run as a tempest plugin against any heat-enabled OpenStack
-cloud, however defaults match running against a recent DevStack.
+These tests can be run against any heat-enabled OpenStack cloud, however
+defaults match running against a recent DevStack.
 
-To run the tests against DevStack, do the following:
-
-    # Define DEST
+To run the tests against DevStack, do the following::
 
     export DEST=/opt/stack
 
-    # create test resources and write tempest config
-
+    # create test resources and write config
     $DEST/heat/heat_integrationtests/prepare_test_env.sh
-
     $DEST/heat/heat_integrationtests/prepare_test_network.sh
 
-    # run tempest selecting only these tests
-
-    cd $DEST/tempest
-
-    tempest run --regex heat_integrationtests
-
-If custom configuration is required, edit the [heat_plugin] section of
+    # run the heat integration tests
+    cd $DEST/heat
+    stestr --test-path=heat_integrationtests
 
-    $DEST/tempest/etc/tempest.conf
+If the Heat Tempest Plugin is also installed, the tests from that will be run
+as well.
 
+If custom configuration is required, add it in the file
+``heat_integrationtests/heat_integrationtests.conf``. A sample configuration is
+available in ``heat_integrationtests/heat_integrationtests.conf.sample``

heat_integrationtests/__init__.py

@@ -0,0 +1,61 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import unittest
+
+from heat_integrationtests.common import config
+
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__, project=__name__)
+
+
+def load_tests(loader, standard_tests, pattern):
+    logging.setup(config.init_conf(), __name__)
+
+    suite = unittest.TestSuite()
+
+    heat_integration_dir = os.path.dirname(os.path.abspath(__file__))
+    top_level_dir = os.path.split(heat_integration_dir)[0]
+    if pattern:
+        discovered = loader.discover(heat_integration_dir, pattern=pattern,
+                                     top_level_dir=top_level_dir)
+    else:
+        discovered = loader.discover(heat_integration_dir,
+                                     top_level_dir=top_level_dir)
+    suite.addTests(discovered)
+
+    # Discover tests from the heat-tempest-plugin if it is present, using
+    # the Tempest plugin mechanism so we don't need a hard dependency on it.
+    from tempest.test_discover import plugins as tempest_plugins
+
+    ext_plugins = tempest_plugins.TempestTestPluginManager()
+    plugin_data = ext_plugins.get_plugin_load_tests_tuple()
+    heat_plugin_data = plugin_data.get('heat')
+    if heat_plugin_data is not None:
+        plugin_dir, plugin_path = heat_plugin_data
+        LOG.info('Found Heat Tempest plugin: %s, %s', plugin_dir, plugin_path)
+        if pattern:
+            discovered = loader.discover(plugin_dir, pattern=pattern,
+                                         top_level_dir=plugin_path)
+        else:
+            discovered = loader.discover(plugin_dir,
+                                         top_level_dir=plugin_path)
+        suite.addTests(discovered)
+    else:
+        LOG.error('Heat Tempest plugin not found')
+        LOG.info('Available Tempest plugins: %s',
+                 ', '.join(plugin_data.keys()))
+
+    return suite

heat_integrationtests/api/test_heat_api.py

@@ -29,7 +29,7 @@ def load_tests(loader, tests, pattern):
     """Provide a TestSuite to the discovery process."""
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
 
-    conf = config.CONF.heat_plugin
+    conf = config.init_conf().heat_plugin
     if conf.auth_url is None:
         # It's not configured, let's not load tests
         return

heat_integrationtests/common/config.py

@@ -10,9 +10,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
+
 from oslo_config import cfg
+from oslo_log import log as logging
+
+import heat_integrationtests
 
-CONF = None
+_CONF = None
 
 heat_group = cfg.OptGroup(name="heat_plugin",
                           title="Heat Service Options")
@@ -158,5 +163,28 @@ HeatGroup = [
 ]
 
 
+def init_conf(read_conf=True):
+    global _CONF
+    if _CONF is not None:
+        return _CONF
+
+    default_config_files = None
+    if read_conf:
+        confpath = os.path.join(
+            os.path.dirname(os.path.realpath(heat_integrationtests.__file__)),
+            'heat_integrationtests.conf')
+        if os.path.isfile(confpath):
+            default_config_files = [confpath]
+
+    _CONF = cfg.ConfigOpts()
+    logging.register_options(_CONF)
+    _CONF(args=[], project='heat_integrationtests',
+          default_config_files=default_config_files)
+
+    for group, opts in list_opts():
+        _CONF.register_opts(opts, group=group)
+    return _CONF
+
+
 def list_opts():
     yield heat_group.name, HeatGroup

heat_integrationtests/common/test.py

@@ -73,7 +73,8 @@ def requires_convergence(test_method):
 
     The decorated test will be skipped when convergence is disabled.
     '''
-    convergence_enabled = config.CONF.heat_plugin.convergence_engine_enabled
+    convergence_enabled = config.init_conf(
+    ).heat_plugin.convergence_engine_enabled
     skipper = testtools.skipUnless(convergence_enabled,
                                    "Convergence-only tests are disabled")
     return skipper(test_method)
@@ -85,7 +86,7 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
     def setUp(self):
         super(HeatIntegrationTest, self).setUp()
 
-        self.conf = config.CONF.heat_plugin
+        self.conf = config.init_conf().heat_plugin
 
         self.assertIsNotNone(self.conf.auth_url,
                              'No auth_url configured')

heat_integrationtests/config-generator.conf

@@ -0,0 +1,4 @@
+[DEFAULT]
+output_file = heat_integrationtests/heat_integrationtests.conf.sample
+wrap_width = 79
+namespace = heat_integrationtests.common.config

heat_integrationtests/post_test_hook.sh

@@ -21,7 +21,6 @@ sudo -E $DEST/heat/heat_integrationtests/prepare_test_env.sh
 sudo -E $DEST/heat/heat_integrationtests/prepare_test_network.sh
 
 cd $DEST/tempest
-sudo sed -i -e '/group_regex/c\group_regex=heat_tempest_plugin\\.tests\\.api\\.test_heat_api(?:\\.|_)([^_]+)' .testr.conf
-sudo tox -evenv-tempest -- tempest run --regex heat_tempest_plugin.tests
+sudo tox -evenv-tempest -- stestr --test-path=$DEST/heat/heat_integrationtests --top-dir=$DEST/heat --group_regex='heat_(?:tempest_plugin\.tests|integrationtests)\.api\.test_heat_api[._]([^_]+)' run
 
 sudo -E $DEST/heat/heat_integrationtests/cleanup_test_env.sh

heat_integrationtests/prepare_test_env.sh

@@ -13,7 +13,8 @@
 # under the License.
 
 # This script creates required cloud resources and sets test options
-# in tempest.conf.
+# in heat_integrationtests.conf and in tempest.conf.
+# Credentials are required for creating nova flavors and glance images.
 
 set -e
 
@@ -23,51 +24,66 @@ source $DEST/devstack/inc/ini-config
 
 set -x
 
-conf_file=$DEST/tempest/etc/tempest.conf
+function _config_iniset {
+    local conf_file=$1
+
+    source $DEST/devstack/openrc demo demo
+    # user creds
+    iniset $conf_file heat_plugin username $OS_USERNAME
+    iniset $conf_file heat_plugin password $OS_PASSWORD
+    iniset $conf_file heat_plugin project_name $OS_PROJECT_NAME
+    iniset $conf_file heat_plugin auth_url $OS_AUTH_URL
+    iniset $conf_file heat_plugin user_domain_id $OS_USER_DOMAIN_ID
+    iniset $conf_file heat_plugin project_domain_id $OS_PROJECT_DOMAIN_ID
+    iniset $conf_file heat_plugin user_domain_name $OS_USER_DOMAIN_NAME
+    iniset $conf_file heat_plugin project_domain_name $OS_PROJECT_DOMAIN_NAME
+    iniset $conf_file heat_plugin region $OS_REGION_NAME
+    iniset $conf_file heat_plugin auth_version $OS_IDENTITY_API_VERSION
+
+    source $DEST/devstack/openrc admin admin
+    iniset $conf_file heat_plugin admin_username $OS_USERNAME
+    iniset $conf_file heat_plugin admin_password $OS_PASSWORD
+
+    # Register the flavors for booting test servers
+    iniset $conf_file heat_plugin instance_type m1.heat_int
+    iniset $conf_file heat_plugin minimal_instance_type m1.heat_micro
+
+    iniset $conf_file heat_plugin image_ref Fedora-Cloud-Base-26-1.5.x86_64
+    iniset $conf_file heat_plugin boot_config_env $DEST/heat-templates/hot/software-config/boot-config/test_image_env.yaml
+    iniset $conf_file heat_plugin heat_config_notify_script $DEST/heat-templates/hot/software-config/elements/heat-config/bin/heat-config-notify
+    iniset $conf_file heat_plugin minimal_image_ref cirros-0.3.5-x86_64-disk
+
+    # Skip ReloadOnSighupTest. Most jobs now run with apache+uwsgi, so the test has no significance
+    # Skip NotificationTest till bug #1721202 is fixed
+    # Skip StackCancelTest till the python-heatclient is bumped
+    iniset $conf_file heat_plugin skip_functional_test_list 'ReloadOnSighupTest, NotificationTest, StackCancelTest'
+
+    # Skip VolumeBackupRestoreIntegrationTest skipped until failure rate can be reduced ref bug #1382300
+    # Skip test_server_signal_userdata_format_software_config is skipped untill bug #1651768 is resolved
+    iniset $conf_file heat_plugin skip_scenario_test_list 'SoftwareConfigIntegrationTest, VolumeBackupRestoreIntegrationTest'
+
+    if [ "$DISABLE_CONVERGENCE" == "true" ]; then
+        iniset $conf_file heat_plugin convergence_engine_enabled false
+    fi
+    cat $conf_file
+}
+
+
+function _config_functionaltests
+{
+    local conf_file=$DEST/heat/heat_integrationtests/heat_integrationtests.conf
+    _config_iniset $conf_file
+}
+
+function _config_tempest_plugin
+{
+    local conf_file=$DEST/tempest/etc/tempest.conf
+    iniset_multiline $conf_file service_available heat_plugin True
+    _config_iniset $conf_file
+}
+
+_config_functionaltests
+_config_tempest_plugin
 
-iniset_multiline $conf_file service_available heat_plugin True
-
-source $DEST/devstack/openrc demo demo
-# user creds
-iniset $conf_file heat_plugin username $OS_USERNAME
-iniset $conf_file heat_plugin password $OS_PASSWORD
-iniset $conf_file heat_plugin project_name $OS_PROJECT_NAME
-iniset $conf_file heat_plugin auth_url $OS_AUTH_URL
-iniset $conf_file heat_plugin user_domain_id $OS_USER_DOMAIN_ID
-iniset $conf_file heat_plugin project_domain_id $OS_PROJECT_DOMAIN_ID
-iniset $conf_file heat_plugin user_domain_name $OS_USER_DOMAIN_NAME
-iniset $conf_file heat_plugin project_domain_name $OS_PROJECT_DOMAIN_NAME
-iniset $conf_file heat_plugin region $OS_REGION_NAME
-iniset $conf_file heat_plugin auth_version $OS_IDENTITY_API_VERSION
-
-source $DEST/devstack/openrc admin admin
-iniset $conf_file heat_plugin admin_username $OS_USERNAME
-iniset $conf_file heat_plugin admin_password $OS_PASSWORD
-
-
-# Register the flavors for booting test servers
-iniset $conf_file heat_plugin instance_type m1.heat_int
-iniset $conf_file heat_plugin minimal_instance_type m1.heat_micro
 openstack flavor create m1.heat_int --ram 512
 openstack flavor create m1.heat_micro --ram 128
-
-iniset $conf_file heat_plugin image_ref Fedora-Cloud-Base-26-1.5.x86_64
-iniset $conf_file heat_plugin boot_config_env $DEST/heat-templates/hot/software-config/boot-config/test_image_env.yaml
-iniset $conf_file heat_plugin heat_config_notify_script $DEST/heat-templates/hot/software-config/elements/heat-config/bin/heat-config-notify
-iniset $conf_file heat_plugin minimal_image_ref cirros-0.3.5-x86_64-disk
-
-# Skip ReloadOnSighupTest. Most jobs now run with apache+uwsgi, so the test has no significance
-# Skip NotificationTest till bug #1721202 is fixed
-# Skip StackCancelTest till the python-heatclient is bumped
-iniset $conf_file heat_plugin skip_functional_test_list 'ReloadOnSighupTest, NotificationTest, StackCancelTest'
-
-# Add scenario tests to skip
-# VolumeBackupRestoreIntegrationTest skipped until failure rate can be reduced ref bug #1382300
-# test_server_signal_userdata_format_software_config is skipped untill bug #1651768 is resolved
-iniset $conf_file heat_plugin skip_scenario_test_list 'SoftwareConfigIntegrationTest, VolumeBackupRestoreIntegrationTest'
-
-if [ "$DISABLE_CONVERGENCE" == "true" ]; then
-    iniset $conf_file heat_plugin convergence_engine_enabled false
-fi
-
-cat $conf_file

tox.ini

@@ -71,6 +71,7 @@ commands =
 [testenv:genconfig]
 commands =
     oslo-config-generator --config-file=config-generator.conf
+    oslo-config-generator --config-file=heat_integrationtests/config-generator.conf
 
 [testenv:genpolicy]
 commands =