Change to use tempest plugin framework

Tempest plugin framework provides more flexibility for projects.
With this plugin, it's not requiring the tests live in the tempest
tree.

Change-Id: I8681140d02926b4b3e6af84b8f03b9385e62cc72

.testr.conf

@@ -2,6 +2,6 @@
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
              OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
-             ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
+             ${PYTHON:-python} -m subunit.run discover -t ./ ./congress/tests $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
-test_list_option=--list
+test_list_option=--list

congress_tempest_tests/config.py

@@ -0,0 +1,30 @@
+# Copyright 2015 Intel Corp
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+
+from tempest import config  # noqa
+
+
+congressha_group = cfg.OptGroup(name="congressha", title="Congress HA Options")
+
+CongressHAGroup = [
+    cfg.StrOpt("replica_type",
+               default="policyha",
+               help="service type used to create a replica congress server."),
+    cfg.IntOpt("replica_port",
+               default=4001,
+               help="The listening port for a replica congress server. "),
+]

congress_tempest_tests/plugin.py

@@ -0,0 +1,39 @@
+# Copyright 2015 Intel
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import os
+
+from tempest import config
+from tempest.test_discover import plugins
+
+from congress_tempest_tests import config as config_congress
+
+
+class CongressTempestPlugin(plugins.TempestPlugin):
+    def load_tests(self):
+        base_path = os.path.split(os.path.dirname(
+            os.path.abspath(__file__)))[0]
+        test_dir = "congress_tempest_tests/tests"
+        full_test_dir = os.path.join(base_path, test_dir)
+        return full_test_dir, base_path
+
+    def register_opts(self, conf):
+        config.register_opt_group(conf, config_congress.congressha_group,
+                                  config_congress.CongressHAGroup)
+
+    def get_opt_lists(self):
+        return [(config_congress.congressha_group.name,
+                 config_congress.CongressHAGroup)]

congress_tempest_tests/services/policy/policy_client.py

@@ -17,8 +17,8 @@ import json
 
 from tempest.common import service_client
 
-class PolicyClient(service_client.ServiceClient):
 
+class PolicyClient(service_client.ServiceClient):
 
     policy = '/v1/policies'
     policy_path = '/v1/policies/%s'
@@ -105,8 +105,7 @@ class PolicyClient(service_client.ServiceClient):
         return self._resp_helper(resp, body)
 
     def show_policy_table(self, policy_name, table_id):
-        resp, body = self.get(self.policy_table_path %
-                                         (policy_name, table_id))
+        resp, body = self.get(self.policy_table_path % (policy_name, table_id))
         return self._resp_helper(resp, body)
 
     def list_datasources(self):
@@ -114,33 +113,30 @@ class PolicyClient(service_client.ServiceClient):
         return self._resp_helper(resp, body)
 
     def list_datasource_tables(self, datasource_name):
-        resp, body = self.get(self.datasource_tables %
-                                         (datasource_name))
+        resp, body = self.get(self.datasource_tables % (datasource_name))
         return self._resp_helper(resp, body)
 
     def list_datasource_rows(self, datasource_name, table_name):
         resp, body = self.get(self.datasource_rows %
-                                         (datasource_name, table_name))
+                              (datasource_name, table_name))
         return self._resp_helper(resp, body)
 
     def list_datasource_status(self, datasource_name):
-        resp, body = self.get(self.datasource_status %
-                                         datasource_name)
+        resp, body = self.get(self.datasource_status % datasource_name)
         return self._resp_helper(resp, body)
 
     def show_datasource_schema(self, datasource_name):
-        resp, body = self.get(self.datasource_schema %
-                                         datasource_name)
+        resp, body = self.get(self.datasource_schema % datasource_name)
         return self._resp_helper(resp, body)
 
     def show_datasource_table_schema(self, datasource_name, table_name):
         resp, body = self.get(self.datasource_table_schema %
-                                         (datasource_name, table_name))
+                              (datasource_name, table_name))
         return self._resp_helper(resp, body)
 
     def show_datasource_table(self, datasource_name, table_id):
         resp, body = self.get(self.datasource_table_path %
-                                         (datasource_name, table_id))
+                              (datasource_name, table_id))
         return self._resp_helper(resp, body)
 
     def create_datasource(self, body=None):
@@ -166,13 +162,12 @@ class PolicyClient(service_client.ServiceClient):
         return self._resp_helper(resp, body)
 
     def show_driver(self, driver):
-        resp, body = self.get(self.driver_path %
-                                         (driver))
+        resp, body = self.get(self.driver_path % (driver))
         return self._resp_helper(resp, body)
 
     def request_refresh(self, driver, body=None):
         body = json.dumps(body)
         resp, body = self.post(self.datasource_path %
-                                          (driver) + "?action=request-refresh",
-                                          body=body)
+                               (driver) + "?action=request-refresh",
+                               body=body)
         return self._resp_helper(resp, body)

congress_tempest_tests/tests/README.rst

@@ -0,0 +1,17 @@
+====================
+Tempest Integration
+====================
+
+This directory contains Tempest tests to cover Congress project.
+
+To list all Congress tempest cases, go to tempest directory, then run::
+
+    $ testr list-tests congress
+
+To run only these tests in tempest, go to tempest directory, then run::
+
+    $ ./run_tempest -N -- congress
+
+To run a single test case, go to tempest directory, then run with test case name, e.g.::
+
+    $ ./run_tempest.sh -N -- congress_tempest_tests.tests.scenario.test_congress_basic_ops.TestPolicyBasicOps.test_policy_basic_op

congress_tempest_tests/tests/scenario/congress_datasources/test_ceilometer.py

@@ -18,9 +18,10 @@ from tempest_lib import decorators
 from tempest import clients  # noqa
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
-from tempest.scenario import manager_congress  # noqa
 from tempest import test  # noqa
 
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
+
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)
@@ -35,7 +36,7 @@ class TestCeilometerDriver(manager_congress.ScenarioPolicyBase):
     def setUp(cls):
         super(TestCeilometerDriver, cls).setUp()
         if not CONF.service_available.ceilometer:
-            msg = ("%s skipped as ceilometer is not available" % \
+            msg = ("%s skipped as ceilometer is not available" %
                    cls.__class__.__name__)
             raise cls.skipException(msg)
         cls.os = clients.Manager(cls.admin_manager.auth_provider.credentials)

congress_tempest_tests/tests/scenario/congress_datasources/test_cinder.py

@@ -18,9 +18,10 @@ from tempest_lib import decorators
 from tempest import clients  # noqa
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
-from tempest.scenario import manager_congress  # noqa
 from tempest import test  # noqa
 
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
+
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)

congress_tempest_tests/tests/scenario/congress_datasources/test_glancev2.py

@@ -18,9 +18,9 @@ from tempest_lib import decorators
 from tempest import clients  # noqa
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
-from tempest.scenario import manager_congress  # noqa
 from tempest import test  # noqa
 
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)

congress_tempest_tests/tests/scenario/congress_datasources/test_keystonev2.py

@@ -18,9 +18,9 @@ from tempest_lib import decorators
 from tempest import clients  # noqa
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
-from tempest.scenario import manager_congress  # noqa
 from tempest import test  # noqa
 
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)

congress_tempest_tests/tests/scenario/congress_datasources/test_murano.py

@@ -11,15 +11,16 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-from tempest_lib import decorators
-
-from tempest import config
-from tempest.scenario import manager_congress
-from tempest import test
-
 import random
 import string
 
+from tempest import config
+from tempest import test
+from tempest_lib import decorators
+
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
+
+
 CONF = config.CONF
 
 

congress_tempest_tests/tests/scenario/congress_datasources/test_neutronv2.py

@@ -20,9 +20,10 @@ from tempest_lib import decorators
 from tempest import clients  # noqa
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
-from tempest.scenario import manager_congress  # noqa
 from tempest import test  # noqa
 
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
+
 CONF = config.CONF
 LOG = logging.getLogger(__name__)
 

congress_tempest_tests/tests/scenario/congress_datasources/test_nova.py

@@ -17,9 +17,9 @@ from tempest_lib import decorators
 
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
-from tempest.scenario import manager_congress  # noqa
 from tempest import test  # noqa
 
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)

congress_tempest_tests/tests/scenario/congress_ha/test_ha.py

@@ -19,16 +19,16 @@ import subprocess
 import tempfile
 
 from oslo_log import log as logging
-from tempest_lib import decorators
-
+from tempest.common import cred_provider
 from tempest import config
 from tempest import exceptions
 from tempest import manager as tempestmanager
-from tempest_lib import exceptions as restexc
-from tempest.common import cred_provider
-from tempest.scenario import manager_congress
-from tempest.services.policy import policy_client
 from tempest import test
+from tempest_lib import decorators
+from tempest_lib import exceptions as restexc
+
+from congress_tempest_tests.services.policy import policy_client
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)
@@ -36,8 +36,6 @@ LOG = logging.getLogger(__name__)
 
 class TestHA(manager_congress.ScenarioPolicyBase):
 
-    REPLICA_TYPE='policyha'
-
     def setUp(self):
         super(TestHA, self).setUp()
         self.keypairs = {}
@@ -47,7 +45,8 @@ class TestHA(manager_congress.ScenarioPolicyBase):
     def _prepare_replica(self, port_num):
         replica_url = "http://127.0.0.1:%d" % port_num
         ksclient = self.admin_manager.identity_client
-        resp = ksclient.create_service('congressha', self.REPLICA_TYPE,
+        resp = ksclient.create_service('congressha',
+                                       CONF.congressha.replica_type,
                                        description='policy ha service')
         self.replica_service_id = resp['id']
         resp = ksclient.create_endpoint(self.replica_service_id,
@@ -165,10 +164,9 @@ class TestHA(manager_congress.ScenarioPolicyBase):
         item = {'id': None,
                 'name': 'fake',
                 'driver': 'fake_datasource',
-                'config': '{"username":"fakeu", \
-                            "tenant_name": "faket", \
-                            "password": "fakep", \
-                            "auth_url": "http://127.0.0.1:5000/v2"}',
+                'config': '{"username":"fakeu", "tenant_name": "faket",' +
+                          '"password": "fakep",' +
+                          '"auth_url": "http://127.0.0.1:5000/v2"}',
                 'description': 'bar',
                 'enabled': True}
         ret = client.create_datasource(item)
@@ -180,7 +178,6 @@ class TestHA(manager_congress.ScenarioPolicyBase):
     def test_datasource_db_sync_add(self):
         # Verify that a replica adds a datasource when a datasource
         # appears in the database.
-        CLIENT2_PORT = 4001
         client1 = self.admin_manager.congress_client
 
         # delete fake if it exists.
@@ -209,10 +206,10 @@ class TestHA(manager_congress.ScenarioPolicyBase):
                     "primary should have fake, but does not")
 
             # start replica
-            self.start_replica(CLIENT2_PORT)
+            self.start_replica(CONF.congressha.replica_port)
 
             # Create session for second server.
-            client2 = self.create_client(self.REPLICA_TYPE)
+            client2 = self.create_client(CONF.congressha.replica_type)
 
             # Verify that second server has fake datasource
             if not test.call_until_true(
@@ -230,7 +227,7 @@ class TestHA(manager_congress.ScenarioPolicyBase):
             if not test.call_until_true(
                     func=lambda: self.datasource_missing(client1, fake_id),
                     duration=60, sleep_for=1):
-                self.stop_replica(CLIENT2_PORT)
+                self.stop_replica(CONF.congressha.replica_port)
                 raise exceptions.TimeoutException(
                     "primary instance still has fake")
             LOG.debug("removed fake datasource from primary instance")
@@ -243,7 +240,7 @@ class TestHA(manager_congress.ScenarioPolicyBase):
                     "replica should remove fake, but still has it")
 
         finally:
-            self.stop_replica(CLIENT2_PORT)
+            self.stop_replica(CONF.congressha.replica_port)
             if need_to_delete_fake:
                 self.admin_manager.congress_client.delete_datasource(fake_id)
 
@@ -252,12 +249,11 @@ class TestHA(manager_congress.ScenarioPolicyBase):
     def test_datasource_db_sync_remove(self):
         # Verify that a replica removes a datasource when a datasource
         # disappears from the database.
-        CLIENT2_PORT = 4001
         client1 = self.admin_manager.congress_client
         fake_id = self.create_fake(client1)
         need_to_delete_fake = True
         try:
-            self.start_replica(CLIENT2_PORT)
+            self.start_replica(CONF.congressha.replica_port)
 
             # Verify that primary server has fake datasource
             if not test.call_until_true(
@@ -267,7 +263,7 @@ class TestHA(manager_congress.ScenarioPolicyBase):
                     "primary should have fake, but does not")
 
             # Create session for second server.
-            client2 = self.create_client(self.REPLICA_TYPE)
+            client2 = self.create_client(CONF.congressha.replica_type)
 
             # Verify that second server has fake datasource
             if not test.call_until_true(
@@ -285,7 +281,7 @@ class TestHA(manager_congress.ScenarioPolicyBase):
             if not test.call_until_true(
                     func=lambda: self.datasource_missing(client1, fake_id),
                     duration=60, sleep_for=1):
-                self.stop_replica(CLIENT2_PORT)
+                self.stop_replica(CONF.congressha.replica_port)
                 raise exceptions.TimeoutException(
                     "primary instance still has fake")
             LOG.debug("removed fake datasource from primary instance")
@@ -298,6 +294,6 @@ class TestHA(manager_congress.ScenarioPolicyBase):
                     "replica should remove fake, but still has it")
 
         finally:
-            self.stop_replica(CLIENT2_PORT)
+            self.stop_replica(CONF.congressha.replica_port)
             if need_to_delete_fake:
                 self.admin_manager.congress_client.delete_datasource(fake_id)

congress_tempest_tests/tests/scenario/manager_congress.py

@@ -17,16 +17,16 @@ import collections
 import re
 
 from oslo_log import log as logging
-from tempest_lib.common.utils import data_utils
-
 from tempest.common import cred_provider
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
 from tempest import manager as tempestmanager
 from tempest.scenario import manager  # noqa
-from tempest.services.policy import policy_client
 from tempest.services.network import resources as net_resources  # noqa
 from tempest import test  # noqa
+from tempest_lib.common.utils import data_utils
+
+from congress_tempest_tests.services.policy import policy_client
 
 CONF = config.CONF
 LOG = logging.getLogger(__name__)

congress_tempest_tests/tests/scenario/test_congress_basic_ops.py

@@ -18,10 +18,11 @@ from tempest_lib import decorators
 
 from tempest import config  # noqa
 from tempest import exceptions  # noqa
-from tempest.scenario import helper
-from tempest.scenario import manager_congress  # noqa
 from tempest import test  # noqa
 
+from congress_tempest_tests.tests.scenario import helper  # noqa
+from congress_tempest_tests.tests.scenario import manager_congress  # noqa
+
 import random
 import string
 
@@ -154,7 +155,8 @@ class TestPolicyBasicOps(manager_congress.ScenarioPolicyBase):
             else:
                 return False
 
-        if not test.call_until_true(func=check_data, duration=100, sleep_for=5):
+        if not test.call_until_true(func=check_data,
+                                    duration=100, sleep_for=5):
             raise exceptions.TimeoutException("Data did not converge in time "
                                               "or failure in server")
 
@@ -230,6 +232,7 @@ class TestCongressDataSources(manager_congress.ScenarioPolicyBase):
                     return False
             return True
 
-        if not test.call_until_true(func=check_data, duration=100, sleep_for=5):
+        if not test.call_until_true(func=check_data,
+                                    duration=100, sleep_for=5):
             raise exceptions.TimeoutException("Data did not converge in time "
                                               "or failure in server")

contrib/tempest/README.rst

@@ -1,27 +0,0 @@
-The contrib/tempest directory contains the files necessary to integrate
-Congress with tempest for functional testing.
-
-NOTE: One needs to have the tempest source codebase to do this. The easiest
-way to get tempest setup on your system is to add tempest to
-ENABLED_SERVICES in devstack (when one runs ./stack.sh). Then:
-
-To setup with tempest:
-
-    $ cp -r contrib/tempest/tempest/ /opt/stack/tempest/
-
-Then, run tests within tempest as you normally would (via ./run_tempest.sh
-or via tox)
-
-To list all Congress test cases, run command in /opt/stack/tempest:
-
-    $ testr list-tests | grep -i congress
-
-To debug one test case with pdb, add pdb.set_trace() in a proper place and
-then run:
-
-    $ ./run_tempest.sh -N -d test-case-path
-
-E.g.
-
-    $ ./run_tempest.sh -N -d tempest.scenario.test_congress_basic_ops.\
-    > TestCongressDataSources.test_all_datasources_have_tables

setup.cfg

@@ -38,6 +38,10 @@ console_scripts =
     congress-server = congress.server.congress_server:main
     congress-db-manage = congress.db.migration.cli:main
 
+tempest.test_plugins =
+    congress_tests = congress_tempest_tests.plugin:CongressTempestPlugin
+
+
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build