Kingbird Refactor(Part 2/Final):

Add configuration for auto discovery of kingbird Tempest testcases.

Restructure and fixed minor issues in existing tempest.

Added README.rst for the steps to run Kingbird tempest plugin.

The configurations which are specific to kingbird
plugin are in kingbird/tests/tempest/scenario/config.py and
currently the default configuration is used from here.

Change-Id: I1b652c82beea61d84b3f42bf657925b1571f7a8d

.testr.conf

@@ -2,7 +2,7 @@
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
              OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
-             ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
+             ${PYTHON:-python} -m subunit.run discover $DISCOVER_DIRECTORY $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list
 

kingbird/tests/tempest/scenario/README.rst

@@ -0,0 +1,106 @@
+Tests for Kingbird in Tempest
+====================================
+
+How to run
+----------
+
+Get the latest kingbird resources from the appropriate mirror:
+
+.. sourcecode:: console
+
+    $ git clone https://github.com/openstack/kingbird.git
+..
+
+Install kingbird, in order to register the tempest plugin interface:
+
+.. sourcecode:: console
+
+    $ cd kingbird
+    $ python setup.py install
+..
+
+Get the latest tempest resources from the appropriate mirror:
+
+.. sourcecode:: console
+
+    $ git clone https://github.com/openstack/tempest.git
+..
+
+Create a configuration file ``tempest/etc/tempest.conf`` for tempest.
+The sample file can be generated and used for this purpose:
+
+.. sourcecode:: console
+
+    $ cd $TEMPEST_ROOT_DIR
+    $ tox -e genconfig
+    $ cp etc/tempest.conf.sample etc/tempest.conf
+..
+
+Some configuration options are required for running tests. Here is the list:
+
+.. sourcecode:: ini
+
+    [auth]
+    admin_username=
+    admin_project_name=
+    admin_password=
+    admin_domain_name=
+
+    [identity]
+    uri=
+    uri_v3=
+    auth_version=
+    username=
+    password=
+    region=
+    tenant_name=
+    domain_name=
+    alt_domain_name=
+    default_domain_id=
+
+    [compute]
+    image_ref=
+
+..
+
+All the parameters above are defined by tempest.
+
+The parameters which are only specific to kingbird tempest plugin
+are configured in kingbird/tests/tempest/scenario/config.py by group name KBGroup.
+
+..
+    endpoint_type=publicURL
+    TIME_TO_SYNC=30
+    endpoint_url=http://127.0.0.1:8118/
+    api_version=v1.0
+
+..
+
+When configuration is finished, you can list the testcases in Kingbird plugin:
+
+.. sourcecode:: console
+
+    $ testr list-tests | grep kingbird
+..
+
+
+If you want to launch the tests from tempest, you can do with:
+
+.. sourcecode:: console
+
+    $ tox -e all-plugin -- scenario.quota_management.client_tests
+..
+
+If you want to launch all Kingbird tests in Tempest, you can do this with ``quota_management`` tag:
+
+.. sourcecode:: console
+
+    $ tox -e all-plugin -- quota_management
+..
+
+If you want to launch a single Kingbird testcase in Tempest, you can do this with:
+
+.. sourcecode:: console
+
+    $ tox -e all-plugin scenario.quota_management.client_tests.test_quota_management_api.KingbirdQMTestJSON.test_kingbird_delete_method
+..

kingbird/tests/tempest/scenario/config.py

@@ -0,0 +1,33 @@
+# Copyright 2016 Ericsson AB
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Configurations for Kingbird Tempest Plugin."""
+
+from oslo_config import cfg
+
+
+KBGroup = [
+    cfg.StrOpt('endpoint_type',
+               default='publicURL',
+               help="Endpoint type of Kingbird service."),
+    cfg.IntOpt('TIME_TO_SYNC',
+               default=30,
+               help="Maximum time to wait for a sync call to complete."),
+    cfg.StrOpt('endpoint_url',
+               default='http://127.0.0.1:8118/',
+               help="Endpoint URL of Kingbird service."),
+    cfg.StrOpt('api_version',
+               default='v1.0',
+               help="Api version of Kingbird service.")
+]

kingbird/tests/tempest/scenario/consts.py

@@ -0,0 +1,27 @@
+# Copyright 2016 Ericsson AB
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+DEFAULT_QUOTAS = {
+    u'quota_set': {
+        u'metadata_items': 128, u'subnet': 10, u'consistencygroups': 10,
+        u'floatingip': 50, u'gigabytes': 1000, u'backup_gigabytes': 1000,
+        u'ram': 51200, u'floating_ips': 10, u'snapshots': 10,
+        u'security_group_rule': 100,
+        u'instances': 10, u'key_pairs': 100, u'volumes': 10, u'router': 10,
+        u'security_group': 10, u'cores': 20, u'backups': 10, u'fixed_ips': -1,
+        u'port': 50, u'security_groups': 10, u'network': 10
+        }
+}

kingbird/tests/tempest/scenario/plugin.py

@@ -18,17 +18,21 @@ import os
 
 from tempest.test_discover import plugins
 
+import kingbird.tests.tempest.scenario.config as kb_config
+
 
 class KingbirdTempestPlugin(plugins.TempestPlugin):
     def load_tests(self):
         base_path = os.path.split(os.path.dirname(
             os.path.abspath(__file__)))[0]
-        test_dir = "tempest/tests/"
+        test_dir = 'scenario'
         full_test_dir = os.path.join(base_path, test_dir)
         return full_test_dir, base_path
 
     def register_opts(self, conf):
-        pass
+        # additional options for Kingbird
+        conf.register_opts(kb_config.KBGroup,
+                           'kingbird')
 
     def get_opt_lists(self):
-        pass
+        return [('kingbird', kb_config.KbGroup)]

kingbird/tests/tempest/scenario/quota_management/client_tests/base.py

@@ -1,4 +1,4 @@
-# Copyright 2012 OpenStack Foundation
+# Copyright 2016 Ericsson AB
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -12,27 +12,22 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+
 import collections
 import time
 
-from tempest.common import kingbird
 from tempest import config
 from tempest.lib.common import api_version_utils
 from tempest.lib.common.utils import data_utils
 import tempest.test
 
+from kingbird.tests.tempest.scenario import consts
+from kingbird.tests.tempest.scenario.quota_management \
+    import sync_client
+
 CONF = config.CONF
 Global_instance_limit = 10
-DEFAULT_QUOTAS = {
-    u'quota_set': {
-        u'metadata_items': 128, u'subnet': 10, u'consistencygroups': 10,
-        u'floatingip': 50, u'gigabytes': 1000, u'backup_gigabytes': 1000,
-        u'ram': 51200, u'floating_ips': 10, u'snapshots': 10,
-        u'instances': 10, u'key_pairs': 100, u'volumes': 10, u'router': 10,
-        u'security_group': 10, u'cores': 20, u'backups': 10, u'fixed_ips': -1,
-        u'port': 50, u'security_groups': 10, u'network': 10
-        }
-    }
+DEFAULT_QUOTAS = consts.DEFAULT_QUOTAS
 # Time to wait for sync to finish
 TIME_TO_SYNC = CONF.kingbird.TIME_TO_SYNC
 
@@ -44,17 +39,14 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
     @classmethod
     def skip_checks(cls):
         super(BaseKingbirdTest, cls).skip_checks()
-        if not CONF.service_available.kingbird:
-            raise cls.skipException("Kingbird is not available")
-        # import config variables
 
     @classmethod
     def setup_credentials(cls):
         super(BaseKingbirdTest, cls).setup_credentials()
-        session = kingbird.get_session()
+        session = sync_client.get_session()
         cls.auth_token = session.get_token()
-        cls.key_client = kingbird.get_key_client(session)
-        cls.regions = kingbird.get_regions(cls.key_client)
+        cls.key_client = sync_client.get_key_client(session)
+        cls.regions = sync_client.get_regions(cls.key_client)
 
     @classmethod
     def setup_clients(cls):
@@ -64,16 +56,16 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
     def resource_setup(cls):
         super(BaseKingbirdTest, cls).resource_setup()
         # Create Project, User, flavor, subnet & network for test
-        project_name = data_utils.rand_name(__name__ + '-project')
-        user_name = data_utils.rand_name(__name__ + '-user')
-        password = data_utils.rand_name(__name__ + '-password')
-        openstack_details = kingbird.get_openstack_drivers(cls.key_client,
-                                                           cls.regions[0],
-                                                           project_name,
-                                                           user_name,
-                                                           password)
+        project_name = data_utils.rand_name('kb-project')
+        user_name = data_utils.rand_name('kb-user')
+        password = data_utils.rand_name('kb-password')
+        openstack_details = sync_client.get_openstack_drivers(cls.key_client,
+                                                              cls.regions[0],
+                                                              project_name,
+                                                              user_name,
+                                                              password)
         cls.openstack_drivers = openstack_details['os_drivers']
-        cls.resource_ids = kingbird.create_resources(cls.openstack_drivers)
+        cls.resource_ids = sync_client.create_resources(cls.openstack_drivers)
         cls.resource_ids.update(openstack_details)
         cls.session = openstack_details['session']
 
@@ -83,61 +75,61 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
         default_quota = {'instances': DEFAULT_QUOTAS['quota_set']['instances'],
                          'cores': DEFAULT_QUOTAS['quota_set']['cores'],
                          'ram': DEFAULT_QUOTAS['quota_set']['ram']}
-        cls.set_default_quota(CONF.kingbird.project_id, default_quota)
-        kingbird.resource_cleanup(cls.openstack_drivers, cls.resource_ids)
-        kingbird.delete_custom_kingbird_quota(
-            cls.auth_token, CONF.kingbird.project_id, None)
+        cls.set_default_quota(cls.resource_ids['project_id'], default_quota)
+        sync_client.resource_cleanup(cls.openstack_drivers, cls.resource_ids)
+        sync_client.delete_custom_kingbird_quota(
+            cls.auth_token, cls.resource_ids['project_id'], None)
 
     def setUp(self):
         super(BaseKingbirdTest, self).setUp()
 
     @classmethod
     def create_custom_kingbird_quota(cls, project_id, new_quota_values):
-        new_values = kingbird.create_custom_kingbird_quota(
+        new_values = sync_client.create_custom_kingbird_quota(
             cls.auth_token, project_id, new_quota_values)
         return new_values
 
     @classmethod
     def get_custom_kingbird_quota(cls, project_id):
-        return_quotas = kingbird.get_custom_kingbird_quota(
+        return_quotas = sync_client.get_custom_kingbird_quota(
             cls.auth_token, project_id)
         return return_quotas
 
     @classmethod
     def delete_custom_kingbird_quota(cls, project_id, quota_to_delete=None):
-        deleted_quotas = kingbird.delete_custom_kingbird_quota(
+        deleted_quotas = sync_client.delete_custom_kingbird_quota(
             cls.auth_token, project_id, quota_to_delete)
         return deleted_quotas
 
     @classmethod
     def get_default_kingbird_quota(cls):
-        return_quotas = kingbird.get_default_kingbird_quota(cls.auth_token)
+        return_quotas = sync_client.get_default_kingbird_quota(cls.auth_token)
         return return_quotas
 
     @classmethod
     def quota_sync_for_project(cls, project_id):
-        sync_status = kingbird.quota_sync_for_project(
+        sync_status = sync_client.quota_sync_for_project(
             cls.auth_token, project_id)
         return sync_status
 
     @classmethod
     def get_quota_usage_for_project(cls, project_id):
-        quota_usage = kingbird.get_quota_usage_for_project(
+        quota_usage = sync_client.get_quota_usage_for_project(
             cls.auth_token, project_id)
         return quota_usage
 
     @classmethod
     def create_custom_kingbird_quota_wrong_token(cls, project_id,
                                                  new_quota_values):
-        new_values = kingbird.create_custom_kingbird_quota_wrong_token(
+        new_values = sync_client.create_custom_kingbird_quota_wrong_token(
             cls.auth_token, project_id, new_quota_values)
         return new_values
 
     @classmethod
     def create_instance(cls, count=1):
         try:
-            server_ids = kingbird.create_instance(cls.openstack_drivers,
-                                                  cls.resource_ids, count)
+            server_ids = sync_client.create_instance(cls.openstack_drivers,
+                                                     cls.resource_ids, count)
         except Exception as e:
             server_ids = {'server_ids': list(e.args)}
             raise
@@ -146,13 +138,13 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
 
     @classmethod
     def delete_instance(cls):
-        kingbird.delete_instance(cls.openstack_drivers, cls.resource_ids)
+        sync_client.delete_instance(cls.openstack_drivers, cls.resource_ids)
         cls.resource_ids['instances'] = None
 
     @classmethod
     def calculate_quota_limits(cls, project_id):
         calculated_quota_limits = collections.defaultdict(dict)
-        resource_usage = kingbird.get_usage_from_os_client(
+        resource_usage = sync_client.get_usage_from_os_client(
             cls.session, cls.regions, project_id)
         total_usages = cls.get_summation(resource_usage)
         for current_region in cls.regions:
@@ -177,14 +169,14 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
 
     @classmethod
     def get_usage_manually(cls, project_id):
-        resource_usage = kingbird.get_usage_from_os_client(
+        resource_usage = sync_client.get_usage_from_os_client(
             cls.session, cls.regions, project_id)
         resource_usage = cls.get_summation(resource_usage)
         return {'quota_set': resource_usage}
 
     @classmethod
     def get_actual_limits(cls, project_id):
-        actual_limits = kingbird.get_actual_limits(
+        actual_limits = sync_client.get_actual_limits(
             cls.session, cls.regions, project_id)
         return actual_limits
 
@@ -194,5 +186,5 @@ class BaseKingbirdTest(api_version_utils.BaseMicroversionTest,
 
     @classmethod
     def set_default_quota(cls, project_id, quota_to_set):
-        kingbird.set_default_quota(
+        sync_client.set_default_quota(
             cls.session, cls.regions, project_id, **quota_to_set)

kingbird/tests/tempest/scenario/quota_management/client_tests/test_kingbird_api.py

@@ -1,4 +1,4 @@
-# Copyright 2012 OpenStack Foundation
+# Copyright 2016 Ericsson AB
 # All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -13,7 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from tempest.api.kingbird import base
+from kingbird.tests.tempest.scenario.quota_management. \
+    client_tests import base
 from tempest import config
 
 import novaclient

kingbird/tests/tempest/scenario/quota_management/sync_client.py

@@ -34,6 +34,7 @@ NETWORK_NAME = "kb_test_network"
 SUBNET_NAME = "kb_test_subnet"
 SERVER_NAME = "kb_test_server"
 SUBNET_RANGE = "192.168.199.0/24"
+quota_api_url = "/os-quota-sets/"
 
 LOG = logging.getLogger(__name__)
 
@@ -42,7 +43,7 @@ def get_session():
     return get_current_session(
         CONF.identity.username,
         CONF.identity.password,
-        CONF.identity.tenant_name
+        CONF.identity.project_name
         )
 
 
@@ -100,35 +101,35 @@ def create_instance(openstack_drivers, resource_ids, count=1):
         raise e
 
 
-def get_urlstring_and_headers(token):
-    admin_tenant_id = CONF.auth.admin_tenant_name
+def get_urlstring_and_headers(token, api_url):
+    admin_tenant_id = CONF.auth.admin_project_name
     headers = {
         'Content-Type': 'application/json',
         'X-Auth-Token': token,
         'X-ROLE': 'admin',
     }
     url_string = CONF.kingbird.endpoint_url + CONF.kingbird.api_version + \
-        "/" + admin_tenant_id + "/os-quota-sets/"
+        "/" + admin_tenant_id + api_url
     return headers, url_string
 
 
 def create_custom_kingbird_quota(token, project_id, new_quota_values):
     body = json.dumps(new_quota_values)
-    headers, url_string = get_urlstring_and_headers(token)
+    headers, url_string = get_urlstring_and_headers(token, quota_api_url)
     url_string = url_string + project_id
     response = requests.put(url_string, headers=headers, data=body)
     return response.text
 
 
 def get_custom_kingbird_quota(token, project_id):
-    headers, url_string = get_urlstring_and_headers(token)
+    headers, url_string = get_urlstring_and_headers(token, quota_api_url)
     url_string = url_string + project_id
     response = requests.get(url_string, headers=headers)
     return response.text
 
 
 def delete_custom_kingbird_quota(token, project_id, quota_to_delete=None):
-    headers, url_string = get_urlstring_and_headers(token)
+    headers, url_string = get_urlstring_and_headers(token, quota_api_url)
     url_string = url_string + project_id
     if quota_to_delete:
         body = json.dumps(quota_to_delete)
@@ -139,21 +140,21 @@ def delete_custom_kingbird_quota(token, project_id, quota_to_delete=None):
 
 
 def get_default_kingbird_quota(token):
-    headers, url_string = get_urlstring_and_headers(token)
+    headers, url_string = get_urlstring_and_headers(token, quota_api_url)
     url_string = url_string + "defaults"
     response = requests.get(url_string, headers=headers)
     return response.text
 
 
 def quota_sync_for_project(token, project_id):
-    headers, url_string = get_urlstring_and_headers(token)
+    headers, url_string = get_urlstring_and_headers(token, quota_api_url)
     url_string = url_string + project_id + "/sync"
     response = requests.put(url_string, headers=headers)
     return response.text
 
 
 def get_quota_usage_for_project(token, project_id):
-    headers, url_string = get_urlstring_and_headers(token)
+    headers, url_string = get_urlstring_and_headers(token, quota_api_url)
     url_string = url_string + project_id + "/detail"
     response = requests.get(url_string, headers=headers)
     return response.text
@@ -161,7 +162,7 @@ def get_quota_usage_for_project(token, project_id):
 
 def create_custom_kingbird_quota_wrong_token(token,
                                              project_id, new_quota_values):
-    headers, url_string = get_urlstring_and_headers(token)
+    headers, url_string = get_urlstring_and_headers(token, quota_api_url)
     headers['X-Auth-Token'] = 'fake_token'
     url_string = url_string + project_id
     body = json.dumps(new_quota_values)

run_tests.sh

@@ -47,14 +47,14 @@ flake8args="kingbird"
 function run_tests {
     echo 'Running tests'
     # Remove any extraneous DB migrations
-    #find kingbird/db/sqlalchemy/migrate_repo/versions/ -name '*.pyc' -delete
+    find kingbird/db/sqlalchemy/migrate_repo/versions/ -name '*.pyc' -delete
 
     if [ $debug -eq 1 ]; then
       echo "Debugging..."
       if [ "$args" = "" ]; then
         # Default to running all tests if specific test is not
         # provided.
-        testrargs="discover ./kingbird/tests"
+        testrargs="discover ./kingbird/tests/unit"
       fi
       ${wrapper} python -m testtools.run $args $testrargs
 

setup.cfg

@@ -29,6 +29,9 @@ console_scripts =
     kingbird-engine = kingbird.cmd.engine:main
     kingbird-manage = kingbird.cmd.manage:main
 
+tempest.test_plugins =
+    kingbird_tests = kingbird.tests.tempest.scenario.plugin:KingbirdTempestPlugin
+
 oslo.config.opts =
     kingbird.common.config = kingbird.common.config:list_opts
     kingbird.common.manager = kingbird.common.manager:list_opts

tox.ini

@@ -4,16 +4,24 @@ envlist = py34,py27,pypy,pep8
 skipsdist = True
 
 [testenv]
-sitepackages = True
 usedevelop = True
-install_command = pip install -U --force-reinstall {opts} {packages}
+install_command = pip install -U {opts} {packages}
 setenv =
-   VIRTUAL_ENV={envdir}
+    VIRTUAL_ENV={envdir}
+    DISCOVER_DIRECTORY=kingbird/tests/unit
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands = python setup.py testr --slowest --testr-args='{posargs}'
 whitelist_externals = rm
 
+[testenv:debug-py27]
+basepython = python2.7
+commands = oslo_debug_helper {posargs}
+
+[testenv:debug-py34]
+basepython = python3.4
+commands = oslo_debug_helper {posargs}
+
 [testenv:pep8]
 deps = hacking<0.11,>=0.10.2
 commands = flake8