Migration to utilize tempest plugin

This is a migration step to utilize tempest plugin
system instead of directly calling functional api tests.

This is the approach used by a number of other projects as well
as an approved process by openstack-qa.

The difference in execution is that we will need to execute tempest's
tox instead of our own:

tox -eall-plugin magnum.tests.functional.api.v1 --  --concurrency=1

Implements: blueprint magnum-tempest
Change-Id: Ic3eadae7fb5d88b776f9ded9589ef25279a2e1be

doc/source/dev/dev-functional-test.rst

@@ -61,15 +61,23 @@ Use follow command lines to check what functional testing is supported::
     cd /opt/stack/magnum
     cat tox.ini | grep functional- | awk -F: '{print $2}' | sed s/]//
 
-To do some specify functional testing, for example, test api functional
+To do some specify functional testing, for example, test kubernetes functional
 cases::
 
-    tox -e functional-api -- --concurrency=1
+    tox -e functional-k8s -- --concurrency=1
 
-Test specified case of api functional cases::
+Test specified case of kubernetes functional cases::
+
+    tox -e functional-k8s -- --concurrency=1 <test path>
 
-    tox -e functional-api -- --concurrency=1 <test path>
 
 The following is an example for test path::
 
-    magnum.tests.functional.api.v1.test_baymodel.BayModelTest.test_create_baymodel
+    magnum.tests.functional.k8s.v1.test_k8s_python_client.TestBayModelResource.test_baymodel_create_and_delete
+
+We also have api tests being added.  To run those, go to
+tempest directory (tempest is installed with devstack by default), install
+magnum, and execute tempest tox::
+
+    cd /opt/stack/tempest
+    tox -eall-plugin magnum.tests.functional.api.v1 -- --concurrency=1

magnum/tests/contrib/post_test_hook.sh

@@ -22,46 +22,17 @@ function function_exists {
     declare -f -F $1 > /dev/null
 }
 
-if ! function_exists echo_summary; then
-    function echo_summary {
-        echo $@
-    }
-fi
+# Set up all necessary test data
+function create_test_data {
+    # First we test Magnum's command line to see if we can stand up
+    # a baymodel, bay and a pod
 
-# Save trace setting
-XTRACE=$(set +o | grep xtrace)
-set -o xtrace
-
-echo_summary "magnum's post_test_hook.sh was called..."
-(set -o posix; set)
+    export NIC_ID=$(neutron net-show public | awk '/ id /{print $4}')
+    export IMAGE_ID=$(glance --os-image-api-version 1 image-show fedora-21-atomic-5 | awk '/ id /{print $4}')
 
-constraints="-c $REQUIREMENTS_DIR/upper-constraints.txt"
-sudo pip install $constraints -U -r requirements.txt -r test-requirements.txt
-
-export MAGNUM_DIR="$BASE/new/magnum"
-sudo chown -R jenkins:stack $MAGNUM_DIR
-
-# Get admin credentials
-pushd ../devstack
-source openrc admin admin
-# NOTE(hongbin): This is a temporary work around. These variables are for
-# keystone v3, but magnum is using v2 API. Therefore, unset them to make the
-# keystoneclient work.
-# Bug: #1473600
-unset OS_PROJECT_DOMAIN_ID
-unset OS_USER_DOMAIN_ID
-unset OS_AUTH_TYPE
-popd
-
-# First we test Magnum's command line to see if we can stand up
-# a baymodel, bay and a pod
-export NIC_ID=$(neutron net-show public | awk '/ id /{print $4}')
-export IMAGE_ID=$(glance --os-image-api-version 1 image-show fedora-21-atomic-5 | awk '/ id /{print $4}')
-
-
-# pass the appropriate variables via a config file
-CREDS_FILE=$MAGNUM_DIR/functional_creds.conf
-cat <<EOF > $CREDS_FILE
+    # pass the appropriate variables via a config file
+    CREDS_FILE=$MAGNUM_DIR/functional_creds.conf
+    cat <<EOF > $CREDS_FILE
 # Credentials for functional testing
 
 [auth]
@@ -84,25 +55,113 @@ flavor_id = m1.magnum
 copy_logs = true
 EOF
 
-# Note(eliqiao): Let's keep this only for debugging on gate.
-echo_summary $CREDS_FILE
-cat $CREDS_FILE
+    # Note(eliqiao): Let's keep this only for debugging on gate.
+    echo_summary $CREDS_FILE
+    cat $CREDS_FILE
 
-# Create a keypair for use in the functional tests.
-echo_summary "Generate a key-pair"
-ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
-nova keypair-add --pub-key ~/.ssh/id_rsa.pub default
+    # Create a keypair for use in the functional tests.
+    echo_summary "Generate a key-pair"
+    ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
+    nova keypair-add  --pub-key ~/.ssh/id_rsa.pub default
 
-# Create magnum specific flavor for use in functional tests.
-echo_summary "Create a flavor"
-nova flavor-create  m1.magnum 100 1024 8 1
+}
+
+function add_flavor {
+    # because of policy.json change in nova, flavor-create is now an admin-only feature
+    # moving this out to only be used by admins
+
+    # Get admin credentials
+    pushd ../devstack
+    source openrc admin admin
+    # NOTE(hongbin): This is a temporary work around. These variables are for
+    # keystone v3, but magnum is using v2 API. Therefore, unset them to make the
+    # keystoneclient work.
+    # Bug: #1473600
+    unset OS_PROJECT_DOMAIN_ID
+    unset OS_USER_DOMAIN_ID
+    unset OS_AUTH_TYPE
+    popd
+
+    # Create magnum specific flavor for use in functional tests.
+    echo_summary "Create a flavor"
+    nova flavor-create  m1.magnum 100 1024 8 1
+}
+
+if ! function_exists echo_summary; then
+    function echo_summary {
+        echo $@
+    }
+fi
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set -o xtrace
+
+echo_summary "magnum's post_test_hook.sh was called..."
+(set -o posix; set)
+
+constraints="-c $REQUIREMENTS_DIR/upper-constraints.txt"
+sudo pip install $constraints -U -r requirements.txt -r test-requirements.txt
+
+export MAGNUM_DIR="$BASE/new/magnum"
+sudo chown -R jenkins:stack $MAGNUM_DIR
 
 # Run functional tests
 # Currently we support functional-api, functional-k8s, will support swarm,
 # mesos later.
 
 echo "Running magnum functional test suite for $1"
-sudo -E -H -u jenkins tox -e functional-$1 -- --concurrency=1
+
+# For api, we will run tempest tests
+if [[ "api" == $1 ]]; then
+    # Import devstack functions 'iniset', 'iniget' and 'trueorfalse'
+    source $BASE/new/devstack/functions
+    echo "TEMPEST_SERVICES+=,magnum" >> $localrc_path
+    pushd $BASE/new/tempest
+    sudo chown -R jenkins:stack $BASE/new/tempest
+
+    add_flavor
+
+    # Set demo credentials
+    source $BASE/new/devstack/accrc/demo/demo
+    unset OS_AUTH_TYPE
+
+    create_test_data
+
+    # Set up tempest config with magnum goodness
+    iniset $BASE/new/tempest/etc/tempest.conf magnum image_id $IMAGE_ID
+    iniset $BASE/new/tempest/etc/tempest.conf magnum nic_id $NIC_ID
+    iniset $BASE/new/tempest/etc/tempest.conf magnum keypair_id default
+    iniset $BASE/new/tempest/etc/tempest.conf magnum flavor_id m1.magnum
+
+    # show tempest config with magnum
+    cat etc/tempest.conf
+
+    # Set up concurrency and test regex
+    export MAGNUM_TEMPEST_CONCURRENCY=${MAGNUM_TEMPEST_CONCURRENCY:-1}
+    export MAGNUM_TESTS=${MAGNUM_TESTS:-'magnum.tests.functional.api.v1'}
+
+    echo "Running tempest magnum test suites"
+    sudo -H -u jenkins tox -eall-plugin -- $MAGNUM_TESTS --concurrency=$MAGNUM_TEMPEST_CONCURRENCY
+else
+    # Get admin credentials
+    pushd ../devstack
+    source openrc admin admin
+    # NOTE(hongbin): This is a temporary work around. These variables are for
+    # keystone v3, but magnum is using v2 API. Therefore, unset them to make the
+    # keystoneclient work.
+    # Bug: #1473600
+    unset OS_PROJECT_DOMAIN_ID
+    unset OS_USER_DOMAIN_ID
+    unset OS_AUTH_TYPE
+    popd
+
+    add_flavor
+
+    create_test_data
+
+    sudo -E -H -u jenkins tox -e functional-$1 -- --concurrency=1
+fi
 EXIT_CODE=$?
 
 # Delete the keypair used in the functional test.

magnum/tests/functional/common/config.py

@@ -10,7 +10,10 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import ConfigParser
+from tempest import config
+
+
+CONF = config.CONF
 
 
 class Config(object):
@@ -19,77 +22,65 @@ class Config(object):
 
     @classmethod
     def set_admin_creds(cls, config):
-        cls.admin_user = config.get('admin', 'user')
-        cls.admin_passwd = config.get('admin', 'pass')
-        cls.admin_tenant = config.get('admin', 'tenant')
+        cls.admin_user = CONF.auth.admin_username
+        cls.admin_passwd = CONF.auth.admin_password
+        cls.admin_tenant = CONF.auth.admin_tenant_name
 
     @classmethod
     def set_user_creds(cls, config):
         # normal user creds
-        cls.user = config.get('auth', 'username')
-        cls.passwd = config.get('auth', 'password')
-        cls.tenant = config.get('auth', 'tenant_name')
+        cls.user = CONF.identity.username
+        cls.passwd = CONF.identity.password
+        cls.tenant = CONF.identity.tenant_name
 
     @classmethod
     def set_auth_version(cls, config):
         # auth version for client authentication
-        if config.has_option('auth', 'auth_version'):
-            cls.auth_version = config.get('auth', 'auth_version')
-        else:
-            cls.auth_version = 'v3'
+        cls.auth_version = CONF.identity.auth_version
 
     @classmethod
     def set_auth_url(cls, config):
         # auth_url for client authentication
         if cls.auth_version == 'v3':
-            if not config.has_option('auth', 'auth_v3_url'):
-                raise Exception('config missing auth_v3_url key')
-            cls.auth_v3_url = config.get('auth', 'auth_v3_url')
+            cls.auth_v3_url = CONF.identity.uri_v3
         else:
-            if not config.has_option('auth', 'auth_url'):
+            if 'uri' not in CONF.identity:
                 raise Exception('config missing auth_url key')
-            cls.auth_url = config.get('auth', 'auth_url')
+            cls.auth_url = CONF.identity.uri
 
     @classmethod
     def set_region(cls, config):
-        if config.has_option('auth', 'region'):
-            cls.region = config.get('auth', 'region')
+        if 'region' in CONF.identity:
+            cls.region = CONF.identity.region
         else:
             cls.region = 'RegionOne'
 
     @classmethod
     def set_image_id(cls, config):
-        cls.image_id = config.get('magnum', 'image_id')
-        if not config.has_option('magnum', 'image_id'):
+        if 'image_id' not in CONF.magnum:
             raise Exception('config missing image_id key')
+        cls.image_id = CONF.magnum.image_id
 
     @classmethod
     def set_nic_id(cls, config):
-        cls.nic_id = config.get('magnum', 'nic_id')
-        if not config.has_option('magnum', 'nic_id'):
+        if 'nic_id' not in CONF.magnum:
             raise Exception('config missing nic_id key')
+        cls.nic_id = CONF.magnum.nic_id
 
     @classmethod
     def set_keypair_id(cls, config):
-        cls.keypair_id = config.get('magnum', 'keypair_id')
-        if not config.has_option('magnum', 'keypair_id'):
+        if 'keypair_id' not in CONF.magnum:
             raise Exception('config missing keypair_id key')
+        cls.keypair_id = CONF.magnum.keypair_id
 
     @classmethod
     def setUp(cls):
-        config = ConfigParser.RawConfigParser()
-        if config.read('functional_creds.conf'):
-            cls.set_admin_creds(config)
-            cls.set_user_creds(config)
-            cls.set_auth_version(config)
-            cls.set_auth_url(config)
-
-            # optional magnum bypass url
-            cls.magnum_url = config.get('auth', 'magnum_url')
-
-            cls.set_region(config)
-            cls.set_image_id(config)
-            cls.set_nic_id(config)
-            cls.set_keypair_id(config)
-        else:
-            raise Exception('missing functional_creds.conf file')
+        cls.set_admin_creds(config)
+        cls.set_user_creds(config)
+        cls.set_auth_version(config)
+        cls.set_auth_url(config)
+
+        cls.set_region(config)
+        cls.set_image_id(config)
+        cls.set_nic_id(config)
+        cls.set_keypair_id(config)

magnum/tests/functional/common/datagen.py

@@ -61,7 +61,7 @@ def generate_random_port():
 
 
 def generate_random_docker_volume_size():
-    return random.randrange(1, 100)
+    return random.randrange(1, 3)
 
 
 def generate_fake_ssh_pubkey():

magnum/tests/functional/tempest_tests/config.py

@@ -0,0 +1,46 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+
+from oslo_config import cfg
+
+from tempest import config  # noqa
+
+service_available_group = cfg.OptGroup(name="service_available",
+                                       title="Available OpenStack Services")
+
+ServiceAvailableGroup = [
+    cfg.BoolOpt("magnum",
+                default=True,
+                help="Whether or not magnum is expected to be available"),
+]
+
+magnum_group = cfg.OptGroup(name="magnum", title="Magnum Options")
+
+MagnumGroup = [
+    cfg.StrOpt("image_id",
+               default="fedora-21-atomic-5",
+               help="Image id to be used for baymodel."),
+
+    cfg.StrOpt("nic_id",
+               default="public",
+               help="NIC id."),
+
+    cfg.StrOpt("keypair_id",
+               default="default",
+               help="Keypair id to use to log into nova instances."),
+
+    cfg.StrOpt("flavor_id",
+               default="m1.magnum",
+               help="Flavor id to use for baymodels."),
+]

magnum/tests/functional/tempest_tests/plugin.py

@@ -0,0 +1,39 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import os
+
+from tempest import config
+from tempest.test_discover import plugins
+
+import magnum
+from magnum.tests.functional.tempest_tests import config as magnum_config
+
+
+class MagnumTempestPlugin(plugins.TempestPlugin):
+    def load_tests(self):
+        base_path = os.path.split(os.path.dirname(
+            os.path.abspath(magnum.__file__)))[0]
+        test_dir = "magnum/tests/functional/api/v1"
+        full_test_dir = os.path.join(base_path, test_dir)
+        return full_test_dir, base_path
+
+    def register_opts(self, conf):
+        config.register_opt_group(
+            conf, magnum_config.service_available_group,
+            magnum_config.ServiceAvailableGroup)
+        config.register_opt_group(conf, magnum_config.magnum_group,
+                                  magnum_config.MagnumGroup)
+
+    def get_opt_lists(self):
+        return [(magnum_config.magnum_group.name, magnum_config.MagnumGroup)]

setup.cfg

@@ -65,5 +65,7 @@ magnum.cert_manager.backend =
     barbican = magnum.common.cert_manager.barbican_cert_manager
     local = magnum.common.cert_manager.local_cert_manager
 
+tempest.test_plugins =
+    magnum_tests = magnum.tests.functional.tempest_tests.plugin:MagnumTempestPlugin
 [wheel]
 universal = 1