From 63de8e55404ed6b089607a496e5151402409d50c Mon Sep 17 00:00:00 2001 From: dimtruck Date: Thu, 19 Nov 2015 00:05:03 +0900 Subject: [PATCH] Migration to utilize tempest plugin This is a migration step to utilize tempest plugin system instead of directly calling functional api tests. This is the approach used by a number of other projects as well as an approved process by openstack-qa. The difference in execution is that we will need to execute tempest's tox instead of our own: tox -eall-plugin magnum.tests.functional.api.v1 -- --concurrency=1 Implements: blueprint magnum-tempest Change-Id: Ic3eadae7fb5d88b776f9ded9589ef25279a2e1be --- doc/source/dev/dev-functional-test.rst | 18 +- magnum/tests/contrib/post_test_hook.sh | 157 ++++++++++++------ magnum/tests/functional/common/config.py | 69 ++++---- magnum/tests/functional/common/datagen.py | 2 +- .../functional/tempest_tests/__init__.py | 0 .../tests/functional/tempest_tests/config.py | 46 +++++ .../tests/functional/tempest_tests/plugin.py | 39 +++++ setup.cfg | 2 + 8 files changed, 239 insertions(+), 94 deletions(-) create mode 100644 magnum/tests/functional/tempest_tests/__init__.py create mode 100644 magnum/tests/functional/tempest_tests/config.py create mode 100644 magnum/tests/functional/tempest_tests/plugin.py diff --git a/doc/source/dev/dev-functional-test.rst b/doc/source/dev/dev-functional-test.rst index c2008e7aec..2a700cc393 100644 --- a/doc/source/dev/dev-functional-test.rst +++ b/doc/source/dev/dev-functional-test.rst @@ -61,15 +61,23 @@ Use follow command lines to check what functional testing is supported:: cd /opt/stack/magnum cat tox.ini | grep functional- | awk -F: '{print $2}' | sed s/]// -To do some specify functional testing, for example, test api functional +To do some specify functional testing, for example, test kubernetes functional cases:: - tox -e functional-api -- --concurrency=1 + tox -e functional-k8s -- --concurrency=1 -Test specified case of api functional cases:: +Test specified case of kubernetes functional cases:: + + tox -e functional-k8s -- --concurrency=1 - tox -e functional-api -- --concurrency=1 The following is an example for test path:: - magnum.tests.functional.api.v1.test_baymodel.BayModelTest.test_create_baymodel + magnum.tests.functional.k8s.v1.test_k8s_python_client.TestBayModelResource.test_baymodel_create_and_delete + +We also have api tests being added. To run those, go to +tempest directory (tempest is installed with devstack by default), install +magnum, and execute tempest tox:: + + cd /opt/stack/tempest + tox -eall-plugin magnum.tests.functional.api.v1 -- --concurrency=1 diff --git a/magnum/tests/contrib/post_test_hook.sh b/magnum/tests/contrib/post_test_hook.sh index 04ece299ad..eca4c67b03 100755 --- a/magnum/tests/contrib/post_test_hook.sh +++ b/magnum/tests/contrib/post_test_hook.sh @@ -22,46 +22,17 @@ function function_exists { declare -f -F $1 > /dev/null } -if ! function_exists echo_summary; then - function echo_summary { - echo $@ - } -fi +# Set up all necessary test data +function create_test_data { + # First we test Magnum's command line to see if we can stand up + # a baymodel, bay and a pod -# Save trace setting -XTRACE=$(set +o | grep xtrace) -set -o xtrace + export NIC_ID=$(neutron net-show public | awk '/ id /{print $4}') + export IMAGE_ID=$(glance --os-image-api-version 1 image-show fedora-21-atomic-5 | awk '/ id /{print $4}') -echo_summary "magnum's post_test_hook.sh was called..." -(set -o posix; set) - -constraints="-c $REQUIREMENTS_DIR/upper-constraints.txt" -sudo pip install $constraints -U -r requirements.txt -r test-requirements.txt - -export MAGNUM_DIR="$BASE/new/magnum" -sudo chown -R jenkins:stack $MAGNUM_DIR - -# Get admin credentials -pushd ../devstack -source openrc admin admin -# NOTE(hongbin): This is a temporary work around. These variables are for -# keystone v3, but magnum is using v2 API. Therefore, unset them to make the -# keystoneclient work. -# Bug: #1473600 -unset OS_PROJECT_DOMAIN_ID -unset OS_USER_DOMAIN_ID -unset OS_AUTH_TYPE -popd - -# First we test Magnum's command line to see if we can stand up -# a baymodel, bay and a pod -export NIC_ID=$(neutron net-show public | awk '/ id /{print $4}') -export IMAGE_ID=$(glance --os-image-api-version 1 image-show fedora-21-atomic-5 | awk '/ id /{print $4}') - - -# pass the appropriate variables via a config file -CREDS_FILE=$MAGNUM_DIR/functional_creds.conf -cat < $CREDS_FILE + # pass the appropriate variables via a config file + CREDS_FILE=$MAGNUM_DIR/functional_creds.conf + cat < $CREDS_FILE # Credentials for functional testing [auth] @@ -84,25 +55,113 @@ flavor_id = m1.magnum copy_logs = true EOF -# Note(eliqiao): Let's keep this only for debugging on gate. -echo_summary $CREDS_FILE -cat $CREDS_FILE + # Note(eliqiao): Let's keep this only for debugging on gate. + echo_summary $CREDS_FILE + cat $CREDS_FILE -# Create a keypair for use in the functional tests. -echo_summary "Generate a key-pair" -ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -nova keypair-add --pub-key ~/.ssh/id_rsa.pub default + # Create a keypair for use in the functional tests. + echo_summary "Generate a key-pair" + ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa + nova keypair-add --pub-key ~/.ssh/id_rsa.pub default -# Create magnum specific flavor for use in functional tests. -echo_summary "Create a flavor" -nova flavor-create m1.magnum 100 1024 8 1 +} + +function add_flavor { + # because of policy.json change in nova, flavor-create is now an admin-only feature + # moving this out to only be used by admins + + # Get admin credentials + pushd ../devstack + source openrc admin admin + # NOTE(hongbin): This is a temporary work around. These variables are for + # keystone v3, but magnum is using v2 API. Therefore, unset them to make the + # keystoneclient work. + # Bug: #1473600 + unset OS_PROJECT_DOMAIN_ID + unset OS_USER_DOMAIN_ID + unset OS_AUTH_TYPE + popd + + # Create magnum specific flavor for use in functional tests. + echo_summary "Create a flavor" + nova flavor-create m1.magnum 100 1024 8 1 +} + +if ! function_exists echo_summary; then + function echo_summary { + echo $@ + } +fi + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set -o xtrace + +echo_summary "magnum's post_test_hook.sh was called..." +(set -o posix; set) + +constraints="-c $REQUIREMENTS_DIR/upper-constraints.txt" +sudo pip install $constraints -U -r requirements.txt -r test-requirements.txt + +export MAGNUM_DIR="$BASE/new/magnum" +sudo chown -R jenkins:stack $MAGNUM_DIR # Run functional tests # Currently we support functional-api, functional-k8s, will support swarm, # mesos later. echo "Running magnum functional test suite for $1" -sudo -E -H -u jenkins tox -e functional-$1 -- --concurrency=1 + +# For api, we will run tempest tests +if [[ "api" == $1 ]]; then + # Import devstack functions 'iniset', 'iniget' and 'trueorfalse' + source $BASE/new/devstack/functions + echo "TEMPEST_SERVICES+=,magnum" >> $localrc_path + pushd $BASE/new/tempest + sudo chown -R jenkins:stack $BASE/new/tempest + + add_flavor + + # Set demo credentials + source $BASE/new/devstack/accrc/demo/demo + unset OS_AUTH_TYPE + + create_test_data + + # Set up tempest config with magnum goodness + iniset $BASE/new/tempest/etc/tempest.conf magnum image_id $IMAGE_ID + iniset $BASE/new/tempest/etc/tempest.conf magnum nic_id $NIC_ID + iniset $BASE/new/tempest/etc/tempest.conf magnum keypair_id default + iniset $BASE/new/tempest/etc/tempest.conf magnum flavor_id m1.magnum + + # show tempest config with magnum + cat etc/tempest.conf + + # Set up concurrency and test regex + export MAGNUM_TEMPEST_CONCURRENCY=${MAGNUM_TEMPEST_CONCURRENCY:-1} + export MAGNUM_TESTS=${MAGNUM_TESTS:-'magnum.tests.functional.api.v1'} + + echo "Running tempest magnum test suites" + sudo -H -u jenkins tox -eall-plugin -- $MAGNUM_TESTS --concurrency=$MAGNUM_TEMPEST_CONCURRENCY +else + # Get admin credentials + pushd ../devstack + source openrc admin admin + # NOTE(hongbin): This is a temporary work around. These variables are for + # keystone v3, but magnum is using v2 API. Therefore, unset them to make the + # keystoneclient work. + # Bug: #1473600 + unset OS_PROJECT_DOMAIN_ID + unset OS_USER_DOMAIN_ID + unset OS_AUTH_TYPE + popd + + add_flavor + + create_test_data + + sudo -E -H -u jenkins tox -e functional-$1 -- --concurrency=1 +fi EXIT_CODE=$? # Delete the keypair used in the functional test. diff --git a/magnum/tests/functional/common/config.py b/magnum/tests/functional/common/config.py index 2cce0d533d..22b36a9738 100644 --- a/magnum/tests/functional/common/config.py +++ b/magnum/tests/functional/common/config.py @@ -10,7 +10,10 @@ # License for the specific language governing permissions and limitations # under the License. -import ConfigParser +from tempest import config + + +CONF = config.CONF class Config(object): @@ -19,77 +22,65 @@ class Config(object): @classmethod def set_admin_creds(cls, config): - cls.admin_user = config.get('admin', 'user') - cls.admin_passwd = config.get('admin', 'pass') - cls.admin_tenant = config.get('admin', 'tenant') + cls.admin_user = CONF.auth.admin_username + cls.admin_passwd = CONF.auth.admin_password + cls.admin_tenant = CONF.auth.admin_tenant_name @classmethod def set_user_creds(cls, config): # normal user creds - cls.user = config.get('auth', 'username') - cls.passwd = config.get('auth', 'password') - cls.tenant = config.get('auth', 'tenant_name') + cls.user = CONF.identity.username + cls.passwd = CONF.identity.password + cls.tenant = CONF.identity.tenant_name @classmethod def set_auth_version(cls, config): # auth version for client authentication - if config.has_option('auth', 'auth_version'): - cls.auth_version = config.get('auth', 'auth_version') - else: - cls.auth_version = 'v3' + cls.auth_version = CONF.identity.auth_version @classmethod def set_auth_url(cls, config): # auth_url for client authentication if cls.auth_version == 'v3': - if not config.has_option('auth', 'auth_v3_url'): - raise Exception('config missing auth_v3_url key') - cls.auth_v3_url = config.get('auth', 'auth_v3_url') + cls.auth_v3_url = CONF.identity.uri_v3 else: - if not config.has_option('auth', 'auth_url'): + if 'uri' not in CONF.identity: raise Exception('config missing auth_url key') - cls.auth_url = config.get('auth', 'auth_url') + cls.auth_url = CONF.identity.uri @classmethod def set_region(cls, config): - if config.has_option('auth', 'region'): - cls.region = config.get('auth', 'region') + if 'region' in CONF.identity: + cls.region = CONF.identity.region else: cls.region = 'RegionOne' @classmethod def set_image_id(cls, config): - cls.image_id = config.get('magnum', 'image_id') - if not config.has_option('magnum', 'image_id'): + if 'image_id' not in CONF.magnum: raise Exception('config missing image_id key') + cls.image_id = CONF.magnum.image_id @classmethod def set_nic_id(cls, config): - cls.nic_id = config.get('magnum', 'nic_id') - if not config.has_option('magnum', 'nic_id'): + if 'nic_id' not in CONF.magnum: raise Exception('config missing nic_id key') + cls.nic_id = CONF.magnum.nic_id @classmethod def set_keypair_id(cls, config): - cls.keypair_id = config.get('magnum', 'keypair_id') - if not config.has_option('magnum', 'keypair_id'): + if 'keypair_id' not in CONF.magnum: raise Exception('config missing keypair_id key') + cls.keypair_id = CONF.magnum.keypair_id @classmethod def setUp(cls): - config = ConfigParser.RawConfigParser() - if config.read('functional_creds.conf'): - cls.set_admin_creds(config) - cls.set_user_creds(config) - cls.set_auth_version(config) - cls.set_auth_url(config) + cls.set_admin_creds(config) + cls.set_user_creds(config) + cls.set_auth_version(config) + cls.set_auth_url(config) - # optional magnum bypass url - cls.magnum_url = config.get('auth', 'magnum_url') - - cls.set_region(config) - cls.set_image_id(config) - cls.set_nic_id(config) - cls.set_keypair_id(config) - else: - raise Exception('missing functional_creds.conf file') + cls.set_region(config) + cls.set_image_id(config) + cls.set_nic_id(config) + cls.set_keypair_id(config) diff --git a/magnum/tests/functional/common/datagen.py b/magnum/tests/functional/common/datagen.py index 700bc1323f..f110d7070d 100644 --- a/magnum/tests/functional/common/datagen.py +++ b/magnum/tests/functional/common/datagen.py @@ -61,7 +61,7 @@ def generate_random_port(): def generate_random_docker_volume_size(): - return random.randrange(1, 100) + return random.randrange(1, 3) def generate_fake_ssh_pubkey(): diff --git a/magnum/tests/functional/tempest_tests/__init__.py b/magnum/tests/functional/tempest_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/magnum/tests/functional/tempest_tests/config.py b/magnum/tests/functional/tempest_tests/config.py new file mode 100644 index 0000000000..ac8d268ac2 --- /dev/null +++ b/magnum/tests/functional/tempest_tests/config.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +from oslo_config import cfg + +from tempest import config # noqa + +service_available_group = cfg.OptGroup(name="service_available", + title="Available OpenStack Services") + +ServiceAvailableGroup = [ + cfg.BoolOpt("magnum", + default=True, + help="Whether or not magnum is expected to be available"), +] + +magnum_group = cfg.OptGroup(name="magnum", title="Magnum Options") + +MagnumGroup = [ + cfg.StrOpt("image_id", + default="fedora-21-atomic-5", + help="Image id to be used for baymodel."), + + cfg.StrOpt("nic_id", + default="public", + help="NIC id."), + + cfg.StrOpt("keypair_id", + default="default", + help="Keypair id to use to log into nova instances."), + + cfg.StrOpt("flavor_id", + default="m1.magnum", + help="Flavor id to use for baymodels."), +] diff --git a/magnum/tests/functional/tempest_tests/plugin.py b/magnum/tests/functional/tempest_tests/plugin.py new file mode 100644 index 0000000000..75d568c0ef --- /dev/null +++ b/magnum/tests/functional/tempest_tests/plugin.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os + +from tempest import config +from tempest.test_discover import plugins + +import magnum +from magnum.tests.functional.tempest_tests import config as magnum_config + + +class MagnumTempestPlugin(plugins.TempestPlugin): + def load_tests(self): + base_path = os.path.split(os.path.dirname( + os.path.abspath(magnum.__file__)))[0] + test_dir = "magnum/tests/functional/api/v1" + full_test_dir = os.path.join(base_path, test_dir) + return full_test_dir, base_path + + def register_opts(self, conf): + config.register_opt_group( + conf, magnum_config.service_available_group, + magnum_config.ServiceAvailableGroup) + config.register_opt_group(conf, magnum_config.magnum_group, + magnum_config.MagnumGroup) + + def get_opt_lists(self): + return [(magnum_config.magnum_group.name, magnum_config.MagnumGroup)] diff --git a/setup.cfg b/setup.cfg index e7df5753da..594e1f9871 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,5 +65,7 @@ magnum.cert_manager.backend = barbican = magnum.common.cert_manager.barbican_cert_manager local = magnum.common.cert_manager.local_cert_manager +tempest.test_plugins = + magnum_tests = magnum.tests.functional.tempest_tests.plugin:MagnumTempestPlugin [wheel] universal = 1