Remove Whitebox tests

This commit removes the whitebox tests from tempest. The proper place
for this type of testing is in nova unit tests not in tempest.

Fixes bug 1222852

Change-Id: I0fac7b030c51985f9d6d93129bf9bab75c18cd11
This commit is contained in:
Matthew Treinish 2013-09-11 14:19:46 -04:00
parent bfcc71228d
commit de8378b65b
12 changed files with 1 additions and 610 deletions

View File

@ -1 +0,0 @@
../../../tempest/whitebox/README.rst

View File

@ -31,7 +31,6 @@ where your test contributions should go.
field_guide/scenario
field_guide/stress
field_guide/thirdparty
field_guide/whitebox
field_guide/unit_tests
------------------

View File

@ -161,29 +161,6 @@ flavor_extra_enabled = true
# Expected first device name when a volume is attached to an instance
volume_device_name = vdb
[whitebox]
# Whitebox options for compute. Whitebox options enable the
# whitebox test cases, which look at internal Nova database state,
# SSH into VMs to check instance state, etc.
# Should we run whitebox tests for Compute?
whitebox_enabled = true
# Path of nova source directory
source_dir = /opt/stack/nova
# Path of nova configuration file
config_path = /etc/nova/nova.conf
# Directory containing nova binaries such as nova-manage
bin_dir = /usr/local/bin
# Connection string to the database of Compute service
db_uri = mysql://nova:secret@localhost/nova
# Path to a private key file for SSH access to remote hosts
path_to_private_key = /home/user/.ssh/id_rsa
[compute-admin]
# This should be the username of a user WITH administrative privileges
# If not defined the admin user from the identity section will be used

View File

@ -18,6 +18,4 @@ testresources>=0.2.4
keyring>=1.6.1
testrepository>=0.0.17
oslo.config>=1.1.0
# Needed for whitebox testing
SQLAlchemy>=0.7.8,<=0.7.99
eventlet>=0.13.0

View File

@ -10,7 +10,6 @@ function usage {
echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added."
echo " -u, --update Update the virtual environment with any newer package versions"
echo " -s, --smoke Only run smoke tests"
echo " -w, --whitebox Only run whitebox tests"
echo " -t, --serial Run testr serially"
echo " -c, --nova-coverage Enable Nova coverage collection"
echo " -C, --config Config file location"
@ -38,7 +37,7 @@ update=0
logging=0
logging_config=etc/logging.conf
if ! options=$(getopt -o VNnfuswtcphdC:lL: -l virtual-env,no-virtual-env,no-site-packages,force,update,smoke,whitebox,serial,nova-coverage,pep8,help,debug,config:,logging,logging-config: -- "$@")
if ! options=$(getopt -o VNnfustcphdC:lL: -l virtual-env,no-virtual-env,no-site-packages,force,update,smoke,serial,nova-coverage,pep8,help,debug,config:,logging,logging-config: -- "$@")
then
# parse error
usage
@ -60,7 +59,6 @@ while [ $# -gt 0 ]; do
-C|--config) config_file=$2; shift;;
-p|--pep8) let just_pep8=1;;
-s|--smoke) testrargs="$testrargs smoke";;
-w|--whitebox) testrargs="$testrargs whitebox";;
-t|--serial) serial=1;;
-l|--logging) logging=1;;
-L|--logging-config) logging_config=$2; shift;;

View File

@ -18,7 +18,6 @@ to make this clear.
| scenario/ - complex scenario tests
| stress/ - stress tests
| thirdparty/ - 3rd party api tests
| whitebox/ - white box testing
Each of these directories contains different types of tests. What
belongs in each directory, the rules and examples for good tests, are
@ -78,11 +77,3 @@ Many openstack components include 3rdparty API support. It is
completely legitimate for Tempest to include tests of 3rdparty APIs,
but those should be kept separate from the normal OpenStack
validation.
whitebox
--------
Whitebox tests are tests which require access to the database of the
target OpenStack machine to verify internal state after operations
are made. White box tests are allowed to use the python clients.

View File

@ -238,36 +238,6 @@ def register_compute_admin_opts(conf):
for opt in ComputeAdminGroup:
conf.register_opt(opt, group='compute-admin')
whitebox_group = cfg.OptGroup(name='whitebox',
title="Whitebox Options")
WhiteboxGroup = [
cfg.BoolOpt('whitebox_enabled',
default=False,
help="Does the test environment support whitebox tests for "
"Compute?"),
cfg.StrOpt('db_uri',
default=None,
help="Connection string to the database of Compute service"),
cfg.StrOpt('source_dir',
default="/opt/stack/nova",
help="Path of nova source directory"),
cfg.StrOpt('config_path',
default='/etc/nova/nova.conf',
help="Path of nova configuration file"),
cfg.StrOpt('bin_dir',
default="/usr/local/bin/",
help="Directory containing nova binaries such as nova-manage"),
]
def register_whitebox_opts(conf):
conf.register_group(whitebox_group)
for opt in WhiteboxGroup:
conf.register_opt(opt, group='whitebox')
image_group = cfg.OptGroup(name='image',
title="Image Service Options")
@ -650,7 +620,6 @@ class TempestConfig:
register_compute_opts(cfg.CONF)
register_identity_opts(cfg.CONF)
register_whitebox_opts(cfg.CONF)
register_image_opts(cfg.CONF)
register_network_opts(cfg.CONF)
register_volume_opts(cfg.CONF)
@ -663,7 +632,6 @@ class TempestConfig:
register_scenario_opts(cfg.CONF)
register_service_available_opts(cfg.CONF)
self.compute = cfg.CONF.compute
self.whitebox = cfg.CONF.whitebox
self.identity = cfg.CONF.identity
self.images = cfg.CONF.image
self.network = cfg.CONF.network

View File

@ -1,46 +0,0 @@
Tempest Guide to Whitebox tests
===============================
What are these tests?
---------------------
When you hit the OpenStack API, this causes internal state changes in
the system. This might be database transitions, vm modifications,
other deep state changes which aren't really accessible from the
OpenStack API. These side effects are sometimes important to
validate.
White box testing is an approach there. In white box testing you are
given database access to the environment, and can verify internal
record changes after an API call.
This is an optional part of testing, and requires extra setup, but can
be useful for validating Tempest internals.
Why are these tests in tempest?
-------------------------------
Especially when it comes to something like VM state changing, which is
a coordination of numerous running daemons, and a functioning VM, it's
very difficult to get a realistic test like this in unit tests.
Scope of these tests
--------------------
White box tests should be limitted to tests where black box testing
(using the OpenStack API to verify results) isn't sufficient.
As these poke at internals of OpenStack, it should also be realized
that these tests are very tightly coupled to current implementation of
OpenStack. They will need to be maintained agressively to keep up with
internals changes in OpenStack projects.
Example of a good test
----------------------
Pushing VMs through a series of state transitions, and ensuring along
the way the database state transitions match what's expected.

View File

@ -1,146 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shlex
import subprocess
import sys
from sqlalchemy import create_engine, MetaData
from tempest.common.ssh import Client
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.openstack.common import log as logging
from tempest.scenario import manager
LOG = logging.getLogger(__name__)
class WhiteboxTest(object):
"""
Base test case class mixin for "whitebox tests"
Whitebox tests are tests that have the following characteristics:
* Test common and advanced operations against a set of servers
* Use a client that it is possible to send random or bad data with
* SSH into either a host or a guest in order to validate server state
* May execute SQL queries directly against internal databases to verify
the state of data records
"""
pass
class ComputeWhiteboxTest(manager.OfficialClientTest):
"""
Base smoke test case class for OpenStack Compute API (Nova)
"""
@classmethod
def setUpClass(cls):
super(ComputeWhiteboxTest, cls).setUpClass()
if not cls.config.whitebox.whitebox_enabled:
msg = "Whitebox testing disabled"
raise cls.skipException(msg)
# Add some convenience attributes that tests use...
cls.nova_dir = cls.config.whitebox.source_dir
cls.compute_bin_dir = cls.config.whitebox.bin_dir
cls.compute_config_path = cls.config.whitebox.config_path
cls.build_interval = cls.config.compute.build_interval
cls.build_timeout = cls.config.compute.build_timeout
cls.ssh_user = cls.config.compute.ssh_user
cls.image_ref = cls.config.compute.image_ref
cls.image_ref_alt = cls.config.compute.image_ref_alt
cls.flavor_ref = cls.config.compute.flavor_ref
cls.flavor_ref_alt = cls.config.compute.flavor_ref_alt
# NOTE(afazekas): Mimics the helper method used in the api tests
@classmethod
def create_server(cls, **kwargs):
flavor_ref = cls.config.compute.flavor_ref
image_ref = cls.config.compute.image_ref
name = rand_name(cls.__name__ + "-instance")
if 'name' in kwargs:
name = kwargs.pop('name')
flavor = kwargs.get('flavor', flavor_ref)
image_id = kwargs.get('image_id', image_ref)
server = cls.compute_client.servers.create(
name, image_id, flavor, **kwargs)
if 'wait_until' in kwargs:
cls.status_timeout(cls.compute_client.servers, server.id,
server['id'], kwargs['wait_until'])
server = cls.compute_client.servers.get(server.id)
cls.set_resource(name, server)
return server
@classmethod
def get_db_handle_and_meta(cls, database='nova'):
"""Return a connection handle and metadata of an OpenStack database."""
engine_args = {"echo": False,
"convert_unicode": True,
"pool_recycle": 3600
}
try:
engine = create_engine(cls.config.whitebox.db_uri, **engine_args)
connection = engine.connect()
meta = MetaData()
meta.reflect(bind=engine)
except Exception as e:
raise exceptions.SQLException(message=e)
return connection, meta
def nova_manage(self, category, action, params):
"""Executes nova-manage command for the given action."""
nova_manage_path = os.path.join(self.compute_bin_dir, 'nova-manage')
cmd = ' '.join([nova_manage_path, category, action, params])
if self.deploy_mode == 'devstack-local':
if not os.path.isdir(self.nova_dir):
sys.exit("Cannot find Nova source directory: %s" %
self.nova_dir)
cmd = shlex.split(cmd)
result = subprocess.Popen(cmd, stdout=subprocess.PIPE)
# TODO(rohitk): Need to define host connection parameters in config
else:
client = self.get_ssh_connection(self.config.whitebox.api_host,
self.config.whitebox.api_user,
self.config.whitebox.api_passwd)
result = client.exec_command(cmd)
return result
def get_ssh_connection(self, host, username, password):
"""Create an SSH connection object to a host."""
ssh_timeout = self.config.compute.ssh_timeout
ssh_client = Client(host, username, password, ssh_timeout)
if not ssh_client.test_connection_auth():
raise exceptions.SSHTimeout()
else:
return ssh_client

View File

@ -1,163 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from tempest.common.utils.data_utils import rand_name
from tempest.openstack.common import log as logging
from tempest.whitebox import manager
from novaclient import exceptions
LOG = logging.getLogger(__name__)
class ImagesWhiteboxTest(manager.ComputeWhiteboxTest):
_interface = 'json'
@classmethod
def setUpClass(cls):
super(ImagesWhiteboxTest, cls).setUpClass()
cls.create_image = cls.compute_client.servers.create_image
cls.connection, cls.meta = cls.get_db_handle_and_meta()
cls.shared_server = cls.create_server()
cls.image_ids = []
@classmethod
def tearDownClass(cls):
"""Delete images and server after a test is executed."""
for image_id in cls.image_ids:
cls.client.delete_image(image_id)
cls.image_ids.remove(image_id)
super(ImagesWhiteboxTest, cls).tearDownClass()
@classmethod
def update_state(self, server_id, vm_state, task_state, deleted=0):
"""Update states of an instance in database for validation."""
if not task_state:
task_state = "NULL"
instances = self.meta.tables['instances']
stmt = instances.update().where(instances.c.uuid == server_id).values(
deleted=deleted,
vm_state=vm_state,
task_state=task_state)
self.connection.execute(stmt, autocommit=True)
def _test_create_image_409_base(self, vm_state, task_state, deleted=0):
"""Base method for create image tests based on vm and task states."""
try:
self.update_state(self.shared_server.id, vm_state,
task_state, deleted)
image_name = rand_name('snap-')
self.assertRaises(exceptions.Conflict,
self.create_image,
self.shared_server.id, image_name)
except Exception:
LOG.error("Should not allow create image when vm_state=%s and "
"task_state=%s" % (vm_state, task_state))
raise
finally:
self.update_state(self.shared_server.id, 'active', None)
def test_create_image_when_vm_eq_building_task_eq_scheduling(self):
# 409 error when instance states are building,scheduling
self._test_create_image_409_base("building", "scheduling")
def test_create_image_when_vm_eq_building_task_eq_networking(self):
# 409 error when instance states are building,networking
self._test_create_image_409_base("building", "networking")
def test_create_image_when_vm_eq_building_task_eq_bdm(self):
# 409 error when instance states are building,block_device_mapping
self._test_create_image_409_base("building", "block_device_mapping")
def test_create_image_when_vm_eq_building_task_eq_spawning(self):
# 409 error when instance states are building,spawning
self._test_create_image_409_base("building", "spawning")
def test_create_image_when_vm_eq_active_task_eq_image_backup(self):
# 409 error when instance states are active,image_backup
self._test_create_image_409_base("active", "image_backup")
def test_create_image_when_vm_eq_resized_task_eq_resize_prep(self):
# 409 error when instance states are resized,resize_prep
self._test_create_image_409_base("resized", "resize_prep")
def test_create_image_when_vm_eq_resized_task_eq_resize_migrating(self):
# 409 error when instance states are resized,resize_migrating
self._test_create_image_409_base("resized", "resize_migrating")
def test_create_image_when_vm_eq_resized_task_eq_resize_migrated(self):
# 409 error when instance states are resized,resize_migrated
self._test_create_image_409_base("resized", "resize_migrated")
def test_create_image_when_vm_eq_resized_task_eq_resize_finish(self):
# 409 error when instance states are resized,resize_finish
self._test_create_image_409_base("resized", "resize_finish")
def test_create_image_when_vm_eq_resized_task_eq_resize_reverting(self):
# 409 error when instance states are resized,resize_reverting
self._test_create_image_409_base("resized", "resize_reverting")
def test_create_image_when_vm_eq_resized_task_eq_resize_confirming(self):
# 409 error when instance states are resized,resize_confirming
self._test_create_image_409_base("resized", "resize_confirming")
def test_create_image_when_vm_eq_active_task_eq_resize_verify(self):
# 409 error when instance states are active,resize_verify
self._test_create_image_409_base("active", "resize_verify")
def test_create_image_when_vm_eq_active_task_eq_updating_password(self):
# 409 error when instance states are active,updating_password
self._test_create_image_409_base("active", "updating_password")
def test_create_image_when_vm_eq_active_task_eq_rebuilding(self):
# 409 error when instance states are active,rebuilding
self._test_create_image_409_base("active", "rebuilding")
def test_create_image_when_vm_eq_active_task_eq_rebooting(self):
# 409 error when instance states are active,rebooting
self._test_create_image_409_base("active", "rebooting")
def test_create_image_when_vm_eq_building_task_eq_deleting(self):
# 409 error when instance states are building,deleting
self._test_create_image_409_base("building", "deleting")
def test_create_image_when_vm_eq_active_task_eq_deleting(self):
# 409 error when instance states are active,deleting
self._test_create_image_409_base("active", "deleting")
def test_create_image_when_vm_eq_error_task_eq_building(self):
# 409 error when instance states are error,building
self._test_create_image_409_base("error", "building")
def test_create_image_when_vm_eq_error_task_eq_none(self):
# 409 error when instance states are error,None
self._test_create_image_409_base("error", None)
def test_create_image_when_vm_eq_deleted_task_eq_none(self):
# 409 error when instance states are deleted,None
self._test_create_image_409_base("deleted", None)
def test_create_image_when_vm_eq_resized_task_eq_none(self):
# 409 error when instance states are resized,None
self._test_create_image_409_base("resized", None)
def test_create_image_when_vm_eq_error_task_eq_resize_prep(self):
# 409 error when instance states are error,resize_prep
self._test_create_image_409_base("error", "resize_prep")

View File

@ -1,184 +0,0 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from tempest.api.identity.base import BaseIdentityAdminTest
from tempest import exceptions
from tempest.openstack.common import log as logging
from tempest.whitebox import manager
LOG = logging.getLogger(__name__)
class ServersWhiteboxTest(manager.ComputeWhiteboxTest):
_interface = 'json'
@classmethod
def setUpClass(cls):
super(ServersWhiteboxTest, cls).setUpClass()
# NOTE(afazekas): Strange relationship
BaseIdentityAdminTest.setUpClass()
cls.client = cls.servers_client
cls.img_client = cls.images_client
cls.admin_client = BaseIdentityAdminTest.client
cls.connection, cls.meta = cls.get_db_handle_and_meta()
resp, tenants = cls.admin_client.list_tenants()
cls.tenant_id = [
tnt['id']
for tnt in tenants if tnt['name'] == cls.config.compute.tenant_name
][0]
cls.shared_server = cls.create_server()
def tearDown(cls):
for server in cls.servers:
try:
cls.client.delete_server(server['id'])
except exceptions.NotFound:
continue
def update_state(self, server_id, vm_state, task_state, deleted=0):
"""Update states of an instance in database for validation."""
if not task_state:
task_state = 'NULL'
instances = self.meta.tables['instances']
stmt = instances.update().where(instances.c.uuid == server_id).values(
deleted=deleted,
vm_state=vm_state,
task_state=task_state)
self.connection.execute(stmt, autocommit=True)
def _test_delete_server_base(self, vm_state, task_state):
"""
Base method for delete server tests based on vm and task states.
Validates for successful server termination.
"""
server = self.create_server()
self.update_state(server['id'], vm_state, task_state)
resp, body = self.client.delete_server(server['id'])
self.assertEqual('204', resp['status'])
self.client.wait_for_server_termination(server['id'],
ignore_error=True)
instances = self.meta.tables['instances']
stmt = instances.select().where(instances.c.uuid == server['id'])
result = self.connection.execute(stmt).first()
self.assertEqual(True, result.deleted > 0)
self.assertEqual('deleted', result.vm_state)
self.assertEqual(None, result.task_state)
def _test_delete_server_403_base(self, vm_state, task_state):
"""
Base method for delete server tests based on vm and task states.
Validates for 403 error code.
"""
try:
self.update_state(self.shared_server['id'], vm_state, task_state)
self.assertRaises(exceptions.Unauthorized,
self.client.delete_server,
self.shared_server['id'])
except Exception:
LOG.error("Should not allow delete server when vm_state=%s and "
"task_state=%s" % (vm_state, task_state))
raise
finally:
self.update_state(self.shared_server['id'], 'active', None)
def test_delete_server_when_vm_eq_building_task_eq_networking(self):
# Delete server when instance states are building,networking
self._test_delete_server_base('building', 'networking')
def test_delete_server_when_vm_eq_building_task_eq_bdm(self):
# Delete server when instance states are building,block device mapping
self._test_delete_server_base('building', 'block_device_mapping')
def test_delete_server_when_vm_eq_building_task_eq_spawning(self):
# Delete server when instance states are building,spawning
self._test_delete_server_base('building', 'spawning')
def test_delete_server_when_vm_eq_active_task_eq_image_backup(self):
# Delete server when instance states are active,image_backup
self._test_delete_server_base('active', 'image_backup')
def test_delete_server_when_vm_eq_active_task_eq_rebuilding(self):
# Delete server when instance states are active,rebuilding
self._test_delete_server_base('active', 'rebuilding')
def test_delete_server_when_vm_eq_error_task_eq_spawning(self):
# Delete server when instance states are error,spawning
self._test_delete_server_base('error', 'spawning')
def test_delete_server_when_vm_eq_resized_task_eq_resize_prep(self):
# Delete server when instance states are resized,resize_prep
self._test_delete_server_403_base('resized', 'resize_prep')
def test_delete_server_when_vm_eq_resized_task_eq_resize_migrating(self):
# Delete server when instance states are resized,resize_migrating
self._test_delete_server_403_base('resized', 'resize_migrating')
def test_delete_server_when_vm_eq_resized_task_eq_resize_migrated(self):
# Delete server when instance states are resized,resize_migrated
self._test_delete_server_403_base('resized', 'resize_migrated')
def test_delete_server_when_vm_eq_resized_task_eq_resize_finish(self):
# Delete server when instance states are resized,resize_finish
self._test_delete_server_403_base('resized', 'resize_finish')
def test_delete_server_when_vm_eq_resized_task_eq_resize_reverting(self):
# Delete server when instance states are resized,resize_reverting
self._test_delete_server_403_base('resized', 'resize_reverting')
def test_delete_server_when_vm_eq_resized_task_eq_resize_confirming(self):
# Delete server when instance states are resized,resize_confirming
self._test_delete_server_403_base('resized', 'resize_confirming')
def test_delete_server_when_vm_eq_active_task_eq_resize_verify(self):
# Delete server when instance states are active,resize_verify
self._test_delete_server_base('active', 'resize_verify')
def test_delete_server_when_vm_eq_active_task_eq_rebooting(self):
# Delete server when instance states are active,rebooting
self._test_delete_server_base('active', 'rebooting')
def test_delete_server_when_vm_eq_building_task_eq_deleting(self):
# Delete server when instance states are building,deleting
self._test_delete_server_base('building', 'deleting')
def test_delete_server_when_vm_eq_active_task_eq_deleting(self):
# Delete server when instance states are active,deleting
self._test_delete_server_base('active', 'deleting')
def test_delete_server_when_vm_eq_error_task_eq_none(self):
# Delete server when instance states are error,None
self._test_delete_server_base('error', None)
def test_delete_server_when_vm_eq_resized_task_eq_none(self):
# Delete server when instance states are resized,None
self._test_delete_server_403_base('resized', None)
def test_delete_server_when_vm_eq_error_task_eq_resize_prep(self):
# Delete server when instance states are error,resize_prep
self._test_delete_server_base('error', 'resize_prep')
def test_delete_server_when_vm_eq_error_task_eq_error(self):
# Delete server when instance states are error,error
self._test_delete_server_base('error', 'error')