openstack/deb-tempest
Code Issues Proposed changes

break out whitebox tests

Browse Source 
move whitebox tests to their own directory, with rules around
them. We had only a small number of tests here, and we don't
currently run these in gate, so these aren't added to tox.ini yet.

Part of bp:tempest-repo-restructure

Change-Id: Ic1d2158cd92d13ae25f026289f680396aa0ec3a8
This commit is contained in:
Sean Dague
2013-05-14 11:50:46 -04:00
parent 09761f63c8
commit 2203a1f55d
6 changed files with 52 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
tempest/whitebox/README.rst Normal file
View File

@@ -0,0 +1,46 @@
Tempest Guide to Whitebox tests
========
What are these tests?
--------
When you hit the OpenStack API, this causes internal state changes in
the system. This might be database transitions, vm modifications,
other deep state changes which aren't really accessible from the
OpenStack API. These side effects are sometimes important to
validate.
White box testing is an approach there. In white box testing you are
given database access to the environment, and can verify internal
record changes after an API call.
This is an optional part of testing, and requires extra setup, but can
be useful for validating Tempest internals.
Why are these tests in tempest?
--------
Especially when it comes to something like VM state changing, which is
a coordination of numerous running daemons, and a functioning VM, it's
very difficult to get a realistic test like this in unit tests.
Scope of these tests
--------
White box tests should be limitted to tests where black box testing
(using the OpenStack API to verify results) isn't sufficient.
As these poke at internals of OpenStack, it should also be realized
that these tests are very tightly coupled to current implementation of
OpenStack. They will need to be maintained agressively to keep up with
internals changes in OpenStack projects.
Example of a good test
--------
Pushing VMs through a series of state transitions, and ensuring along
the way the database state transitions match what's expected.

0
tempest/whitebox/__init__.py Normal file
View File

166
tempest/whitebox/manager.py Normal file
View File

@@ -0,0 +1,166 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import shlex
import subprocess
import sys
from sqlalchemy import create_engine, MetaData
from tempest.common.ssh import Client
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest import test
LOG = logging.getLogger(__name__)
class WhiteboxTest(object):
"""
Base test case class mixin for "whitebox tests"
Whitebox tests are tests that have the following characteristics:
* Test common and advanced operations against a set of servers
* Use a client that it is possible to send random or bad data with
* SSH into either a host or a guest in order to validate server state
* May execute SQL queries directly against internal databases to verify
the state of data records
"""
pass
class ComputeWhiteboxTest(test.ComputeFuzzClientTest, WhiteboxTest):
"""
Base smoke test case class for OpenStack Compute API (Nova)
"""
@classmethod
def setUpClass(cls):
super(ComputeWhiteboxTest, cls).setUpClass()
if not cls.config.whitebox.whitebox_enabled:
msg = "Whitebox testing disabled"
raise cls.skipException(msg)
# Add some convenience attributes that tests use...
cls.nova_dir = cls.config.whitebox.source_dir
cls.compute_bin_dir = cls.config.whitebox.bin_dir
cls.compute_config_path = cls.config.whitebox.config_path
cls.servers_client = cls.manager.servers_client
cls.images_client = cls.manager.images_client
cls.flavors_client = cls.manager.flavors_client
cls.extensions_client = cls.manager.extensions_client
cls.floating_ips_client = cls.manager.floating_ips_client
cls.keypairs_client = cls.manager.keypairs_client
cls.security_groups_client = cls.manager.security_groups_client
cls.limits_client = cls.manager.limits_client
cls.volumes_client = cls.manager.volumes_client
cls.build_interval = cls.config.compute.build_interval
cls.build_timeout = cls.config.compute.build_timeout
cls.ssh_user = cls.config.compute.ssh_user
cls.image_ref = cls.config.compute.image_ref
cls.image_ref_alt = cls.config.compute.image_ref_alt
cls.flavor_ref = cls.config.compute.flavor_ref
cls.flavor_ref_alt = cls.config.compute.flavor_ref_alt
cls.servers = []
@classmethod
def tearDownClass(cls):
# NOTE(jaypipes): Tests often add things in a particular order
# so we destroy resources in the reverse order in which resources
# are added to the test class object
if not cls.os_resources:
return
thing = cls.os_resources.pop()
while True:
LOG.debug("Deleting %r from shared resources of %s" %
(thing, cls.__name__))
# Resources in novaclient all have a delete() method
# which destroys the resource...
thing.delete()
if not cls.os_resources:
return
thing = cls.os_resources.pop()
@classmethod
def create_server(cls, image_id=None):
"""Wrapper utility that returns a test server."""
server_name = rand_name(cls.__name__ + "-instance")
flavor = cls.flavor_ref
if not image_id:
image_id = cls.image_ref
resp, server = cls.servers_client.create_server(
server_name, image_id, flavor)
cls.servers_client.wait_for_server_status(server['id'], 'ACTIVE')
cls.servers.append(server)
return server
@classmethod
def get_db_handle_and_meta(cls, database='nova'):
"""Return a connection handle and metadata of an OpenStack database."""
engine_args = {"echo": False,
"convert_unicode": True,
"pool_recycle": 3600
}
try:
engine = create_engine(cls.config.whitebox.db_uri, **engine_args)
connection = engine.connect()
meta = MetaData()
meta.reflect(bind=engine)
except Exception, e:
raise exceptions.SQLException(message=e)
return connection, meta
def nova_manage(self, category, action, params):
"""Executes nova-manage command for the given action."""
nova_manage_path = os.path.join(self.compute_bin_dir, 'nova-manage')
cmd = ' '.join([nova_manage_path, category, action, params])
if self.deploy_mode == 'devstack-local':
if not os.path.isdir(self.nova_dir):
sys.exit("Cannot find Nova source directory: %s" %
self.nova_dir)
cmd = shlex.split(cmd)
result = subprocess.Popen(cmd, stdout=subprocess.PIPE)
#Todo(rohitk): Need to define host connection parameters in config
else:
client = self.get_ssh_connection(self.config.whitebox.api_host,
self.config.whitebox.api_user,
self.config.whitebox.api_passwd)
result = client.exec_command(cmd)
return result
def get_ssh_connection(self, host, username, password):
"""Create an SSH connection object to a host."""
ssh_timeout = self.config.compute.ssh_timeout
ssh_client = Client(host, username, password, ssh_timeout)
if not ssh_client.test_connection_auth():
raise exceptions.SSHTimeout()
else:
return ssh_client

161
tempest/whitebox/test_images_whitebox.py Normal file
View File

@@ -0,0 +1,161 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
from tempest.tests.compute import base
from tempest.whitebox import manager
@attr(type='whitebox')
class ImagesWhiteboxTest(manager.ComputeWhiteboxTest, base.BaseComputeTest):
_interface = 'json'
@classmethod
def setUpClass(cls):
super(ImagesWhiteboxTest, cls).setUpClass()
cls.client = cls.images_client
cls.connection, cls.meta = cls.get_db_handle_and_meta()
cls.shared_server = cls.create_server()
cls.image_ids = []
@classmethod
def tearDownClass(cls):
"""Delete images after a test is executed."""
for image_id in cls.image_ids:
cls.client.delete_image(image_id)
cls.image_ids.remove(image_id)
super(ImagesWhiteboxTest, cls).tearDownClass()
@classmethod
def update_state(self, server_id, vm_state, task_state, deleted=0):
"""Update states of an instance in database for validation."""
if not task_state:
task_state = "NULL"
instances = self.meta.tables['instances']
stmt = instances.update().where(instances.c.uuid == server_id).values(
deleted=deleted,
vm_state=vm_state,
task_state=task_state)
self.connection.execute(stmt, autocommit=True)
def _test_create_image_409_base(self, vm_state, task_state, deleted=0):
"""Base method for create image tests based on vm and task states."""
try:
self.update_state(self.shared_server['id'], vm_state,
task_state, deleted)
image_name = rand_name('snap-')
self.assertRaises(exceptions.Duplicate,
self.client.create_image,
self.shared_server['id'], image_name)
except Exception:
self.fail("Should not allow create image when vm_state=%s and "
"task_state=%s" % (vm_state, task_state))
finally:
self.update_state(self.shared_server['id'], 'active', None)
def test_create_image_when_vm_eq_building_task_eq_scheduling(self):
# 409 error when instance states are building,scheduling
self._test_create_image_409_base("building", "scheduling")
def test_create_image_when_vm_eq_building_task_eq_networking(self):
# 409 error when instance states are building,networking
self._test_create_image_409_base("building", "networking")
def test_create_image_when_vm_eq_building_task_eq_bdm(self):
# 409 error when instance states are building,block_device_mapping
self._test_create_image_409_base("building", "block_device_mapping")
def test_create_image_when_vm_eq_building_task_eq_spawning(self):
# 409 error when instance states are building,spawning
self._test_create_image_409_base("building", "spawning")
def test_create_image_when_vm_eq_active_task_eq_image_backup(self):
# 409 error when instance states are active,image_backup
self._test_create_image_409_base("active", "image_backup")
def test_create_image_when_vm_eq_resized_task_eq_resize_prep(self):
# 409 error when instance states are resized,resize_prep
self._test_create_image_409_base("resized", "resize_prep")
def test_create_image_when_vm_eq_resized_task_eq_resize_migrating(self):
# 409 error when instance states are resized,resize_migrating
self._test_create_image_409_base("resized", "resize_migrating")
def test_create_image_when_vm_eq_resized_task_eq_resize_migrated(self):
# 409 error when instance states are resized,resize_migrated
self._test_create_image_409_base("resized", "resize_migrated")
def test_create_image_when_vm_eq_resized_task_eq_resize_finish(self):
# 409 error when instance states are resized,resize_finish
self._test_create_image_409_base("resized", "resize_finish")
def test_create_image_when_vm_eq_resized_task_eq_resize_reverting(self):
# 409 error when instance states are resized,resize_reverting
self._test_create_image_409_base("resized", "resize_reverting")
def test_create_image_when_vm_eq_resized_task_eq_resize_confirming(self):
# 409 error when instance states are resized,resize_confirming
self._test_create_image_409_base("resized", "resize_confirming")
def test_create_image_when_vm_eq_active_task_eq_resize_verify(self):
# 409 error when instance states are active,resize_verify
self._test_create_image_409_base("active", "resize_verify")
def test_create_image_when_vm_eq_active_task_eq_updating_password(self):
# 409 error when instance states are active,updating_password
self._test_create_image_409_base("active", "updating_password")
def test_create_image_when_vm_eq_active_task_eq_rebuilding(self):
# 409 error when instance states are active,rebuilding
self._test_create_image_409_base("active", "rebuilding")
def test_create_image_when_vm_eq_active_task_eq_rebooting(self):
# 409 error when instance states are active,rebooting
self._test_create_image_409_base("active", "rebooting")
def test_create_image_when_vm_eq_building_task_eq_deleting(self):
# 409 error when instance states are building,deleting
self._test_create_image_409_base("building", "deleting")
def test_create_image_when_vm_eq_active_task_eq_deleting(self):
# 409 error when instance states are active,deleting
self._test_create_image_409_base("active", "deleting")
def test_create_image_when_vm_eq_error_task_eq_building(self):
# 409 error when instance states are error,building
self._test_create_image_409_base("error", "building")
def test_create_image_when_vm_eq_error_task_eq_none(self):
# 409 error when instance states are error,None
self._test_create_image_409_base("error", None)
def test_create_image_when_vm_eq_deleted_task_eq_none(self):
# 409 error when instance states are deleted,None
self._test_create_image_409_base("deleted", None)
def test_create_image_when_vm_eq_resized_task_eq_none(self):
# 409 error when instance states are resized,None
self._test_create_image_409_base("resized", None)
def test_create_image_when_vm_eq_error_task_eq_resize_prep(self):
# 409 error when instance states are error,resize_prep
self._test_create_image_409_base("error", "resize_prep")

251
tempest/whitebox/test_servers_whitebox.py Normal file
View File

@@ -0,0 +1,251 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 OpenStack, LLC
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from tempest import exceptions
from tempest.test import attr
from tempest.tests.identity.base import BaseIdentityAdminTest
from tempest.whitebox import manager
@attr(type='whitebox')
class ServersWhiteboxTest(manager.ComputeWhiteboxTest):
_interface = 'json'
@classmethod
def setUpClass(cls):
raise cls.skipException("Until Bug 1034129 is fixed")
super(ServersWhiteboxTest, cls).setUpClass()
#NOTE(afazekas): Strange relationship
BaseIdentityAdminTest.setUpClass()
cls.client = cls.servers_client
cls.img_client = cls.images_client
cls.admin_client = BaseIdentityAdminTest.client
cls.connection, cls.meta = cls.get_db_handle_and_meta()
resp, tenants = cls.admin_client.list_tenants()
cls.tenant_id = [
tnt['id']
for tnt in tenants if tnt['name'] == cls.config.compute.tenant_name
][0]
cls.shared_server = cls.create_server()
def tearDown(cls):
for server in cls.servers:
try:
cls.client.delete_server(server['id'])
except exceptions.NotFound:
continue
def test_create_server_vcpu_quota_full(self):
# Disallow server creation when tenant's vcpu quota is full
quotas = self.meta.tables['quotas']
stmt = (quotas.select().
where(quotas.c.project_id == self.tenant_id).
where(quotas.c.resource == 'cores'))
result = self.connection.execute(stmt).first()
# Set vcpu quota for tenant if not already set
if not result:
cores_hard_limit = 2
stmt = quotas.insert().values(deleted=0,
project_id=self.tenant_id,
resource='cores',
hard_limit=cores_hard_limit)
self.connection.execute(stmt, autocommit=True)
else:
cores_hard_limit = result.hard_limit
# Create servers assuming 1 VCPU per instance i.e flavor_id=1
try:
for count in range(cores_hard_limit + 1):
self.create_server()
except exceptions.OverLimit:
pass
else:
self.fail("Could create servers over the VCPU quota limit")
finally:
stmt = quotas.delete()
self.connection.execute(stmt, autocommit=True)
def test_create_server_memory_quota_full(self):
# Disallow server creation when tenant's memory quota is full
quotas = self.meta.tables['quotas']
stmt = (quotas.select().
where(quotas.c.project_id == self.tenant_id).
where(quotas.c.resource == 'ram'))
result = self.connection.execute(stmt).first()
# Set memory quota for tenant if not already set
if not result:
ram_hard_limit = 1024
stmt = quotas.insert().values(deleted=0,
project_id=self.tenant_id,
resource='ram',
hard_limit=ram_hard_limit)
self.connection.execute(stmt, autocommit=True)
else:
ram_hard_limit = result.hard_limit
try:
# Set a hard range of 3 servers for reaching the RAM quota
for count in range(3):
self.create_server()
except exceptions.OverLimit:
pass
else:
self.fail("Could create servers over the RAM quota limit")
finally:
stmt = quotas.delete()
self.connection.execute(stmt, autocommit=True)
def update_state(self, server_id, vm_state, task_state, deleted=0):
"""Update states of an instance in database for validation."""
if not task_state:
task_state = 'NULL'
instances = self.meta.tables['instances']
stmt = instances.update().where(instances.c.uuid == server_id).values(
deleted=deleted,
vm_state=vm_state,
task_state=task_state)
self.connection.execute(stmt, autocommit=True)
def _test_delete_server_base(self, vm_state, task_state):
"""
Base method for delete server tests based on vm and task states.
Validates for successful server termination.
"""
try:
server = self.create_server()
self.update_state(server['id'], vm_state, task_state)
resp, body = self.client.delete_server(server['id'])
self.assertEqual('204', resp['status'])
self.client.wait_for_server_termination(server['id'],
ignore_error=True)
instances = self.meta.tables['instances']
stmt = instances.select().where(instances.c.uuid == server['id'])
result = self.connection.execute(stmt).first()
self.assertEqual(1, result.deleted)
self.assertEqual('deleted', result.vm_state)
self.assertEqual(None, result.task_state)
except Exception:
self.fail("Should be able to delete a server when vm_state=%s and "
"task_state=%s" % (vm_state, task_state))
def _test_delete_server_403_base(self, vm_state, task_state):
"""
Base method for delete server tests based on vm and task states.
Validates for 403 error code.
"""
try:
self.update_state(self.shared_server['id'], vm_state, task_state)
self.assertRaises(exceptions.Unauthorized,
self.client.delete_server,
self.shared_server['id'])
except Exception:
self.fail("Should not allow delete server when vm_state=%s and "
"task_state=%s" % (vm_state, task_state))
finally:
self.update_state(self.shared_server['id'], 'active', None)
def test_delete_server_when_vm_eq_building_task_eq_networking(self):
# Delete server when instance states are building,networking
self._test_delete_server_base('building', 'networking')
def test_delete_server_when_vm_eq_building_task_eq_bdm(self):
# Delete server when instance states are building,block device mapping
self._test_delete_server_base('building', 'block_device_mapping')
def test_delete_server_when_vm_eq_building_task_eq_spawning(self):
# Delete server when instance states are building,spawning
self._test_delete_server_base('building', 'spawning')
def test_delete_server_when_vm_eq_active_task_eq_image_backup(self):
# Delete server when instance states are active,image_backup
self._test_delete_server_base('active', 'image_backup')
def test_delete_server_when_vm_eq_active_task_eq_rebuilding(self):
# Delete server when instance states are active,rebuilding
self._test_delete_server_base('active', 'rebuilding')
def test_delete_server_when_vm_eq_error_task_eq_spawning(self):
# Delete server when instance states are error,spawning
self._test_delete_server_base('error', 'spawning')
def test_delete_server_when_vm_eq_resized_task_eq_resize_prep(self):
# Delete server when instance states are resized,resize_prep
self._test_delete_server_403_base('resized', 'resize_prep')
def test_delete_server_when_vm_eq_resized_task_eq_resize_migrating(self):
# Delete server when instance states are resized,resize_migrating
self._test_delete_server_403_base('resized', 'resize_migrating')
def test_delete_server_when_vm_eq_resized_task_eq_resize_migrated(self):
# Delete server when instance states are resized,resize_migrated
self._test_delete_server_403_base('resized', 'resize_migrated')
def test_delete_server_when_vm_eq_resized_task_eq_resize_finish(self):
# Delete server when instance states are resized,resize_finish
self._test_delete_server_403_base('resized', 'resize_finish')
def test_delete_server_when_vm_eq_resized_task_eq_resize_reverting(self):
# Delete server when instance states are resized,resize_reverting
self._test_delete_server_403_base('resized', 'resize_reverting')
def test_delete_server_when_vm_eq_resized_task_eq_resize_confirming(self):
# Delete server when instance states are resized,resize_confirming
self._test_delete_server_403_base('resized', 'resize_confirming')
def test_delete_server_when_vm_eq_active_task_eq_resize_verify(self):
# Delete server when instance states are active,resize_verify
self._test_delete_server_base('active', 'resize_verify')
def test_delete_server_when_vm_eq_active_task_eq_rebooting(self):
# Delete server when instance states are active,rebooting
self._test_delete_server_base('active', 'rebooting')
def test_delete_server_when_vm_eq_building_task_eq_deleting(self):
# Delete server when instance states are building,deleting
self._test_delete_server_base('building', 'deleting')
def test_delete_server_when_vm_eq_active_task_eq_deleting(self):
# Delete server when instance states are active,deleting
self._test_delete_server_base('active', 'deleting')
def test_delete_server_when_vm_eq_error_task_eq_none(self):
# Delete server when instance states are error,None
self._test_delete_server_base('error', None)
def test_delete_server_when_vm_eq_resized_task_eq_none(self):
# Delete server when instance states are resized,None
self._test_delete_server_403_base('resized', None)
def test_delete_server_when_vm_eq_error_task_eq_resize_prep(self):
# Delete server when instance states are error,resize_prep
self._test_delete_server_base('error', 'resize_prep')
def test_delete_server_when_vm_eq_error_task_eq_error(self):
# Delete server when instance states are error,error
self._test_delete_server_base('error', 'error')