Add full-stack tests framework

This patch introduces the full-stack tests framework, as specified in
the blueprint. In short, this adds the neutron.tests.fullstack module,
which supports test-managed neutron daemons. Currently only
neutron-server is supported and follow-up patches will support for
multiple agents.

Implements: blueprint integration-tests
Co-Authored-By: Maru Newby <marun@redhat.com>
Change-Id: Iff24fc7cd428488e918c5f06bc7f923095760b07
This commit is contained in:
John Schwarz 2014-10-14 14:09:14 +03:00
parent 41486020f6
commit e0ea5edc12
16 changed files with 528 additions and 41 deletions

View File

@ -138,6 +138,20 @@ to install and configure all of Neutron's package dependencies. It is
not necessary to provide this option if devstack has already been used
to deploy Neutron to the target host.
To run all the full-stack tests, you may use: ::
tox -e dsvm-fullstack
Since full-stack tests often require the same resources and
dependencies as the functional tests, using the configuration script
tools/configure_for_func_testing.sh is advised (as described above).
When running full-stack tests on a clean VM for the first time, we
advise to run ./stack.sh successfully to make sure all Neutron's
dependencies are met. Also note that in order to preserve resources
on the gate, running the dsvm-functional suite will also run all
full-stack tests (and a new worker won't be assigned specifically for
dsvm-fullstack).
To run the api tests against a live Neutron daemon, deploy tempest and
neutron with devstack and then run the following commands: ::

View File

@ -172,6 +172,9 @@ class AsyncProcess(object):
LOG.exception(_LE('An error occurred while killing [%s].'),
self.cmd)
return False
if self._process:
self._process.wait()
return True
def _handle_process_error(self):
@ -188,7 +191,8 @@ class AsyncProcess(object):
def _watch_process(self, callback, kill_event):
while not kill_event.ready():
try:
if not callback():
output = callback()
if not output and output != "":
break
except Exception:
LOG.exception(_LE('An error occurred while communicating '

View File

@ -286,23 +286,23 @@ def get_cmdline_from_pid(pid):
return f.readline().split('\0')[:-1]
def cmdlines_are_equal(cmd1, cmd2):
"""Validate provided lists containing output of /proc/cmdline are equal
This function ignores absolute paths of executables in order to have
correct results in case one list uses absolute path and the other does not.
"""
cmd1 = remove_abs_path(cmd1)
cmd2 = remove_abs_path(cmd2)
return cmd1 == cmd2
def cmd_matches_expected(cmd, expected_cmd):
abs_cmd = remove_abs_path(cmd)
abs_expected_cmd = remove_abs_path(expected_cmd)
if abs_cmd != abs_expected_cmd:
# Commands executed with #! are prefixed with the script
# executable. Check for the expected cmd being a subset of the
# actual cmd to cover this possibility.
abs_cmd = remove_abs_path(abs_cmd[1:])
return abs_cmd == abs_expected_cmd
def pid_invoked_with_cmdline(pid, expected_cmd):
"""Validate process with given pid is running with provided parameters
"""
cmdline = get_cmdline_from_pid(pid)
return cmdlines_are_equal(expected_cmd, cmdline)
cmd = get_cmdline_from_pid(pid)
return cmd_matches_expected(cmd, expected_cmd)
def wait_until_true(predicate, timeout=60, sleep=1, exception=None):

View File

@ -55,19 +55,6 @@ import testtools
from neutron.tests import sub_base
class AttributeDict(dict):
"""
Provide attribute access (dict.key) to dictionary values.
"""
def __getattr__(self, name):
"""Allow attribute access for all keys in the dict."""
if name in self:
return self[name]
raise AttributeError(_("Unknown attribute '%s'.") % name)
@six.add_metaclass(abc.ABCMeta)
class BaseNeutronClient(object):
"""

View File

@ -22,6 +22,7 @@ from tempest_lib import exceptions
import testscenarios
from neutron.tests.api import base_v2
from neutron.tests import sub_base
from neutron.tests.tempest import test as t_test
# Required to generate tests from scenarios. Not compatible with nose.
@ -55,19 +56,19 @@ class TempestRestClient(base_v2.BaseNeutronClient):
def _create_network(self, **kwargs):
# Internal method - use create_network() instead
body = self.client.create_network(**kwargs)
return base_v2.AttributeDict(body['network'])
return sub_base.AttributeDict(body['network'])
def update_network(self, id_, **kwargs):
body = self.client.update_network(id_, **kwargs)
return base_v2.AttributeDict(body['network'])
return sub_base.AttributeDict(body['network'])
def get_network(self, id_, **kwargs):
body = self.client.show_network(id_, **kwargs)
return base_v2.AttributeDict(body['network'])
return sub_base.AttributeDict(body['network'])
def get_networks(self, **kwargs):
body = self.client.list_networks(**kwargs)
return [base_v2.AttributeDict(x) for x in body['networks']]
return [sub_base.AttributeDict(x) for x in body['networks']]
def delete_network(self, id_):
self.client.delete_network(id_)

View File

@ -0,0 +1,31 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import neutron
def find_file(filename, path):
"""Find a file with name 'filename' located in 'path'."""
for root, _, files in os.walk(path):
if filename in files:
return os.path.abspath(os.path.join(root, filename))
def find_sample_file(filename):
"""Find a file with name 'filename' located in the sample directory."""
return find_file(
filename,
path=os.path.join(neutron.__path__[0], '..', 'etc'))

View File

View File

@ -0,0 +1,60 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_db.sqlalchemy import test_base
from neutron.db.migration.models import head # noqa
from neutron.db import model_base
from neutron.tests.fullstack import fullstack_fixtures as f_fixtures
class BaseFullStackTestCase(test_base.MySQLOpportunisticTestCase):
"""Base test class for full-stack tests.
:param process_fixtures: a list of fixture classes (not instances).
"""
def setUp(self):
super(BaseFullStackTestCase, self).setUp()
self.create_db_tables()
self.neutron_server = self.useFixture(
f_fixtures.NeutronServerFixture())
self.client = self.neutron_server.client
@property
def test_name(self):
"""Return the name of the test currently running."""
return self.id().split(".")[-1]
def create_db_tables(self):
"""Populate the new database.
MySQLOpportunisticTestCase creates a new database for each test, but
these need to be populated with the appropriate tables. Before we can
do that, we must change the 'connection' option which the Neutron code
knows to look at.
Currently, the username and password options are hard-coded by
oslo.db and neutron/tests/functional/contrib/gate_hook.sh. Also,
we only support MySQL for now, but the groundwork for adding Postgres
is already laid.
"""
conn = "mysql://%(username)s:%(password)s@127.0.0.1/%(db_name)s" % {
'username': test_base.DbFixture.USERNAME,
'password': test_base.DbFixture.PASSWORD,
'db_name': self.engine.url.database}
cfg.CONF.set_override('connection', conn, group='database')
model_base.BASEV2.metadata.create_all(self.engine)

View File

@ -0,0 +1,183 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os.path
import tempfile
import fixtures
import six
from neutron.common import constants
from neutron.tests.common import helpers as c_helpers
from neutron.tests.functional.agent.linux import helpers
from neutron.tests import sub_base
class ConfigDict(sub_base.AttributeDict):
def update(self, other):
self.convert_to_attr_dict(other)
super(ConfigDict, self).update(other)
def convert_to_attr_dict(self, other):
"""Convert nested dicts to AttributeDict.
:param other: dictionary to be directly modified.
"""
for key, value in other.iteritems():
if isinstance(value, dict):
if not isinstance(value, sub_base.AttributeDict):
other[key] = sub_base.AttributeDict(value)
self.convert_to_attr_dict(value)
class ConfigFileFixture(fixtures.Fixture):
"""A fixture that knows how to translate configurations to files.
:param base_filename: the filename to use on disk.
:param config: a ConfigDict instance.
:param temp_dir: an existing temporary directory to use for storage.
"""
def __init__(self, base_filename, config, temp_dir):
super(ConfigFileFixture, self).__init__()
self.base_filename = base_filename
self.config = config
self.temp_dir = temp_dir
def setUp(self):
super(ConfigFileFixture, self).setUp()
config_parser = self.dict_to_config_parser(self.config)
# Need to randomly generate a unique folder to put the file in
self.filename = os.path.join(self.temp_dir, self.base_filename)
with open(self.filename, 'w') as f:
config_parser.write(f)
f.flush()
def dict_to_config_parser(self, config_dict):
config_parser = six.moves.configparser.SafeConfigParser()
for section, section_dict in six.iteritems(config_dict):
if section != 'DEFAULT':
config_parser.add_section(section)
for option, value in six.iteritems(section_dict):
config_parser.set(section, option, value)
return config_parser
class ConfigFixture(fixtures.Fixture):
"""A fixture that holds an actual Neutron configuration.
Note that 'self.config' is intended to only be updated once, during
the constructor, so if this fixture is re-used (setUp is called twice),
then the dynamic configuration values won't change. The correct usage
is initializing a new instance of the class.
"""
def __init__(self, temp_dir, base_filename):
self.config = ConfigDict()
self.temp_dir = temp_dir
self.base_filename = base_filename
def setUp(self):
super(ConfigFixture, self).setUp()
cfg_fixture = ConfigFileFixture(
self.base_filename, self.config, self.temp_dir)
self.useFixture(cfg_fixture)
self.filename = cfg_fixture.filename
class NeutronConfigFixture(ConfigFixture):
def __init__(self, temp_dir, connection):
super(NeutronConfigFixture, self).__init__(
temp_dir, base_filename='neutron.conf')
self.config.update({
'DEFAULT': {
'host': self._generate_host(),
'state_path': self._generate_state_path(temp_dir),
'bind_port': self._generate_port(),
'api_paste_config': self._generate_api_paste(),
'policy_file': self._generate_policy_json(),
'core_plugin': 'neutron.plugins.ml2.plugin.Ml2Plugin',
'rabbit_userid': 'stackrabbit',
'rabbit_password': 'secretrabbit',
'rabbit_hosts': '127.0.0.1',
'auth_strategy': 'noauth',
'verbose': 'True',
'debug': 'True',
},
'database': {
'connection': connection,
}
})
def _generate_host(self):
return sub_base.get_rand_name(prefix='host-')
def _generate_state_path(self, temp_dir):
# Assume that temp_dir will be removed by the caller
self.state_path = tempfile.mkdtemp(prefix='state_path', dir=temp_dir)
return self.state_path
def _generate_port(self):
"""Get a free TCP port from the Operating System and return it.
This might fail if some other process occupies this port after this
function finished but before the neutron-server process started.
"""
return str(helpers.get_free_namespace_port())
def _generate_api_paste(self):
return c_helpers.find_sample_file('api-paste.ini')
def _generate_policy_json(self):
return c_helpers.find_sample_file('policy.json')
class ML2ConfigFixture(ConfigFixture):
def __init__(self, temp_dir):
super(ML2ConfigFixture, self).__init__(
temp_dir, base_filename='ml2_conf.ini')
self.config.update({
'ml2': {
'tenant_network_types': 'vlan',
'mechanism_drivers': 'openvswitch',
},
'ml2_type_vlan': {
'network_vlan_ranges': 'physnet1:1000:2999',
},
'ml2_type_gre': {
'tunnel_id_ranges': '1:1000',
},
'ml2_type_vxlan': {
'vni_ranges': '1001:2000',
},
'ovs': {
'enable_tunneling': 'False',
'local_ip': '127.0.0.1',
'bridge_mappings': self._generate_bridge_mappings(),
'integration_bridge': self._generate_integration_bridge(),
}
})
def _generate_bridge_mappings(self):
return ('physnet1:%s' %
sub_base.get_rand_name(
prefix='br-eth',
max_length=constants.DEVICE_NAME_MAX_LEN))
def _generate_integration_bridge(self):
return sub_base.get_rand_name(prefix='br-int',
max_length=constants.DEVICE_NAME_MAX_LEN)

View File

@ -0,0 +1,104 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from distutils import spawn
import fixtures
from neutronclient.common import exceptions as nc_exc
from neutronclient.v2_0 import client
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import timeutils
from neutron.agent.linux import async_process
from neutron.agent.linux import utils
from neutron.tests.fullstack import config_fixtures
LOG = logging.getLogger(__name__)
# This should correspond the directory from which infra retrieves log files
DEFAULT_LOG_DIR = '/opt/stack/logs'
class ProcessFixture(fixtures.Fixture):
def __init__(self, name, exec_name, config_filenames):
super(ProcessFixture, self).__init__()
self.name = name
self.exec_name = exec_name
self.config_filenames = config_filenames
self.process = None
def setUp(self):
super(ProcessFixture, self).setUp()
self.start()
def start(self):
fmt = self.name + "--%Y-%m-%d--%H%M%S.log"
cmd = [spawn.find_executable(self.exec_name),
'--log-dir', DEFAULT_LOG_DIR,
'--log-file', timeutils.strtime(fmt=fmt)]
for filename in self.config_filenames:
cmd += ['--config-file', filename]
self.process = async_process.AsyncProcess(cmd)
self.process.start(block=True)
def stop(self):
self.process.stop(block=True)
def cleanUp(self, *args, **kwargs):
self.stop()
super(ProcessFixture, self, *args, **kwargs)
class NeutronServerFixture(fixtures.Fixture):
def setUp(self):
super(NeutronServerFixture, self).setUp()
self.temp_dir = self.useFixture(fixtures.TempDir()).path
self.neutron_cfg_fixture = config_fixtures.NeutronConfigFixture(
self.temp_dir, cfg.CONF.database.connection)
self.plugin_cfg_fixture = config_fixtures.ML2ConfigFixture(
self.temp_dir)
self.useFixture(self.neutron_cfg_fixture)
self.useFixture(self.plugin_cfg_fixture)
self.neutron_config = self.neutron_cfg_fixture.config
config_filenames = [self.neutron_cfg_fixture.filename,
self.plugin_cfg_fixture.filename]
self.process_fixture = self.useFixture(ProcessFixture(
name='neutron_server',
exec_name='neutron-server',
config_filenames=config_filenames,
))
utils.wait_until_true(self.processes_are_ready)
@property
def client(self):
url = "http://127.0.0.1:%s" % self.neutron_config.DEFAULT.bind_port
return client.Client(auth_strategy="noauth", endpoint_url=url)
def processes_are_ready(self):
# ProcessFixture will ensure that the server has started, but
# that doesn't mean that the server will be serving commands yet, nor
# that all processes are up.
try:
return len(self.client.list_agents()['agents']) == 0
except nc_exc.NeutronClientException:
LOG.debug("Processes aren't up yet.")
return False

View File

@ -0,0 +1,25 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#TODO(jschwarz): This is an example test file which demonstrates the
# general usage of fullstack. Once we add more FullStack tests, this should
# be deleted.
from neutron.tests.fullstack import base
class TestSanity(base.BaseFullStackTestCase):
def test_sanity(self):
self.assertEqual(self.client.list_networks(), {'networks': []})

View File

@ -0,0 +1,44 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Previously, running 'tox -e dsvm-functional' simply ran a normal test discovery
of the ./neutron/tests/functional tree. In order to save gate resources, we
decided to forgo adding a new gate job specifically for the full-stack
framework, and instead discover tests that are present in
./neutron/tests/fullstack.
In short, running 'tox -e dsvm-functional' now runs both functional tests and
full-stack tests, and this code allows for the test discovery needed.
"""
import os
import unittest
def _discover(loader, path, pattern):
return loader.discover(path, pattern=pattern, top_level_dir=".")
def load_tests(_, tests, pattern):
suite = unittest.TestSuite()
suite.addTests(tests)
loader = unittest.loader.TestLoader()
suite.addTests(_discover(loader, "./neutron/tests/functional", pattern))
if os.getenv('OS_RUN_FULLSTACK') == '1':
suite.addTests(_discover(loader, "./neutron/tests/fullstack", pattern))
return suite

View File

@ -25,6 +25,7 @@ from neutron.common import exceptions as q_exc
from neutron import context
from neutron import manager
from neutron.tests.api import base_v2
from neutron.tests import sub_base
from neutron.tests.unit.ml2 import test_ml2_plugin
from neutron.tests.unit import testlib_api
from neutron.tests.unit import testlib_plugin
@ -68,20 +69,20 @@ class PluginClient(base_v2.BaseNeutronClient):
kwargs.setdefault('shared', False)
data = dict(network=kwargs)
result = self.plugin.create_network(self.ctx, data)
return base_v2.AttributeDict(result)
return sub_base.AttributeDict(result)
def update_network(self, id_, **kwargs):
data = dict(network=kwargs)
result = self.plugin.update_network(self.ctx, id_, data)
return base_v2.AttributeDict(result)
return sub_base.AttributeDict(result)
def get_network(self, *args, **kwargs):
result = self.plugin.get_network(self.ctx, *args, **kwargs)
return base_v2.AttributeDict(result)
return sub_base.AttributeDict(result)
def get_networks(self, *args, **kwargs):
result = self.plugin.get_networks(self.ctx, *args, **kwargs)
return [base_v2.AttributeDict(x) for x in result]
return [sub_base.AttributeDict(x) for x in result]
def delete_network(self, id_):
self.plugin.delete_network(self.ctx, id_)

View File

@ -52,6 +52,19 @@ def bool_from_env(key, strict=False, default=False):
return strutils.bool_from_string(value, strict=strict, default=default)
class AttributeDict(dict):
"""
Provide attribute access (dict.key) to dictionary values.
"""
def __getattr__(self, name):
"""Allow attribute access for all keys in the dict."""
if name in self:
return self[name]
raise AttributeError(_("Unknown attribute '%s'.") % name)
class SubBaseTestCase(testtools.TestCase):
def setUp(self):

View File

@ -198,15 +198,16 @@ class TestPathUtilities(base.BaseTestCase):
self.assertEqual(['ping', '8.8.8.8'],
utils.remove_abs_path(['/usr/bin/ping', '8.8.8.8']))
def test_cmdlines_are_equal(self):
self.assertTrue(utils.cmdlines_are_equal(
['ping', '8.8.8.8'],
['/usr/bin/ping', '8.8.8.8']))
def test_cmd_matches_expected_matches_abs_path(self):
cmd = ['/bar/../foo']
self.assertTrue(utils.cmd_matches_expected(cmd, cmd))
def test_cmdlines_are_equal_different_commands(self):
self.assertFalse(utils.cmdlines_are_equal(
['ping', '8.8.8.8'],
['/usr/bin/ping6', '8.8.8.8']))
def test_cmd_matches_expected_matches_script(self):
self.assertTrue(utils.cmd_matches_expected(['python', 'script'],
['script']))
def test_cmd_matches_expected_doesnt_match(self):
self.assertFalse(utils.cmd_matches_expected('foo', 'bar'))
class TestBaseOSUtils(base.BaseTestCase):

19
tox.ini
View File

@ -43,6 +43,25 @@ setenv = OS_TEST_PATH=./neutron/tests/functional
OS_ROOTWRAP_DAEMON_CMD=sudo {envbindir}/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
OS_FAIL_ON_MISSING_DEPS=1
OS_TEST_TIMEOUT=90
OS_RUN_FULLSTACK=1
sitepackages=True
deps =
{[testenv:functional]deps}
[testenv:fullstack]
setenv = OS_TEST_PATH=./neutron/tests/fullstack
OS_TEST_TIMEOUT=90
deps =
{[testenv]deps}
-r{toxinidir}/neutron/tests/functional/requirements.txt
[testenv:dsvm-fullstack]
setenv = OS_TEST_PATH=./neutron/tests/fullstack
OS_SUDO_TESTING=1
OS_ROOTWRAP_CMD=sudo {envbindir}/neutron-rootwrap {envdir}/etc/neutron/rootwrap.conf
OS_ROOTWRAP_DAEMON_CMD=sudo {envbindir}/neutron-rootwrap-daemon {envdir}/etc/neutron/rootwrap.conf
OS_FAIL_ON_MISSING_DEPS=1
OS_TEST_TIMEOUT=90
sitepackages=True
deps =
{[testenv:functional]deps}