8f75894bad
This patch introduces the benchmark test framework for rally. The presented framework relies heavily on the usage of test configuration files which allow to set up which tests should be run in which order and how many times, as well as parameterize them (e.g., to configure, say, the number of compute nodes used in load testing). The test configuration also allows to set whether the benchmark tests should be launched concurrently in several threads. Test configuration processing is provided by the further development of TestEngine, which now also can validate the test configuration passed to its constructor. The patch also contains some refactoring of the Tester class which makes it configurable through the test configuration. This functionality is supported by the TestConfigManager class, another child class of the base ConfigManager defined in one of the previous patches. Blueprint test-engine Change-Id: Idd1296537d5a65c0b353ee9812d2e360d3c8ea39
107 lines
3.6 KiB
Python
107 lines
3.6 KiB
Python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright 2013: Mirantis Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Tests for the Test engine."""
|
|
import mock
|
|
import os
|
|
|
|
from rally.benchmark import engine
|
|
from rally import exceptions
|
|
from rally import test
|
|
|
|
|
|
class TestEngineTestCase(test.NoDBTestCase):
|
|
|
|
def setUp(self):
|
|
super(TestEngineTestCase, self).setUp()
|
|
|
|
self.valid_test_config = {
|
|
'verify': {
|
|
'tests_to_run': ['sanity', 'smoke']
|
|
},
|
|
'benchmark': {
|
|
'tests_to_run': {}
|
|
}
|
|
}
|
|
self.invalid_test_config_bad_test_name = {
|
|
'verify': {
|
|
'tests_to_run': ['sanity', 'some_not_existing_test']
|
|
},
|
|
'benchmark': {}
|
|
}
|
|
self.invalid_test_config_bad_key = {
|
|
'verify': {
|
|
'tests_to_run': ['sanity', 'smoke']
|
|
},
|
|
'benchmarck': {}
|
|
}
|
|
self.valid_cloud_config = {
|
|
'identity': {
|
|
'admin_name': 'admin',
|
|
'admin_password': 'admin'
|
|
},
|
|
'compute': {
|
|
'controller_nodes': 'localhost'
|
|
}
|
|
}
|
|
|
|
run_success = {
|
|
'proc': {'msg': ['msg'], 'status': 0, 'proc_name': 'proc'}
|
|
}
|
|
self.run_mock = mock.patch('rally.benchmark.utils.Tester.run',
|
|
mock.Mock(return_value=run_success))
|
|
self.run_mock.start()
|
|
|
|
def tearDown(self):
|
|
self.run_mock.stop()
|
|
super(TestEngineTestCase, self).tearDown()
|
|
|
|
def test_verify_test_config(self):
|
|
try:
|
|
engine.TestEngine(self.valid_test_config)
|
|
except Exception as e:
|
|
self.fail("Unexpected exception in test config" +
|
|
"verification: %s" % str(e))
|
|
self.assertRaises(exceptions.NoSuchTestException,
|
|
engine.TestEngine,
|
|
self.invalid_test_config_bad_test_name)
|
|
self.assertRaises(exceptions.InvalidConfigException,
|
|
engine.TestEngine,
|
|
self.invalid_test_config_bad_key)
|
|
|
|
def test_bind(self):
|
|
test_engine = engine.TestEngine(self.valid_test_config)
|
|
with test_engine.bind(self.valid_cloud_config):
|
|
self.assertTrue(os.path.exists(test_engine.cloud_config_path))
|
|
self.assertTrue(os.path.exists(test_engine.test_config_path))
|
|
self.assertFalse(os.path.exists(test_engine.cloud_config_path))
|
|
self.assertFalse(os.path.exists(test_engine.test_config_path))
|
|
|
|
def test_verify(self):
|
|
test_engine = engine.TestEngine(self.valid_test_config)
|
|
with test_engine.bind(self.valid_cloud_config):
|
|
try:
|
|
test_engine.verify()
|
|
except Exception as e:
|
|
self.fail("Unexpected exception in TestEngine.verify: %s" %
|
|
str(e))
|
|
|
|
def test_benchmark(self):
|
|
test_engine = engine.TestEngine(self.valid_test_config)
|
|
with test_engine.bind(self.valid_cloud_config):
|
|
test_engine.benchmark()
|