e4f58bf63b
Implements decorator for adding validators to method. Validators are called in benchmark runner. Validation method should return instance of rally.benchmark.validation.ValidationResult class. Blueprint validate-benchmark-preconditions Change-Id: I68c8f5cc5d8758026bbe7ef211aa914b313ebb81
474 lines
22 KiB
Python
474 lines
22 KiB
Python
# Copyright 2013: Mirantis Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""Tests for utils."""
|
|
import mock
|
|
import multiprocessing
|
|
|
|
from rally.benchmark import runner
|
|
from rally.benchmark import validation
|
|
from rally import exceptions
|
|
from tests import fakes
|
|
from tests import test
|
|
|
|
|
|
class MockedPool(object):
|
|
|
|
def __init__(self, concurrent=1):
|
|
pass
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
def join(self):
|
|
pass
|
|
|
|
def apply_async(self, func, args=()):
|
|
func(*args)
|
|
|
|
|
|
class ScenarioTestCase(test.TestCase):
|
|
|
|
def setUp(self):
|
|
super(ScenarioTestCase, self).setUp()
|
|
admin_keys = ["username", "password", "tenant_name", "auth_url"]
|
|
self.fake_kw = dict(zip(admin_keys, admin_keys))
|
|
|
|
def test_init_calls_register(self):
|
|
with mock.patch("rally.benchmark.utils.osclients") as mock_osclients:
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
with mock.patch("rally.benchmark.runner.base") as mock_base:
|
|
runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
self.assertEqual(mock_base.mock_calls,
|
|
[mock.call.Scenario.register()])
|
|
|
|
def test_create_temp_tenants_and_users(self):
|
|
with mock.patch("rally.benchmark.utils.osclients") as mock_osclients:
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
tenants = 10
|
|
users_per_tenant = 5
|
|
endpoints = srunner._create_temp_tenants_and_users(
|
|
tenants, users_per_tenant)
|
|
self.assertEqual(len(endpoints), tenants * users_per_tenant)
|
|
endpoint_keys = set(["username", "password", "tenant_name",
|
|
"auth_url"])
|
|
for endpoint in endpoints:
|
|
self.assertTrue(endpoint_keys.issubset(endpoint.keys()))
|
|
|
|
def test_run_scenario(self):
|
|
with mock.patch("rally.benchmark.utils.osclients") as mock_osclients:
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
with mock.patch("rally.benchmark.runner.rutils") as mock_utils:
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
runner.__openstack_clients__ = ["client"]
|
|
active_users = 2
|
|
times = 3
|
|
duration = 0.01
|
|
|
|
mock_utils.Timer = fakes.FakeTimer
|
|
results = srunner._run_scenario(fakes.FakeScenario,
|
|
"do_it", {}, "continuous",
|
|
{"times": times,
|
|
"active_users": active_users,
|
|
"timeout": 2})
|
|
expected = [{"time": 10, "idle_time": 0, "error": None,
|
|
"scenario_output": None}
|
|
for i in range(times)]
|
|
self.assertEqual(results, expected)
|
|
|
|
results = srunner._run_scenario(fakes.FakeScenario,
|
|
"do_it", {}, "continuous",
|
|
{"duration": duration,
|
|
"active_users": active_users,
|
|
"timeout": 2})
|
|
expected = [{"time": 10, "idle_time": 0, "error": None,
|
|
"scenario_output": None}
|
|
for i in range(active_users)]
|
|
self.assertEqual(results, expected)
|
|
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
@mock.patch("multiprocessing.pool.IMapIterator.next")
|
|
@mock.patch("rally.benchmark.runner.time.time")
|
|
@mock.patch("rally.benchmark.utils._prepare_for_instance_ssh")
|
|
def test_run_scenario_timeout(self, mock_prepare_for_instance_ssh,
|
|
mock_time, mock_next, mock_osclients):
|
|
|
|
mock_time.side_effect = [1, 2, 3, 10]
|
|
mock_next.side_effect = multiprocessing.TimeoutError()
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
runner.__openstack_clients__ = ["client"]
|
|
times = 4
|
|
active_users = 2
|
|
results = srunner._run_scenario(fakes.FakeScenario,
|
|
"too_long", {}, "continuous",
|
|
{"times": times,
|
|
"active_users": active_users,
|
|
"timeout": 0.01})
|
|
self.assertEqual(len(results), times)
|
|
for r in results:
|
|
self.assertEqual(r['time'], 0.01)
|
|
self.assertEqual(r['error'][0],
|
|
str(multiprocessing.TimeoutError))
|
|
|
|
duration = 0.1
|
|
results = srunner._run_scenario(fakes.FakeScenario,
|
|
"too_long", {}, "continuous",
|
|
{"duration": duration,
|
|
"active_users": active_users,
|
|
"timeout": 0.01})
|
|
self.assertEqual(len(results), active_users)
|
|
for r in results:
|
|
self.assertEqual(r['time'], 0.01)
|
|
self.assertEqual(r['error'][0],
|
|
str(multiprocessing.TimeoutError))
|
|
|
|
def test_run_scenario_exception_inside_test(self):
|
|
with mock.patch("rally.benchmark.utils.osclients") as mock_osclients:
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
runner.__openstack_clients__ = ["client"]
|
|
times = 1
|
|
duration = 0.01
|
|
active_users = 2
|
|
with mock.patch("rally.benchmark.runner.rutils") as mock_utils:
|
|
mock_utils.Timer = fakes.FakeTimer
|
|
results = srunner._run_scenario(fakes.FakeScenario,
|
|
"something_went_wrong", {},
|
|
"continuous",
|
|
{"times": times,
|
|
"active_users": active_users,
|
|
"timeout": 1})
|
|
self.assertEqual(len(results), times)
|
|
for r in results:
|
|
self.assertEqual(r['time'], 10)
|
|
self.assertEqual(r['error'][:2],
|
|
[str(Exception), "Something went wrong"])
|
|
|
|
results = srunner._run_scenario(fakes.FakeScenario,
|
|
"something_went_wrong", {},
|
|
"continuous",
|
|
{"duration": duration,
|
|
"active_users": active_users,
|
|
"timeout": 1})
|
|
self.assertEqual(len(results), active_users)
|
|
for r in results:
|
|
self.assertEqual(r['time'], 10)
|
|
self.assertEqual(r['error'][:2],
|
|
[str(Exception), "Something went wrong"])
|
|
|
|
def test_run_scenario_exception_outside_test(self):
|
|
pass
|
|
|
|
@mock.patch("rally.benchmark.runner.multiprocessing")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
def test_run_scenario_continuously_for_times(self, mock_osclients,
|
|
mock_multi):
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
runner.__openstack_clients__ = ["client"]
|
|
times = 3
|
|
active_users = 4
|
|
timeout = 5
|
|
mock_multi.Pool = mock.MagicMock()
|
|
srunner._run_scenario_continuously_for_times(fakes.FakeScenario,
|
|
"do_it", {},
|
|
times, active_users,
|
|
timeout)
|
|
expect = [
|
|
mock.call(active_users),
|
|
mock.call().imap(
|
|
runner._run_scenario_loop,
|
|
[(i, fakes.FakeScenario, "do_it", {})
|
|
for i in xrange(times)]
|
|
)
|
|
]
|
|
expect.extend([mock.call().imap().next(timeout) for i in range(times)])
|
|
expect.extend([
|
|
mock.call().close(),
|
|
mock.call().join()
|
|
])
|
|
self.assertEqual(mock_multi.Pool.mock_calls, expect)
|
|
|
|
@mock.patch("rally.benchmark.utils.infinite_run_args")
|
|
@mock.patch("rally.benchmark.runner.multiprocessing")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
def test_run_scenario_continuously_for_duration(self, mock_osclients,
|
|
mock_multi, mock_generate):
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
runner.__openstack_clients__ = ["client"]
|
|
duration = 0
|
|
active_users = 4
|
|
timeout = 5
|
|
mock_multi.Pool = mock.MagicMock()
|
|
mock_generate.return_value = {}
|
|
srunner._run_scenario_continuously_for_duration(fakes.FakeScenario,
|
|
"do_it", {}, duration,
|
|
active_users, timeout)
|
|
expect = [
|
|
mock.call(active_users),
|
|
mock.call().imap(runner._run_scenario_loop, {}),
|
|
mock.call().terminate(),
|
|
mock.call().join()
|
|
]
|
|
self.assertEqual(mock_multi.Pool.mock_calls, expect)
|
|
|
|
@mock.patch("rally.benchmark.runner._run_scenario_loop")
|
|
@mock.patch("rally.benchmark.runner.time.sleep")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
def test_run_scenario_periodically(self, mock_osclients,
|
|
mock_sleep, mock_run_scenario_loop):
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
runner.__openstack_clients__ = ["client"]
|
|
times = 3
|
|
period = 4
|
|
timeout = 5
|
|
srunner._run_scenario_periodically(fakes.FakeScenario, "do_it", {},
|
|
times, period, timeout)
|
|
|
|
expected = [mock.call((i, fakes.FakeScenario, "do_it", {}))
|
|
for i in xrange(times)]
|
|
self.assertEqual(mock_run_scenario_loop.mock_calls, expected)
|
|
|
|
expected = [mock.call(period * 60) for i in xrange(times - 1)]
|
|
mock_sleep.has_calls(expected)
|
|
|
|
@mock.patch("rally.benchmark.runner.base")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
def test_run_continuous(self, mock_osclients, mock_base):
|
|
FakeScenario = mock.MagicMock()
|
|
FakeScenario.init = mock.MagicMock(return_value={})
|
|
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
srunner._run_scenario_continuously_for_times = \
|
|
mock.MagicMock(return_value="result")
|
|
srunner._run_scenario_continuously_for_duration = \
|
|
mock.MagicMock(return_value="result")
|
|
srunner._create_temp_tenants_and_users = mock.MagicMock(
|
|
return_value=[])
|
|
srunner._delete_temp_tenants_and_users = mock.MagicMock()
|
|
|
|
mock_base.Scenario.get_by_name = \
|
|
mock.MagicMock(return_value=FakeScenario)
|
|
mock_osclients.return_value = ["client"]
|
|
result = srunner._run_scenario(FakeScenario, "do_it", {"a": 1},
|
|
"continuous", {"times": 2,
|
|
"active_users": 3,
|
|
"timeout": 1})
|
|
self.assertEqual(result, "result")
|
|
srunner._run_scenario_continuously_for_times.assert_called_once_with(
|
|
FakeScenario, "do_it", {"a": 1}, 2, 3, 1)
|
|
result = srunner._run_scenario(FakeScenario, "do_it", {"a": 1},
|
|
"continuous", {"duration": 2,
|
|
"active_users": 3,
|
|
"timeout": 1})
|
|
self.assertEqual(result, "result")
|
|
srunner._run_scenario_continuously_for_duration.\
|
|
assert_called_once_with(FakeScenario, "do_it", {"a": 1}, 2, 3, 1)
|
|
|
|
@mock.patch("rally.benchmark.runner.base")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
def test_run_periodic(self, mock_osclients, mock_base):
|
|
FakeScenario = mock.MagicMock()
|
|
FakeScenario.init = mock.MagicMock(return_value={})
|
|
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
srunner._run_scenario_periodically = mock.MagicMock(
|
|
return_value="result")
|
|
srunner._create_temp_tenants_and_users = mock.MagicMock(
|
|
return_value=[])
|
|
srunner._delete_temp_tenants_and_users = mock.MagicMock()
|
|
|
|
mock_base.Scenario.get_by_name = \
|
|
mock.MagicMock(return_value=FakeScenario)
|
|
mock_osclients.return_value = ["client"]
|
|
result = srunner._run_scenario(FakeScenario, "do_it", {"a": 1},
|
|
"periodic", {"times": 2, "period": 3,
|
|
"timeout": 1})
|
|
self.assertEqual(result, "result")
|
|
srunner._run_scenario_periodically.assert_called_once_with(
|
|
FakeScenario, "do_it", {"a": 1}, 2, 3, 1)
|
|
|
|
def _set_mocks_for_run(self, mock_osclients, mock_base, mock_clients,
|
|
validators=None):
|
|
FakeScenario = mock.MagicMock()
|
|
FakeScenario.init = mock.MagicMock(return_value={})
|
|
if validators:
|
|
FakeScenario.do_it.validators = validators
|
|
|
|
mock_osclients.Clients.return_value = fakes.FakeClients()
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
srunner._run_scenario = mock.MagicMock(return_value="result")
|
|
srunner._create_temp_tenants_and_users = mock.MagicMock(
|
|
return_value=[])
|
|
srunner._delete_temp_tenants_and_users = mock.MagicMock()
|
|
|
|
mock_base.Scenario.get_by_name = \
|
|
mock.MagicMock(return_value=FakeScenario)
|
|
return FakeScenario, srunner
|
|
|
|
@mock.patch("rally.benchmark.utils.create_openstack_clients")
|
|
@mock.patch("rally.benchmark.runner.base")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
def test_run(self, mock_osclients, mock_base, mock_clients):
|
|
FakeScenario, srunner = self._set_mocks_for_run(mock_osclients,
|
|
mock_base,
|
|
mock_clients)
|
|
|
|
result = srunner.run("FakeScenario.do_it", {})
|
|
self.assertEqual(result, "result")
|
|
srunner.run("FakeScenario.do_it",
|
|
{"args": {"a": 1}, "init": {"arg": 1},
|
|
"config": {"timeout": 1, "times": 2, "active_users": 3,
|
|
"tenants": 5, "users_per_tenant": 2}})
|
|
srunner.run("FakeScenario.do_it",
|
|
{"args": {"a": 1}, "init": {"fake": "arg"},
|
|
"execution_type": "continuous",
|
|
"config": {"timeout": 1, "duration": 40,
|
|
"active_users": 3, "tenants": 5,
|
|
"users_per_tenant": 2}})
|
|
|
|
expected = [
|
|
mock.call(FakeScenario, "do_it", {}, "continuous", {}),
|
|
mock.call(FakeScenario, "do_it", {"a": 1}, "continuous",
|
|
{"timeout": 1, "times": 2, "active_users": 3,
|
|
"tenants": 5, "users_per_tenant": 2}),
|
|
mock.call(FakeScenario, "do_it", {"a": 1}, "continuous",
|
|
{"timeout": 1, "duration": 40, "active_users": 3,
|
|
"tenants": 5, "users_per_tenant": 2})
|
|
]
|
|
self.assertEqual(srunner._run_scenario.mock_calls, expected)
|
|
|
|
expected = [
|
|
mock.call(1, 1),
|
|
mock.call(5, 2),
|
|
mock.call(5, 2)
|
|
]
|
|
self.assertEqual(srunner._create_temp_tenants_and_users.mock_calls,
|
|
expected)
|
|
|
|
expected = [
|
|
mock.call.init({}),
|
|
mock.call.init({"arg": 1}),
|
|
mock.call.init({"fake": "arg"}),
|
|
]
|
|
# NOTE(olkonami): Ignore __iter__ calls in loop
|
|
mock_calls = filter(lambda call: '__iter__' not in call[0],
|
|
FakeScenario.mock_calls)
|
|
self.assertEqual(mock_calls, expected)
|
|
|
|
@mock.patch("rally.benchmark.utils.create_openstack_clients")
|
|
@mock.patch("rally.benchmark.runner.base")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
def test_run_validation_failure(self, mock_osclients, mock_base,
|
|
mock_clients):
|
|
def evil_validator(**kwargs):
|
|
return validation.ValidationResult(is_valid=False)
|
|
|
|
FakeScenario, srunner = self._set_mocks_for_run(mock_osclients,
|
|
mock_base,
|
|
mock_clients,
|
|
[evil_validator])
|
|
self.assertRaises(exceptions.InvalidScenarioArgument,
|
|
srunner.run, "FakeScenario.do_it", {})
|
|
|
|
@mock.patch("rally.benchmark.utils.create_openstack_clients")
|
|
@mock.patch("rally.benchmark.runner.base")
|
|
@mock.patch("rally.benchmark.utils.osclients")
|
|
@mock.patch("rally.benchmark.utils.delete_keystone_resources")
|
|
@mock.patch("multiprocessing.Pool")
|
|
def test_generic_cleanup(self, mock_pool, mock_del_keystone_res,
|
|
mock_osclients, mock_base, mock_clients):
|
|
FakeScenario = mock.MagicMock()
|
|
FakeScenario.init = mock.MagicMock(return_value={})
|
|
|
|
mock_cms = [fakes.FakeClients(), fakes.FakeClients(),
|
|
fakes.FakeClients()]
|
|
clients = [
|
|
dict((
|
|
("nova", cl.get_nova_client()),
|
|
("keystone", cl.get_keystone_client()),
|
|
("glance", cl.get_glance_client()),
|
|
("cinder", cl.get_cinder_client())
|
|
)) for cl in mock_cms
|
|
]
|
|
mock_clients.return_value = clients
|
|
|
|
srunner = runner.ScenarioRunner(mock.MagicMock(), self.fake_kw)
|
|
srunner._run_scenario = mock.MagicMock(return_value="result")
|
|
srunner._create_temp_tenants_and_users = mock.MagicMock(
|
|
return_value=[])
|
|
srunner._delete_temp_tenants_and_users = mock.MagicMock()
|
|
|
|
mock_base.Scenario.get_by_name = \
|
|
mock.MagicMock(return_value=FakeScenario)
|
|
|
|
for index in range(len(clients)):
|
|
client = clients[index]
|
|
nova = client["nova"]
|
|
cinder = client["cinder"]
|
|
for count in range(3):
|
|
uid = index + count
|
|
img = nova.images.create()
|
|
nova.servers.create("svr-%s" % (uid), img.uuid, index)
|
|
nova.keypairs.create("keypair-%s" % (uid))
|
|
nova.security_groups.create("secgroup-%s" % (uid))
|
|
nova.networks.create("net-%s" % (uid))
|
|
cinder.volumes.create("vol-%s" % (uid))
|
|
cinder.volume_types.create("voltype-%s" % (uid))
|
|
cinder.transfers.create("voltransfer-%s" % (uid))
|
|
cinder.volume_snapshots.create("snap-%s" % (uid))
|
|
cinder.backups.create("backup-%s" % (uid))
|
|
|
|
mock_pool.return_value = MockedPool()
|
|
|
|
srunner.run("FakeScenario.do_it",
|
|
{"args": {"a": 1}, "init": {"arg": 1},
|
|
"config": {"timeout": 1, "times": 2, "active_users": 3,
|
|
"tenants": 5, "users_per_tenant": 2}})
|
|
|
|
def _assert_purged(manager, resource_type):
|
|
resources = manager.list()
|
|
self.assertEqual([], resources, "%s not purged: %s" %
|
|
(resource_type, resources))
|
|
|
|
for client in clients:
|
|
nova = client["nova"]
|
|
cinder = client["cinder"]
|
|
_assert_purged(nova.servers, "servers")
|
|
_assert_purged(nova.keypairs, "key pairs")
|
|
_assert_purged(nova.security_groups, "security groups")
|
|
_assert_purged(nova.networks, "networks")
|
|
|
|
_assert_purged(cinder.volumes, "volumes")
|
|
_assert_purged(cinder.volume_types, "volume types")
|
|
_assert_purged(cinder.backups, "volume backups")
|
|
_assert_purged(cinder.transfers, "volume transfers")
|
|
_assert_purged(cinder.volume_snapshots, "volume snapshots")
|
|
|
|
for image in nova.images.list():
|
|
self.assertEqual("DELETED", image.status,
|
|
"image not purged: %s" % (image))
|
|
|
|
expected = [mock.call(clients[0]["keystone"], resource) for resource in
|
|
["users", "tenants", "services", "roles"]]
|
|
|
|
self.assertEqual(mock_del_keystone_res.mock_calls, expected)
|