Heat rally scenario tests

This patch contains next main changes:
 - two scenarios for heat project ("create_and_list_stack",
   "create_and_delete_stack").
 - three atomic action for heat project ("stack-list" , "stack-create",
   "stack-delete").
 - two task samples how to use new scenarios.
 - a little changes in utils for supporting heat stack_status property.

Change-Id: Ice2bd4dc2fcefb7351df69dd8463abc6f8d4fd24
This commit is contained in:
Sergey Kraynev 2014-04-16 10:34:03 -04:00
parent b40b67037c
commit 90c24f7dd6
15 changed files with 437 additions and 5 deletions

View File

@ -0,0 +1,17 @@
{
"HeatStacks.create_and_delete_stack": [
{
"runner": {
"type": "constant",
"times": 10,
"concurrency": 2
},
"context": {
"users": {
"tenants": 2,
"users_per_tenant": 3
}
}
}
]
}

View File

@ -0,0 +1,11 @@
---
HeatStacks.create_and_delete_stack:
-
runner:
type: "constant"
times: 10
concurrency: 2
context:
users:
tenants: 2
users_per_tenant: 3

View File

@ -0,0 +1,17 @@
{
"HeatStacks.create_and_list_stack": [
{
"runner": {
"type": "constant",
"times": 10,
"concurrency": 1
},
"context": {
"users": {
"tenants": 1,
"users_per_tenant": 1
}
}
}
]
}

View File

@ -0,0 +1,11 @@
---
HeatStacks.create_and_list_stack:
-
runner:
type: "constant"
times: 10
concurrency: 1
context:
users:
tenants: 1
users_per_tenant: 1

View File

@ -207,3 +207,25 @@
type: "constant" type: "constant"
times: 10 times: 10
concurrency: 10 concurrency: 10
HeatStacks.create_and_list_stack:
-
runner:
type: "constant"
times: 10
concurrency: 1
context:
users:
tenants: 1
users_per_tenant: 1
HeatStacks.create_and_delete_stack:
-
runner:
type: "constant"
times: 10
concurrency: 2
context:
users:
tenants: 2
users_per_tenant: 3

View File

@ -42,7 +42,7 @@ class ResourceCleaner(base.Context):
"items": { "items": {
"type": "string", "type": "string",
"enum": ["nova", "glance", "cinder", "enum": ["nova", "glance", "cinder",
"quotas", "neutron", "ceilometer"] "quotas", "neutron", "ceilometer", "heat"]
}, },
"uniqueItems": True "uniqueItems": True
} }
@ -68,7 +68,8 @@ class ResourceCleaner(base.Context):
"neutron": (utils.delete_neutron_resources, clients.neutron, "neutron": (utils.delete_neutron_resources, clients.neutron,
tenant_id), tenant_id),
"ceilometer": (utils.delete_ceilometer_resources, "ceilometer": (utils.delete_ceilometer_resources,
clients.ceilometer, tenant_id) clients.ceilometer, tenant_id),
"heat": (utils.delete_heat_resources, clients.heat)
} }
for service_name in self.config: for service_name in self.config:

View File

@ -32,6 +32,10 @@ def delete_glance_resources(glance, project_uuid):
delete_images(glance, project_uuid) delete_images(glance, project_uuid)
def delete_heat_resources(heat):
delete_stacks(heat)
def delete_keystone_resources(keystone): def delete_keystone_resources(keystone):
for resource in ["users", "tenants", "services", "roles"]: for resource in ["users", "tenants", "services", "roles"]:
_delete_single_keystone_resource_type(keystone, resource) _delete_single_keystone_resource_type(keystone, resource)
@ -57,6 +61,13 @@ def delete_quotas(admin_clients, project_uuid):
admin_clients.nova().quotas.delete(project_uuid) admin_clients.nova().quotas.delete(project_uuid)
def delete_stacks(heat):
for stack in heat.stacks.list():
stack.delete()
_wait_for_list_statuses(heat.stacks, statuses=["DELETE_COMPLETE"],
timeout=600, check_interval=3)
def delete_volumes(cinder): def delete_volumes(cinder):
for vol in cinder.volumes.list(): for vol in cinder.volumes.list():
vol.delete() vol.delete()
@ -144,7 +155,8 @@ def _wait_for_list_statuses(mgr, statuses, list_query=None,
def _list_statuses(mgr): def _list_statuses(mgr):
for resource in mgr.list(**list_query): for resource in mgr.list(**list_query):
if resource.status not in statuses: status = bench_utils.get_status(resource)
if status not in statuses:
return False return False
return True return True

View File

@ -0,0 +1,69 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios.heat import utils
class HeatStacks(utils.HeatScenario):
RESOURCE_NAME_PREFIX = "rally_stack_"
RESOURCE_NAME_LENGTH = 7
def _get_template_from_file(self, template_path):
template = None
if template_path:
try:
with open(template_path, "r") as f:
template = f.read()
except IOError:
raise IOError("Provided path '%(template_path)s' is not valid"
% {"template_path": template_path})
return template
@base.scenario(context={"cleanup": ["heat"],
"roles": ["heat_stack_owner"]})
def create_and_list_stack(self, template_path=None):
"""Test adding an stack and then listing all stacks.
This scenario is a very useful tool to measure
the "heat stack-create" and "heat stack-list" command performance.
:param template_path: path to template file. if it's None or incorrect,
will be used default empty template.
"""
stack_name = self._generate_random_name()
template = self._get_template_from_file(template_path)
self._create_stack(stack_name, template)
self._list_stacks()
@base.scenario(context={"cleanup": ["heat"],
"roles": ["heat_stack_owner"]})
def create_and_delete_stack(self, template_path=None):
"""Test adds and then deletes stack.
This scenario is a very useful tool to measure
the "heat stack-create" and "heat stack-delete" command performance.
:param template_path: path to template file. if it's None or incorrect,
will be used default empty template.
"""
stack_name = self._generate_random_name()
template = self._get_template_from_file(template_path)
stack = self._create_stack(stack_name, template)
self._delete_stack(stack)

View File

@ -0,0 +1,117 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from oslo.config import cfg
from rally.benchmark.scenarios import base
from rally.benchmark.scenarios import utils as scenario_utils
from rally.benchmark import utils as bench_utils
heat_benchmark_opts = [
cfg.FloatOpt('heat_stack_create_prepoll_delay',
default=2.0,
help='Time to sleep after creating a resource before '
'polling for it status'),
cfg.FloatOpt('heat_stack_create_timeout',
default=3600.0,
help='Time to wait for heat stack to be created.'),
cfg.FloatOpt('heat_stack_create_poll_interval',
default=1.0,
help='Interval between checks when waiting for stack '
'creation.'),
cfg.FloatOpt('heat_stack_delete_timeout',
default=3600.0,
help='Time to wait for heat stack to be deleted.'),
cfg.FloatOpt('heat_stack_delete_poll_interval',
default=1.0,
help='Interval between checks when waiting for stack '
'deletion.')
]
CONF = cfg.CONF
benchmark_group = cfg.OptGroup(name='benchmark', title='benchmark options')
CONF.register_opts(heat_benchmark_opts, group=benchmark_group)
def heat_resource_is(status):
"""Check status of stack."""
return lambda resource: resource.stack_status.upper() == status.upper()
class HeatScenario(base.Scenario):
default_template = "HeatTemplateFormatVersion: '2012-12-12'"
@scenario_utils.atomic_action_timer('heat.list_stacks')
def _list_stacks(self):
"""Return user stack list."""
return list(self.clients("heat").stacks.list())
@scenario_utils.atomic_action_timer('heat.create_stack')
def _create_stack(self, stack_name, template=None):
"""Create a new stack.
:param stack_name: string. Name for created stack.
:param template: optional parameter. Template with stack description.
returns: object of stack
"""
template = template or self.default_template
kw = {
"stack_name": stack_name,
"disable_rollback": True,
"parameters": {},
"template": template,
"files": {},
"environment": {}
}
# heat client returns body instead manager object, so we should
# get manager object using stack_id
stack_id = self.clients("heat").stacks.create(**kw)["stack"]["id"]
stack = self.clients("heat").stacks.get(stack_id)
time.sleep(CONF.benchmark.heat_stack_create_prepoll_delay)
stack = bench_utils.wait_for(
stack,
is_ready=heat_resource_is("CREATE_COMPLETE"),
update_resource=bench_utils.get_from_manager("CREATE_FAILED"),
timeout=CONF.benchmark.heat_stack_create_timeout,
check_interval=CONF.benchmark.heat_stack_create_poll_interval)
return stack
@scenario_utils.atomic_action_timer('heat.delete_stack')
def _delete_stack(self, stack):
"""Delete the given stack.
Returns when the stack is actually deleted.
:param stack: stack object
"""
stack.delete()
bench_utils.wait_for_delete(
stack,
update_resource=bench_utils.get_from_manager(),
timeout=CONF.benchmark.heat_stack_delete_timeout,
check_interval=CONF.benchmark.heat_stack_delete_poll_interval)

View File

@ -43,6 +43,14 @@ def resource_is(status):
return lambda resource: resource.status.upper() == status.upper() return lambda resource: resource.status.upper() == status.upper()
def get_status(resource):
# workaround for heat resources - using stack_status instead of status
if ((hasattr(resource, "stack_status") and
isinstance(resource.stack_status, basestring))):
return resource.stack_status.upper()
return resource.status.upper()
def get_from_manager(error_statuses=None): def get_from_manager(error_statuses=None):
error_statuses = error_statuses or ["ERROR"] error_statuses = error_statuses or ["ERROR"]
error_statuses = map(lambda str: str.upper(), error_statuses) error_statuses = map(lambda str: str.upper(), error_statuses)
@ -57,8 +65,9 @@ def get_from_manager(error_statuses=None):
raise exceptions.GetResourceFailure(resource=resource, err=e) raise exceptions.GetResourceFailure(resource=resource, err=e)
# catch abnormal status, such as "no valid host" for servers # catch abnormal status, such as "no valid host" for servers
status = res.status.upper() status = get_status(res)
if status == "DELETED":
if status in ("DELETED", "DELETE_COMPLETE"):
raise exceptions.GetResourceNotFound(resource=res) raise exceptions.GetResourceNotFound(resource=res)
if status in error_statuses: if status in error_statuses:
if isinstance(res.manager, servers.ServerManager): if isinstance(res.manager, servers.ServerManager):

View File

@ -0,0 +1,49 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from rally.benchmark.scenarios.heat import stacks
from tests import test
HEAT_STACKS = "rally.benchmark.scenarios.heat.stacks.HeatStacks"
class HeatStacksTestCase(test.TestCase):
@mock.patch(HEAT_STACKS + "._generate_random_name")
@mock.patch(HEAT_STACKS + "._list_stacks")
@mock.patch(HEAT_STACKS + "._create_stack")
def test_create_and_list_stack(self, mock_create, mock_list,
mock_random_name):
heat_scenario = stacks.HeatStacks()
mock_random_name.return_value = "test-rally-stack"
heat_scenario.create_and_list_stack()
self.assertEqual(1, mock_create.called)
mock_list.assert_called_once_with()
@mock.patch(HEAT_STACKS + "._generate_random_name")
@mock.patch(HEAT_STACKS + "._delete_stack")
@mock.patch(HEAT_STACKS + "._create_stack")
def test_create_and_delete_stack(self, mock_create, mock_delete,
mock_random_name):
heat_scenario = stacks.HeatStacks()
fake_stack = object()
mock_create.return_value = fake_stack
mock_random_name.return_value = "test-rally-stack"
heat_scenario.create_and_delete_stack()
self.assertEqual(1, mock_create.called)
mock_delete.assert_called_once_with(fake_stack)

View File

@ -0,0 +1,88 @@
# Copyright 2014: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from oslotest import mockpatch
from rally.benchmark.scenarios.heat import utils
from tests.benchmark.scenarios import test_utils
from tests import test
BM_UTILS = 'rally.benchmark.utils'
HEAT_UTILS = 'rally.benchmark.scenarios.heat.utils'
class HeatScenarioTestCase(test.TestCase):
def setUp(self):
super(HeatScenarioTestCase, self).setUp()
self.stack = mock.Mock()
self.res_is = mockpatch.Patch(HEAT_UTILS + ".heat_resource_is")
self.get_fm = mockpatch.Patch(BM_UTILS + '.get_from_manager')
self.wait_for = mockpatch.Patch(HEAT_UTILS + ".bench_utils.wait_for")
self.wait_for_delete = mockpatch.Patch(
HEAT_UTILS + ".bench_utils.wait_for_delete")
self.useFixture(self.wait_for)
self.useFixture(self.wait_for_delete)
self.useFixture(self.res_is)
self.useFixture(self.get_fm)
self.gfm = self.get_fm.mock
self.useFixture(mockpatch.Patch('time.sleep'))
self.scenario = utils.HeatScenario()
def _test_atomic_action_timer(self, atomic_actions, name):
action_duration = test_utils.get_atomic_action_timer_value_by_name(
atomic_actions, name)
self.assertIsNotNone(action_duration)
self.assertIsInstance(action_duration, float)
@mock.patch(HEAT_UTILS + '.HeatScenario.clients')
def test_list_stacks(self, mock_clients):
stacks_list = []
mock_clients("heat").stacks.list.return_value = stacks_list
scenario = utils.HeatScenario()
return_stacks_list = scenario._list_stacks()
self.assertEqual(stacks_list, return_stacks_list)
self._test_atomic_action_timer(scenario.atomic_actions(),
'heat.list_stacks')
@mock.patch(HEAT_UTILS + '.HeatScenario.clients')
def test_create_stack(self, mock_clients):
mock_clients("heat").stacks.create.return_value = \
{'stack': {'id': 'test_id'}}
mock_clients("heat").stacks.get.return_value = self.stack
scenario = utils.HeatScenario()
return_stack = scenario._create_stack('stack_name')
self.wait_for.mock.assert_called_once_with(self.stack,
update_resource=self.gfm(),
is_ready=self.res_is.mock(),
check_interval=1,
timeout=3600)
self.res_is.mock.assert_has_calls(mock.call('CREATE_COMPLETE'))
self.assertEqual(self.wait_for.mock(), return_stack)
self._test_atomic_action_timer(scenario.atomic_actions(),
'heat.create_stack')
def test_delete_stack(self):
scenario = utils.HeatScenario()
scenario._delete_stack(self.stack)
self.stack.delete.assert_called_once_with()
self.wait_for_delete.\
mock.assert_called_once_with(self.stack,
update_resource=self.gfm(),
check_interval=1,
timeout=3600)
self._test_atomic_action_timer(scenario.atomic_actions(),
'heat.delete_stack')

View File

@ -82,6 +82,15 @@ class BenchmarkUtilsTestCase(test.TestCase):
self.assertRaises(exceptions.GetResourceNotFound, self.assertRaises(exceptions.GetResourceNotFound,
get_from_manager, resource) get_from_manager, resource)
def test_get_from_manager_in_deleted_state_for_heat_resource(self):
get_from_manager = utils.get_from_manager()
manager = fakes.FakeManager()
resource = fakes.FakeResource(manager=manager)
resource.stack_status = "DELETE_COMPLETE"
manager._cache(resource)
self.assertRaises(exceptions.GetResourceNotFound,
get_from_manager, resource)
def test_get_from_manager_not_found(self): def test_get_from_manager_not_found(self):
get_from_manager = utils.get_from_manager() get_from_manager = utils.get_from_manager()
manager = mock.MagicMock() manager = mock.MagicMock()