Heat rally scenario tests

This patch contains next main changes:
 - two scenarios for heat project ("create_and_list_stack",
   "create_and_delete_stack").
 - three atomic action for heat project ("stack-list" , "stack-create",
   "stack-delete").
 - two task samples how to use new scenarios.
 - a little changes in utils for supporting heat stack_status property.

Change-Id: Ice2bd4dc2fcefb7351df69dd8463abc6f8d4fd24

doc/samples/tasks/heat/create-and-delete-stack.json

@@ -0,0 +1,17 @@
+{
+    "HeatStacks.create_and_delete_stack": [
+        {
+            "runner": {
+                "type": "constant",
+                "times": 10,
+                "concurrency": 2
+            },
+            "context": {
+                "users": {
+                    "tenants": 2,
+                    "users_per_tenant": 3
+                }
+            }
+        }
+    ]
+}

doc/samples/tasks/heat/create-and-delete-stack.yaml

@@ -0,0 +1,11 @@
+---
+  HeatStacks.create_and_delete_stack:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 3

doc/samples/tasks/heat/create-and-list-stack.json

@@ -0,0 +1,17 @@
+{
+    "HeatStacks.create_and_list_stack": [
+        {
+            "runner": {
+                "type": "constant",
+                "times": 10,
+                "concurrency": 1
+            },
+            "context": {
+                "users": {
+                    "tenants": 1,
+                    "users_per_tenant": 1
+                }
+            }
+        }
+    ]
+}

doc/samples/tasks/heat/create-and-list-stack.yaml

@@ -0,0 +1,11 @@
+---
+  HeatStacks.create_and_list_stack:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 1
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1

rally-scenarios/rally.yaml

@@ -207,3 +207,25 @@
         type: "constant"
         times: 10
         concurrency: 10
+
+  HeatStacks.create_and_list_stack:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 1
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+
+  HeatStacks.create_and_delete_stack:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 3

rally/benchmark/context/cleanup/cleanup.py

@@ -42,7 +42,7 @@ class ResourceCleaner(base.Context):
         "items": {
             "type": "string",
             "enum": ["nova", "glance", "cinder",
-                     "quotas", "neutron", "ceilometer"]
+                     "quotas", "neutron", "ceilometer", "heat"]
         },
         "uniqueItems": True
     }
@@ -68,7 +68,8 @@ class ResourceCleaner(base.Context):
                 "neutron": (utils.delete_neutron_resources, clients.neutron,
                             tenant_id),
                 "ceilometer": (utils.delete_ceilometer_resources,
-                               clients.ceilometer, tenant_id)
+                               clients.ceilometer, tenant_id),
+                "heat": (utils.delete_heat_resources, clients.heat)
             }
 
             for service_name in self.config:

rally/benchmark/context/cleanup/utils.py

@@ -32,6 +32,10 @@ def delete_glance_resources(glance, project_uuid):
     delete_images(glance, project_uuid)
 
 
+def delete_heat_resources(heat):
+    delete_stacks(heat)
+
+
 def delete_keystone_resources(keystone):
     for resource in ["users", "tenants", "services", "roles"]:
         _delete_single_keystone_resource_type(keystone, resource)
@@ -57,6 +61,13 @@ def delete_quotas(admin_clients, project_uuid):
     admin_clients.nova().quotas.delete(project_uuid)
 
 
+def delete_stacks(heat):
+    for stack in heat.stacks.list():
+        stack.delete()
+    _wait_for_list_statuses(heat.stacks, statuses=["DELETE_COMPLETE"],
+                            timeout=600, check_interval=3)
+
+
 def delete_volumes(cinder):
     for vol in cinder.volumes.list():
         vol.delete()
@@ -144,7 +155,8 @@ def _wait_for_list_statuses(mgr, statuses, list_query=None,
 
     def _list_statuses(mgr):
         for resource in mgr.list(**list_query):
-            if resource.status not in statuses:
+            status = bench_utils.get_status(resource)
+            if status not in statuses:
                 return False
         return True
 

rally/benchmark/scenarios/heat/stacks.py

@@ -0,0 +1,69 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally.benchmark.scenarios import base
+from rally.benchmark.scenarios.heat import utils
+
+
+class HeatStacks(utils.HeatScenario):
+
+    RESOURCE_NAME_PREFIX = "rally_stack_"
+    RESOURCE_NAME_LENGTH = 7
+
+    def _get_template_from_file(self, template_path):
+        template = None
+        if template_path:
+            try:
+                with open(template_path, "r") as f:
+                    template = f.read()
+            except IOError:
+                raise IOError("Provided path '%(template_path)s' is not valid"
+                              % {"template_path": template_path})
+        return template
+
+    @base.scenario(context={"cleanup": ["heat"],
+                            "roles": ["heat_stack_owner"]})
+    def create_and_list_stack(self, template_path=None):
+        """Test adding an stack and then listing all stacks.
+
+        This scenario is a very useful tool to measure
+        the "heat stack-create" and "heat stack-list" command performance.
+
+        :param template_path: path to template file. if it's None or incorrect,
+                              will be used default empty template.
+        """
+
+        stack_name = self._generate_random_name()
+        template = self._get_template_from_file(template_path)
+
+        self._create_stack(stack_name, template)
+        self._list_stacks()
+
+    @base.scenario(context={"cleanup": ["heat"],
+                            "roles": ["heat_stack_owner"]})
+    def create_and_delete_stack(self, template_path=None):
+        """Test adds and then deletes stack.
+
+        This scenario is a very useful tool to measure
+        the "heat stack-create" and "heat stack-delete" command performance.
+
+        :param template_path: path to template file. if it's None or incorrect,
+                              will be used default empty template.
+        """
+        stack_name = self._generate_random_name()
+        template = self._get_template_from_file(template_path)
+
+        stack = self._create_stack(stack_name, template)
+        self._delete_stack(stack)

rally/benchmark/scenarios/heat/utils.py

@@ -0,0 +1,117 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import time
+
+from oslo.config import cfg
+
+from rally.benchmark.scenarios import base
+from rally.benchmark.scenarios import utils as scenario_utils
+from rally.benchmark import utils as bench_utils
+
+
+heat_benchmark_opts = [
+    cfg.FloatOpt('heat_stack_create_prepoll_delay',
+                 default=2.0,
+                 help='Time to sleep after creating a resource before '
+                      'polling for it status'),
+    cfg.FloatOpt('heat_stack_create_timeout',
+                 default=3600.0,
+                 help='Time to wait for heat stack to be created.'),
+    cfg.FloatOpt('heat_stack_create_poll_interval',
+                 default=1.0,
+                 help='Interval between checks when waiting for stack '
+                      'creation.'),
+    cfg.FloatOpt('heat_stack_delete_timeout',
+                 default=3600.0,
+                 help='Time to wait for heat stack to be deleted.'),
+    cfg.FloatOpt('heat_stack_delete_poll_interval',
+                 default=1.0,
+                 help='Interval between checks when waiting for stack '
+                      'deletion.')
+]
+
+
+CONF = cfg.CONF
+benchmark_group = cfg.OptGroup(name='benchmark', title='benchmark options')
+CONF.register_opts(heat_benchmark_opts, group=benchmark_group)
+
+
+def heat_resource_is(status):
+    """Check status of stack."""
+
+    return lambda resource: resource.stack_status.upper() == status.upper()
+
+
+class HeatScenario(base.Scenario):
+
+    default_template = "HeatTemplateFormatVersion: '2012-12-12'"
+
+    @scenario_utils.atomic_action_timer('heat.list_stacks')
+    def _list_stacks(self):
+        """Return user stack list."""
+
+        return list(self.clients("heat").stacks.list())
+
+    @scenario_utils.atomic_action_timer('heat.create_stack')
+    def _create_stack(self, stack_name, template=None):
+        """Create a new stack.
+
+        :param stack_name: string. Name for created stack.
+        :param template: optional parameter. Template with stack description.
+
+        returns: object of stack
+        """
+        template = template or self.default_template
+
+        kw = {
+            "stack_name": stack_name,
+            "disable_rollback": True,
+            "parameters": {},
+            "template": template,
+            "files": {},
+            "environment": {}
+        }
+
+        # heat client returns body instead manager object, so we should
+        # get manager object using stack_id
+        stack_id = self.clients("heat").stacks.create(**kw)["stack"]["id"]
+        stack = self.clients("heat").stacks.get(stack_id)
+
+        time.sleep(CONF.benchmark.heat_stack_create_prepoll_delay)
+
+        stack = bench_utils.wait_for(
+            stack,
+            is_ready=heat_resource_is("CREATE_COMPLETE"),
+            update_resource=bench_utils.get_from_manager("CREATE_FAILED"),
+            timeout=CONF.benchmark.heat_stack_create_timeout,
+            check_interval=CONF.benchmark.heat_stack_create_poll_interval)
+
+        return stack
+
+    @scenario_utils.atomic_action_timer('heat.delete_stack')
+    def _delete_stack(self, stack):
+        """Delete the given stack.
+
+        Returns when the stack is actually deleted.
+
+        :param stack: stack object
+        """
+        stack.delete()
+        bench_utils.wait_for_delete(
+            stack,
+            update_resource=bench_utils.get_from_manager(),
+            timeout=CONF.benchmark.heat_stack_delete_timeout,
+            check_interval=CONF.benchmark.heat_stack_delete_poll_interval)

rally/benchmark/utils.py

@@ -43,6 +43,14 @@ def resource_is(status):
     return lambda resource: resource.status.upper() == status.upper()
 
 
+def get_status(resource):
+    # workaround for heat resources - using stack_status instead of status
+    if ((hasattr(resource, "stack_status") and
+         isinstance(resource.stack_status, basestring))):
+        return resource.stack_status.upper()
+    return resource.status.upper()
+
+
 def get_from_manager(error_statuses=None):
     error_statuses = error_statuses or ["ERROR"]
     error_statuses = map(lambda str: str.upper(), error_statuses)
@@ -57,8 +65,9 @@ def get_from_manager(error_statuses=None):
             raise exceptions.GetResourceFailure(resource=resource, err=e)
 
         # catch abnormal status, such as "no valid host" for servers
-        status = res.status.upper()
-        if status == "DELETED":
+        status = get_status(res)
+
+        if status in ("DELETED", "DELETE_COMPLETE"):
             raise exceptions.GetResourceNotFound(resource=res)
         if status in error_statuses:
             if isinstance(res.manager, servers.ServerManager):

tests/benchmark/scenarios/heat/test_stacks.py

@@ -0,0 +1,49 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally.benchmark.scenarios.heat import stacks
+from tests import test
+
+HEAT_STACKS = "rally.benchmark.scenarios.heat.stacks.HeatStacks"
+
+
+class HeatStacksTestCase(test.TestCase):
+
+    @mock.patch(HEAT_STACKS + "._generate_random_name")
+    @mock.patch(HEAT_STACKS + "._list_stacks")
+    @mock.patch(HEAT_STACKS + "._create_stack")
+    def test_create_and_list_stack(self, mock_create, mock_list,
+                                   mock_random_name):
+        heat_scenario = stacks.HeatStacks()
+        mock_random_name.return_value = "test-rally-stack"
+        heat_scenario.create_and_list_stack()
+        self.assertEqual(1, mock_create.called)
+        mock_list.assert_called_once_with()
+
+    @mock.patch(HEAT_STACKS + "._generate_random_name")
+    @mock.patch(HEAT_STACKS + "._delete_stack")
+    @mock.patch(HEAT_STACKS + "._create_stack")
+    def test_create_and_delete_stack(self, mock_create, mock_delete,
+                                     mock_random_name):
+        heat_scenario = stacks.HeatStacks()
+        fake_stack = object()
+        mock_create.return_value = fake_stack
+        mock_random_name.return_value = "test-rally-stack"
+        heat_scenario.create_and_delete_stack()
+
+        self.assertEqual(1, mock_create.called)
+        mock_delete.assert_called_once_with(fake_stack)

tests/benchmark/scenarios/heat/test_utils.py

@@ -0,0 +1,88 @@
+# Copyright 2014: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+from oslotest import mockpatch
+
+from rally.benchmark.scenarios.heat import utils
+from tests.benchmark.scenarios import test_utils
+from tests import test
+
+BM_UTILS = 'rally.benchmark.utils'
+HEAT_UTILS = 'rally.benchmark.scenarios.heat.utils'
+
+
+class HeatScenarioTestCase(test.TestCase):
+
+    def setUp(self):
+        super(HeatScenarioTestCase, self).setUp()
+        self.stack = mock.Mock()
+        self.res_is = mockpatch.Patch(HEAT_UTILS + ".heat_resource_is")
+        self.get_fm = mockpatch.Patch(BM_UTILS + '.get_from_manager')
+        self.wait_for = mockpatch.Patch(HEAT_UTILS + ".bench_utils.wait_for")
+        self.wait_for_delete = mockpatch.Patch(
+            HEAT_UTILS + ".bench_utils.wait_for_delete")
+        self.useFixture(self.wait_for)
+        self.useFixture(self.wait_for_delete)
+        self.useFixture(self.res_is)
+        self.useFixture(self.get_fm)
+        self.gfm = self.get_fm.mock
+        self.useFixture(mockpatch.Patch('time.sleep'))
+        self.scenario = utils.HeatScenario()
+
+    def _test_atomic_action_timer(self, atomic_actions, name):
+        action_duration = test_utils.get_atomic_action_timer_value_by_name(
+            atomic_actions, name)
+        self.assertIsNotNone(action_duration)
+        self.assertIsInstance(action_duration, float)
+
+    @mock.patch(HEAT_UTILS + '.HeatScenario.clients')
+    def test_list_stacks(self, mock_clients):
+        stacks_list = []
+        mock_clients("heat").stacks.list.return_value = stacks_list
+        scenario = utils.HeatScenario()
+        return_stacks_list = scenario._list_stacks()
+        self.assertEqual(stacks_list, return_stacks_list)
+        self._test_atomic_action_timer(scenario.atomic_actions(),
+                                       'heat.list_stacks')
+
+    @mock.patch(HEAT_UTILS + '.HeatScenario.clients')
+    def test_create_stack(self, mock_clients):
+        mock_clients("heat").stacks.create.return_value = \
+            {'stack': {'id': 'test_id'}}
+        mock_clients("heat").stacks.get.return_value = self.stack
+        scenario = utils.HeatScenario()
+        return_stack = scenario._create_stack('stack_name')
+        self.wait_for.mock.assert_called_once_with(self.stack,
+                                                   update_resource=self.gfm(),
+                                                   is_ready=self.res_is.mock(),
+                                                   check_interval=1,
+                                                   timeout=3600)
+        self.res_is.mock.assert_has_calls(mock.call('CREATE_COMPLETE'))
+        self.assertEqual(self.wait_for.mock(), return_stack)
+        self._test_atomic_action_timer(scenario.atomic_actions(),
+                                       'heat.create_stack')
+
+    def test_delete_stack(self):
+        scenario = utils.HeatScenario()
+        scenario._delete_stack(self.stack)
+        self.stack.delete.assert_called_once_with()
+        self.wait_for_delete.\
+            mock.assert_called_once_with(self.stack,
+                                         update_resource=self.gfm(),
+                                         check_interval=1,
+                                         timeout=3600)
+        self._test_atomic_action_timer(scenario.atomic_actions(),
+                                       'heat.delete_stack')

tests/benchmark/test_utils.py

@@ -82,6 +82,15 @@ class BenchmarkUtilsTestCase(test.TestCase):
         self.assertRaises(exceptions.GetResourceNotFound,
                           get_from_manager, resource)
 
+    def test_get_from_manager_in_deleted_state_for_heat_resource(self):
+        get_from_manager = utils.get_from_manager()
+        manager = fakes.FakeManager()
+        resource = fakes.FakeResource(manager=manager)
+        resource.stack_status = "DELETE_COMPLETE"
+        manager._cache(resource)
+        self.assertRaises(exceptions.GetResourceNotFound,
+                          get_from_manager, resource)
+
     def test_get_from_manager_not_found(self):
         get_from_manager = utils.get_from_manager()
         manager = mock.MagicMock()