Add Mistral benchmark

- added Mistral client initialization
- added Mistral scenarios
- added unit tests

Change-Id: Ie60a36504a970e6acb369adc570a556e549c20e1

optional-requirements.txt

@@ -0,0 +1 @@
+git+git://github.com/stackforge/python-mistralclient.git

rally-jobs/extra/mistral_wb.yaml

@@ -0,0 +1,13 @@
+---
+version: "2.0"
+
+name: wb
+
+workflows:
+  wf1:
+    type: direct
+    tasks:
+      hello:
+        action: std.echo output="Hello"
+        publish:
+            result: $

rally-jobs/rally-mistral.yaml

@@ -0,0 +1,46 @@
+---
+  MistralWorkbooks.list_workbooks:
+    -
+      runner:
+        type: "constant"
+        times: 50
+        concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+      sla:
+        failure_rate:
+          max: 0
+
+  MistralWorkbooks.create_workbook:
+    -
+      args:
+        definition: "/home/jenkins/.rally/extra/mistral_wb.yaml"
+      runner:
+        type: "constant"
+        times: 50
+        concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+      sla:
+        failure_rate:
+          max: 0
+
+    -
+      args:
+        definition: "/home/jenkins/.rally/extra/mistral_wb.yaml"
+        do_delete: true
+      runner:
+        type: "constant"
+        times: 50
+        concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+      sla:
+        failure_rate:
+          max: 0

rally/benchmark/context/cleanup/resources.py

@@ -284,6 +284,14 @@ class DesignateServer(SynchronizedDeletion, base.ResourceManager):
     pass
 
 
+# MISTRAL
+
+@base.resource("mistral", "workbooks", order=1100, tenant_resource=True)
+class MistralWorkbooks(SynchronizedDeletion, base.ResourceManager):
+    def delete(self):
+        self._manager().delete(self.raw_resource.name)
+
+
 # KEYSTONE
 
 _keystone_order = get_order(9000)

rally/benchmark/scenarios/mistral/utils.py

@@ -0,0 +1,49 @@
+# Copyright 2015: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import yaml
+
+from rally.benchmark.scenarios import base
+
+
+class MistralScenario(base.Scenario):
+    """Base class for Mistral scenarios with basic atomic actions."""
+
+    @base.atomic_action_timer("mistral.list_workbooks")
+    def _list_workbooks(self):
+        """Gets list of existing workbooks."""
+        return self.clients("mistral").workbooks.list()
+
+    @base.atomic_action_timer("mistral.create_workbook")
+    def _create_workbook(self, definition):
+        """Create a new workbook.
+
+        :param definition: workbook description in string
+                           (yaml string) format
+        :returns: workbook object
+        """
+        definition = yaml.safe_load(definition)
+        definition["name"] = self._generate_random_name(definition["name"])
+        definition = yaml.safe_dump(definition)
+
+        return self.clients("mistral").workbooks.create(definition)
+
+    @base.atomic_action_timer("mistral.delete_workbook")
+    def _delete_workbook(self, wb_name):
+        """Delete the given workbook.
+
+        :param wb_name: the name of workbook that would be deleted.
+        """
+        self.clients("mistral").workbooks.delete(wb_name)

rally/benchmark/scenarios/mistral/workbooks.py

@@ -0,0 +1,59 @@
+# Copyright 2015: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally.benchmark.scenarios import base
+from rally.benchmark.scenarios.mistral import utils
+from rally.benchmark import types as types
+from rally.benchmark import validation
+from rally import consts
+
+
+class MistralWorkbooks(utils.MistralScenario):
+    """Benchmark scenarios for Mistral workbook."""
+
+    @validation.required_clients("mistral")
+    @validation.required_openstack(users=True)
+    @validation.required_services(consts.Service.MISTRAL)
+    @base.scenario()
+    def list_workbooks(self):
+        """Scenario test mistral workbook-list command.
+
+        This simple scenario tests the Mistral workbook-list
+        command by listing all the workbooks.
+        """
+        self._list_workbooks()
+
+    @types.set(definition=types.FileType)
+    @validation.file_exists("definition")
+    @validation.required_parameters("definition")
+    @validation.required_clients("mistral")
+    @validation.required_openstack(users=True)
+    @validation.required_services(consts.Service.MISTRAL)
+    @base.scenario(context={"cleanup": ["mistral"]})
+    def create_workbook(self, definition, do_delete=False):
+        """Scenario tests workbook creation and deletion.
+
+        This scenario is a very useful tool to measure the
+        "mistral workbook-create" and "mistral workbook-delete"
+        commands performance.
+        :param definition: string (yaml string) representation of given
+                           file content (Mistral workbook definition)
+        :param do_delete: if False than it allows to check performance
+                          in "create only" mode.
+        """
+        wb = self._create_workbook(definition)
+
+        if do_delete:
+            self._delete_workbook(wb.name)

rally/benchmark/types.py

@@ -236,3 +236,19 @@ class NeutronNetworkResourceType(ResourceType):
         raise exceptions.InvalidScenarioArgument(
             "Neutron network with name '{name}' not found".format(
                 name=resource_config.get("name")))
+
+
+class FileType(ResourceType):
+
+    @classmethod
+    def transform(cls, clients, resource_config):
+        """Returns content of the file by its path.
+
+        :param clients: openstack admin client handles
+        :param resource_config: path to file
+
+        :returns: content of the file
+        """
+
+        with open(resource_config, "r") as f:
+            return f.read()

rally/benchmark/validation.py

@@ -376,6 +376,23 @@ def required_services(config, clients, deployment, *required_services):
                 False, _("Service is not available: %s") % service)
 
 
+@validator
+def required_clients(config, clients, task, *components):
+    """Validator checks if specified OpenStack clients are available.
+
+    :param *components: list of client components names
+    """
+    for client_component in components:
+        try:
+            getattr(clients, client_component)()
+        except ImportError:
+            return ValidationResult(
+                False,
+                _("Client for %s is not installed. To install it run "
+                  "`pip install -r"
+                  " optional-requirements.txt`") % client_component)
+
+
 @validator
 def required_contexts(config, clients, deployment, *context_names):
     """Validator hecks if required benchmark contexts are specified.

rally/consts.py

@@ -108,6 +108,7 @@ class _Service(utils.ImmutableMixin, utils.EnumMixin):
     TROVE = "trove"
     SAHARA = "sahara"
     SWIFT = "swift"
+    MISTRAL = "mistral"
 
 
 class _ServiceType(utils.ImmutableMixin, utils.EnumMixin):
@@ -130,6 +131,7 @@ class _ServiceType(utils.ImmutableMixin, utils.EnumMixin):
     DATABASE = "database"
     DATA_PROCESSING = "data_processing"
     OBJECT_STORE = "object-store"
+    WORKFLOW_EXECUTION = "workflowv2"
 
     def __init__(self):
         self.__names = {
@@ -149,7 +151,8 @@ class _ServiceType(utils.ImmutableMixin, utils.EnumMixin):
             self.S3: _Service.S3,
             self.DATABASE: _Service.TROVE,
             self.DATA_PROCESSING: _Service.SAHARA,
-            self.OBJECT_STORE: _Service.SWIFT
+            self.OBJECT_STORE: _Service.SWIFT,
+            self.WORKFLOW_EXECUTION: _Service.MISTRAL,
         }
 
     def __getitem__(self, service_type):

rally/osclients.py

@@ -301,6 +301,23 @@ class Clients(object):
                               cacert=CONF.https_cacert)
         return client
 
+    @cached
+    def mistral(self):
+        """Return Mistral client."""
+        from mistralclient.api import client
+        kc = self.keystone()
+
+        mistral_url = kc.service_catalog.url_for(
+            service_type="workflowv2",
+            endpoint_type=self.endpoint.endpoint_type,
+            region_name=self.endpoint.region_name)
+
+        client = client.client(mistral_url=mistral_url,
+                               service_type="workflowv2",
+                               auth_token=kc.auth_token)
+
+        return client
+
     @cached
     def services(self):
         """Return available services names and types.

samples/tasks/scenarios/mistral/create-delete-workbook.json

@@ -0,0 +1,21 @@
+{
+    "MistralWorkbooks.create_workbook": [
+        {
+            "args": {
+                "definition": "rally-jobs/extra/mistral_wb.yaml",
+                "do_delete": true
+            },
+            "runner": {
+                "type": "constant",
+                "times": 50,
+                "concurrency": 10
+            },
+            "context": {
+                "users": {
+                    "tenants": 1,
+                    "users_per_tenant": 1
+                }
+            }
+        }
+    ]
+}

samples/tasks/scenarios/mistral/create-delete-workbook.yaml

@@ -0,0 +1,15 @@
+---
+  MistralWorkbooks.create_workbook:
+    -
+      args:
+        definition: rally-jobs/extra/mistral_wb.yaml
+        do_delete: true
+      runner:
+        type: "constant"
+        times: 50
+        concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+

samples/tasks/scenarios/mistral/create-workbook.json

@@ -0,0 +1,20 @@
+{
+    "MistralWorkbooks.create_workbook": [
+        {
+            "args": {
+                "definition": "rally-jobs/extra/mistral_wb.yaml"
+            },
+            "runner": {
+                "type": "constant",
+                "times": 50,
+                "concurrency": 10
+            },
+            "context": {
+                "users": {
+                    "tenants": 1,
+                    "users_per_tenant": 1
+                }
+            }
+        }
+    ]
+}

samples/tasks/scenarios/mistral/create-workbook.yaml

@@ -0,0 +1,14 @@
+---
+  MistralWorkbooks.create_workbook:
+    -
+      args:
+        definition: rally-jobs/extra/mistral_wb.yaml
+      runner:
+        type: "constant"
+        times: 50
+        concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+

samples/tasks/scenarios/mistral/list-workbooks.json

@@ -0,0 +1,17 @@
+{
+    "MistralWorkbooks.list_workbooks": [
+        {
+            "runner": {
+                "type": "constant",
+                "times": 50,
+                "concurrency": 10
+            },
+            "context": {
+                "users": {
+                    "tenants": 1,
+                    "users_per_tenant": 1
+                }
+            }
+        }
+    ]
+}

samples/tasks/scenarios/mistral/list-workbooks.yaml

@@ -0,0 +1,11 @@
+---
+  MistralWorkbooks.list_workbooks:
+    -
+      runner:
+        type: "constant"
+        times: 50
+        concurrency: 10
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1

tests/unit/benchmark/scenarios/mistral/test_utils.py

@@ -0,0 +1,66 @@
+# Copyright 2015: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally.benchmark.scenarios.mistral import utils
+from tests.unit import test
+
+BM_UTILS = "rally.benchmark.utils"
+MISTRAL_UTILS = "rally.benchmark.scenarios.mistral.utils"
+
+
+class MistralScenarioTestCase(test.TestCase):
+
+    def setUp(self):
+        super(MistralScenarioTestCase, self).setUp()
+
+    def _test_atomic_action_timer(self, atomic_actions, name):
+        action_duration = atomic_actions.get(name)
+        self.assertIsNotNone(action_duration)
+        self.assertIsInstance(action_duration, float)
+
+    @mock.patch(MISTRAL_UTILS + ".MistralScenario.clients")
+    def test_list_workbooks(self, mock_clients):
+        wbs_list = []
+        mock_clients("mistral").workbooks.list.return_value = wbs_list
+        scenario = utils.MistralScenario()
+        return_wbs_list = scenario._list_workbooks()
+        self.assertEqual(wbs_list, return_wbs_list)
+        self._test_atomic_action_timer(scenario.atomic_actions(),
+                                       "mistral.list_workbooks")
+
+    @mock.patch(MISTRAL_UTILS + ".MistralScenario.clients")
+    def test_create_workbook(self, mock_clients):
+        definition = "version: \"2.0\"\nname: wb"
+        mock_clients("mistral").workbooks.create.return_value = "wb"
+        scenario = utils.MistralScenario()
+        wb = scenario._create_workbook(definition)
+        self.assertEqual("wb", wb)
+        self._test_atomic_action_timer(scenario.atomic_actions(),
+                                       "mistral.create_workbook")
+
+    @mock.patch(MISTRAL_UTILS + ".MistralScenario.clients")
+    def test_delete_workbook(self, mock_clients):
+        wb = mock.Mock()
+        wb.name = "wb"
+        mock_clients("mistral").workbooks.delete.return_value = "ok"
+        scenario = utils.MistralScenario()
+        scenario._delete_workbook(wb.name)
+        mock_clients("mistral").workbooks.delete.assert_called_once_with(
+            wb.name
+        )
+        self._test_atomic_action_timer(scenario.atomic_actions(),
+                                       "mistral.delete_workbook")

tests/unit/benchmark/scenarios/mistral/test_workbooks.py

@@ -0,0 +1,54 @@
+# Copyright 2015: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from rally.benchmark.scenarios.mistral import workbooks
+from tests.unit import test
+
+MISTRAL_WBS = "rally.benchmark.scenarios.mistral.workbooks.MistralWorkbooks"
+
+
+class MistralWorkbooksTestCase(test.TestCase):
+
+    @mock.patch(MISTRAL_WBS + "._list_workbooks")
+    def test_list_workbooks(self, mock_list):
+        mistral_scenario = workbooks.MistralWorkbooks()
+        mistral_scenario.list_workbooks()
+        mock_list.assert_called_once_with()
+
+    @mock.patch(MISTRAL_WBS + "._create_workbook")
+    def test_create_workbook(self, mock_create):
+        mistral_scenario = workbooks.MistralWorkbooks()
+        definition = "---\nversion: \"2.0\"\nname: wb"
+        fake_wb = mock.MagicMock()
+        fake_wb.name = "wb"
+        mock_create.return_value = fake_wb
+        mistral_scenario.create_delete_workbook(definition)
+
+        self.assertEqual(1, mock_create.called)
+
+    @mock.patch(MISTRAL_WBS + "._delete_workbook")
+    @mock.patch(MISTRAL_WBS + "._create_workbook")
+    def test_create_delete_workbook(self, mock_create, mock_delete):
+        mistral_scenario = workbooks.MistralWorkbooks()
+        definition = "---\nversion: \"2.0\"\nname: wb"
+        fake_wb = mock.MagicMock()
+        fake_wb.name = "wb"
+        mock_create.return_value = fake_wb
+        mistral_scenario.create_delete_workbook(definition, do_delete=True)
+
+        self.assertEqual(1, mock_create.called)
+        mock_delete.assert_called_once_with(fake_wb.name)

tests/unit/benchmark/test_types.py

@@ -251,3 +251,26 @@ class PreprocessTestCase(test.TestCase):
         mock_osclients.Clients.assert_called_once_with(
             context["admin"]["endpoint"])
         self.assertEqual({"a": 20, "b": 20}, result)
+
+
+class FileTypeTestCase(test.TestCase):
+
+    def setUp(self):
+        super(FileTypeTestCase, self).setUp()
+        self.clients = fakes.FakeClients()
+
+    @mock.patch("rally.benchmark.types.open",
+                side_effect=mock.mock_open(read_data="file_context"),
+                create=True)
+    def test_transform_by_path(self, mock_open):
+        resource_config = "file.yaml"
+        file_context = types.FileType.transform(
+                           clients=self.clients,
+                           resource_config=resource_config)
+        self.assertEqual(file_context, "file_context")
+
+    def test_transform_by_path_no_match(self):
+        resource_config = "nonexistant.yaml"
+        self.assertRaises(IOError,
+                          types.FileType.transform,
+                          self.clients, resource_config)

tests/unit/benchmark/test_validation.py

@@ -566,3 +566,17 @@ class ValidatorsTestCase(test.TestCase):
 
         result = validator(context, clients, mock.MagicMock())
         self.assertFalse(result.is_valid, result.msg)
+
+    def test_required_clients(self):
+        validator = self._unwrap_validator(validation.required_clients,
+                                           "keystone", "nova")
+
+        clients = mock.MagicMock()
+        clients.keystone.return_value = "keystone"
+        clients.nova.return_value = "nova"
+        result = validator({}, clients, None)
+        self.assertTrue(result.is_valid, result.msg)
+
+        clients.nova.side_effect = ImportError
+        result = validator({}, clients, None)
+        self.assertFalse(result.is_valid, result.msg)

tests/unit/fakes.py

@@ -279,6 +279,12 @@ class FakeAvailabilityZone(FakeResource):
         self.hosts = mock.MagicMock()
 
 
+class FakeWorkbook(FakeResource):
+    def __init__(self, manager=None):
+        super(FakeWorkbook, self).__init__(manager)
+        self.workbook = mock.MagicMock()
+
+
 class FakeManager(object):
 
     def __init__(self):
@@ -806,6 +812,15 @@ class FakeAvailabilityZonesManager(FakeManager):
         return [self.zones]
 
 
+class FakeWorkbookManager(FakeManager):
+    def __init__(self):
+        super(FakeWorkbookManager, self).__init__()
+        self.workbook = FakeWorkbook()
+
+    def list(self):
+        return [self.workbook]
+
+
 class FakeServiceCatalog(object):
     def get_endpoints(self):
         return {"image": [{"publicURL": "http://fake.to"}],
@@ -1188,6 +1203,12 @@ class FakeTroveClient(object):
         self.instances = FakeDbInstanceManager()
 
 
+class FakeMistralClient(object):
+
+    def __init__(self):
+        self.workbook = FakeWorkbookManager()
+
+
 class FakeClients(object):
 
     def __init__(self, endpoint_=None):
@@ -1202,6 +1223,7 @@ class FakeClients(object):
         self._ceilometer = None
         self._zaqar = None
         self._trove = None
+        self._mistral = None
         self._endpoint = endpoint_ or objects.Endpoint(
             "http://fake.example.org:5000/v2.0/",
             "fake_username",
@@ -1266,6 +1288,11 @@ class FakeClients(object):
             self._trove = FakeTroveClient()
         return self._trove
 
+    def mistral(self):
+        if not self._mistral:
+            self._mistral = FakeMistralClient()
+        return self._mistral
+
 
 class FakeRunner(object):
 

tests/unit/test_osclients.py

@@ -262,6 +262,30 @@ class OSClientsTestCase(test.TestCase):
         mock_trove.Client.assert_called_once_with("1.0", **kw)
         self.assertEqual(self.clients.cache["trove"], fake_trove)
 
+    def test_mistral(self):
+        fake_mistral = fakes.FakeMistralClient()
+        mock_mistral = mock.Mock()
+        mock_mistral.client.client.return_value = fake_mistral
+
+        self.assertNotIn("mistral", self.clients.cache)
+        with mock.patch.dict(
+                "sys.modules", {"mistralclient": mock_mistral,
+                                "mistralclient.api": mock_mistral}):
+            client = self.clients.mistral()
+            self.assertEqual(fake_mistral, client)
+            self.service_catalog.url_for.assert_called_once_with(
+                service_type="workflowv2",
+                endpoint_type=consts.EndpointType.PUBLIC,
+                region_name=self.endpoint.region_name
+            )
+            fake_mistral_url = self.service_catalog.url_for.return_value
+            mock_mistral.client.client.assert_called_once_with(
+                 mistral_url=fake_mistral_url,
+                 service_type="workflowv2",
+                 auth_token=self.fake_keystone.auth_token
+            )
+            self.assertEqual(fake_mistral, self.clients.cache["mistral"])
+
     @mock.patch("rally.osclients.Clients.keystone")
     def test_services(self, mock_keystone):
         available_services = {consts.ServiceType.IDENTITY: {},