Add monasca benchmark in plugin - Part 0: metrics

* add new context of creating metrics via monasca
* add metric list benchmark
* add metric list benchmark task config example

Change-Id: Ib16941a4b9d59a2ff51e93cfb178bc12c3df26cf

optional-requirements.txt

@@ -1,7 +1,6 @@
 python-mistralclient>=2.0.0
 python-fuelclient==6.1.0                               # Apache Software License
 python-muranoclient>=0.8.2                             # Apache License, Version 2.0
-python-monascaclient>=1.1.0                            # Apache Software License
 python-cueclient>=1.0.0,<=1.0.0                        # Apache License, Version 2.0
 python-senlinclient>=0.3.0                             # Apache Software License
 python-magnumclient>=2.0.0                             # Apache Software License

rally-jobs/rally-monasca.yaml

@@ -0,0 +1,39 @@
+---
+  MonascaMetrics.list_metrics:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+        roles:
+         - "monasca-user"
+        monasca_metrics:
+          "dimensions":
+              "region": "RegionOne"
+              "service": "identity"
+              "hostname": "fake_host"
+              "url": "http://fake_host:5000/v2.0"
+          "metrics_per_tenant": 10
+      sla:
+        failure_rate:
+          max: 0
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 2
+      context:
+        users:
+          tenants: 2
+          users_per_tenant: 2
+        roles:
+         - "monasca-user"
+        monasca_metrics:
+          "metrics_per_tenant": 10
+      sla:
+        failure_rate:
+          max: 0

rally/common/opts.py

@@ -24,6 +24,7 @@ from rally.plugins.openstack.scenarios.ec2 import utils as ec2_utils
 from rally.plugins.openstack.scenarios.heat import utils as heat_utils
 from rally.plugins.openstack.scenarios.ironic import utils as ironic_utils
 from rally.plugins.openstack.scenarios.manila import utils as manila_utils
+from rally.plugins.openstack.scenarios.monasca import utils as monasca_utils
 from rally.plugins.openstack.scenarios.murano import utils as murano_utils
 from rally.plugins.openstack.scenarios.nova import utils as nova_utils
 from rally.plugins.openstack.scenarios.sahara import utils as sahara_utils
@@ -44,6 +45,7 @@ def list_opts():
                          heat_utils.HEAT_BENCHMARK_OPTS,
                          ironic_utils.IRONIC_BENCHMARK_OPTS,
                          manila_utils.MANILA_BENCHMARK_OPTS,
+                         monasca_utils.MONASCA_BENCHMARK_OPTS,
                          murano_utils.MURANO_BENCHMARK_OPTS,
                          nova_utils.NOVA_BENCHMARK_OPTS,
                          sahara_utils.SAHARA_BENCHMARK_OPTS,

rally/plugins/openstack/context/monasca/metrics.py

@@ -0,0 +1,106 @@
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from six import moves
+
+from rally.common.i18n import _
+from rally.common import logging
+from rally.common import utils as rutils
+from rally import consts
+from rally.plugins.openstack.scenarios.monasca import utils as monasca_utils
+from rally.task import context
+
+
+LOG = logging.getLogger(__name__)
+
+
+@context.configure(name="monasca_metrics", order=510)
+class MonascaMetricGenerator(context.Context):
+    """Context for creating metrics  for benchmarks."""
+
+    CONFIG_SCHEMA = {
+        "type": "object",
+        "$schema": consts.JSON_SCHEMA,
+        "properties": {
+            "name": {
+                "type": "string"
+            },
+            "dimensions": {
+                "type": "object",
+                "properties": {
+                    "region": {
+                        "type": "string"
+                    },
+                    "service": {
+                        "type": "string"
+                    },
+                    "hostname": {
+                        "type": "string"
+                    },
+                    "url": {
+                        "type": "string"
+                    }
+                }
+            },
+            "metrics_per_tenant": {
+                "type": "integer",
+                "minimum": 1
+            },
+            "value_meta": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "value_meta_key": {
+                            "type": "string"
+                        },
+                        "value_meta_value": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
+        "additionalProperties": False
+    }
+
+    DEFAULT_CONFIG = {
+        "metrics_per_tenant": 2
+    }
+
+    @logging.log_task_wrapper(LOG.info, _("Enter context: `Monasca`"))
+    def setup(self):
+        new_metric = {}
+
+        if "dimensions" in self.config:
+            new_metric = {
+                "dimensions": self.config["dimensions"]
+            }
+
+        for user, tenant_id in rutils.iterate_per_tenants(
+                self.context["users"]):
+            scenario = monasca_utils.MonascaScenario(
+                context={"user": user, "task": self.context["task"]}
+            )
+            for i in moves.xrange(self.config["metrics_per_tenant"]):
+                scenario._create_metrics(**new_metric)
+                rutils.interruptable_sleep(0.001)
+        rutils.interruptable_sleep(
+            monasca_utils.CONF.benchmark.monasca_metric_create_prepoll_delay,
+            atomic_delay=1)
+
+    @logging.log_task_wrapper(LOG.info, _("Exit context: `Monasca`"))
+    def cleanup(self):
+        # We don't have API for removal of metrics
+        pass

rally/plugins/openstack/scenarios/monasca/metrics.py

@@ -0,0 +1,37 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from rally.common import log as logging
+from rally import consts
+from rally.plugins.openstack import scenario
+from rally.plugins.openstack.scenarios.monasca import utils as monascautils
+from rally.task import validation
+
+LOG = logging.getLogger(__name__)
+
+
+class MonascaMetrics(monascautils.MonascaScenario):
+    """Benchmark scenarios for monasca Metrics API."""
+
+    @validation.required_clients("monasca")
+    @validation.required_services(consts.Service.MONASCA)
+    @validation.required_openstack(users=True)
+    @scenario.configure()
+    def list_metrics(self, **kwargs):
+        """Fetch user's metrics.
+
+        :param kwargs: optional arguments for list query:
+               name, dimensions, start_time, etc
+        """
+        self._list_metrics(**kwargs)

rally/plugins/openstack/scenarios/monasca/utils.py

@@ -0,0 +1,64 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import random
+import time
+import uuid
+
+from oslo_config import cfg
+
+from rally.plugins.openstack import scenario
+from rally.task import atomic
+
+
+MONASCA_BENCHMARK_OPTS = [
+    cfg.FloatOpt(
+        "monasca_metric_create_prepoll_delay",
+        default=15.0,
+        help="Delay between creating Monasca metrics and polling for "
+             "its elements.")
+]
+
+CONF = cfg.CONF
+benchmark_group = cfg.OptGroup(name="benchmark", title="benchmark options")
+CONF.register_opts(MONASCA_BENCHMARK_OPTS, group=benchmark_group)
+
+
+class MonascaScenario(scenario.OpenStackScenario):
+    """Base class for Monasca scenarios with basic atomic actions."""
+
+    @atomic.action_timer("monasca.list_metrics")
+    def _list_metrics(self, **kwargs):
+        """Get list of user's metrics.
+
+        :param kwargs: optional arguments for list query:
+                       name, dimensions, start_time, etc
+        :returns list of monasca metrics
+        """
+        return self.clients("monasca").metrics.list(**kwargs)
+
+    @atomic.action_timer("monasca.create_metrics")
+    def _create_metrics(self, **kwargs):
+        """Create user metrics.
+
+        :param kwargs: attributes for metric creation:
+                       name, dimension, timestamp, value, etc
+        """
+        timestamp = int(time.time() * 1000)
+        kwargs.update({"name": self.generate_random_name(),
+                       "timestamp": timestamp,
+                       "value": random.random(),
+                       "value_meta": {
+                           "key": str(uuid.uuid4())[:10]}})
+        self.clients("monasca").metrics.create(**kwargs)

requirements.txt

@@ -27,6 +27,7 @@ python-novaclient!=2.33.0,>=2.29.0                     # Apache License, Version
 python-neutronclient>=4.2.0                            # Apache Software License
 python-cinderclient!=1.7.0,!=1.7.1,>=1.6.0             # Apache Software License
 python-manilaclient>=1.10.0                            # Apache Software License
+python-monascaclient>=1.2.0                            # Apache Software License
 python-heatclient>=1.1.0                               # Apache Software License
 python-ceilometerclient>=2.2.1                         # Apache Software License
 python-ironicclient>=1.1.0                             # Apache Software License

samples/tasks/scenarios/monasca/list-metrics.json

@@ -0,0 +1,33 @@
+{
+    "MonascaMetrics.list_metrics": [
+        {
+            "runner": {
+                "type": "constant",
+                "times": 10,
+                "concurrency": 1
+            },
+            "context": {
+                "users": {
+                    "tenants": 1,
+                    "users_per_tenant": 1
+                },
+                "roles": [
+                    "monasca-user"
+                ],
+                "monasca_metrics": {
+                    "dimensions": {
+                      "region": "RegionOne",
+                      "service": "identity",
+                      "hostname": "fake_host",
+                      "url": "http://fake_host:5000/v2.0"
+                    },
+                    "metrics_per_tenant": 10
+                }
+            },
+            "args": {
+                "region": "RegionOne",
+                "limit": 5
+            }
+        }
+    ]
+}

samples/tasks/scenarios/monasca/list-metrics.yaml

@@ -0,0 +1,23 @@
+---
+  MonascaMetrics.list_metrics:
+    -
+      runner:
+        type: "constant"
+        times: 10
+        concurrency: 1
+      context:
+        users:
+          tenants: 1
+          users_per_tenant: 1
+        roles:
+         - "monasca-user"
+        monasca_metrics:
+          "dimensions":
+              "region": "RegionOne"
+              "service": "identity"
+              "hostname": "fake_host"
+              "url": "http://fake_host:5000/v2.0"
+          "metrics_per_tenant": 10
+      args:
+        "region": "RegionOne"
+        "limit": 5

tests/unit/plugins/openstack/context/monasca/test_metrics.py

@@ -0,0 +1,100 @@
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import six
+
+from rally.plugins.openstack.context.monasca import metrics
+from rally.plugins.openstack.scenarios.monasca import utils as monasca_utils
+from tests.unit import test
+
+CTX = "rally.plugins.openstack.context.monasca"
+
+
+class MonascaMetricGeneratorTestCase(test.TestCase):
+
+    def _gen_tenants(self, count):
+        tenants = {}
+        for id in six.moves.range(count):
+            tenants[str(id)] = {"name": str(id)}
+        return tenants
+
+    def _gen_context(self, tenants_count, users_per_tenant,
+                     metrics_per_tenant):
+        tenants = self._gen_tenants(tenants_count)
+        users = []
+        for id in tenants.keys():
+            for i in six.moves.range(users_per_tenant):
+                users.append({"id": i, "tenant_id": id,
+                              "endpoint": mock.MagicMock()})
+        context = test.get_test_context()
+        context.update({
+            "config": {
+                "users": {
+                    "tenants": tenants_count,
+                    "users_per_tenant": users_per_tenant,
+                    "concurrent": 10,
+                },
+                "monasca_metrics": {
+                    "name": "fake-metric-name",
+                    "dimensions": {
+                        "region": "fake-region",
+                        "service": "fake-identity",
+                        "hostname": "fake-hostname",
+                        "url": "fake-url"
+                    },
+                    "metrics_per_tenant": metrics_per_tenant,
+                },
+                "roles": [
+                    "monasca-user"
+                ]
+            },
+            "admin": {
+                "endpoint": mock.MagicMock()
+            },
+            "users": users,
+            "tenants": tenants
+        })
+        return tenants, context
+
+    @mock.patch("%s.metrics.rutils.interruptable_sleep" % CTX)
+    @mock.patch("%s.metrics.monasca_utils.MonascaScenario" % CTX)
+    def test_setup(self, mock_monasca_scenario, mock_interruptable_sleep):
+        tenants_count = 2
+        users_per_tenant = 4
+        metrics_per_tenant = 5
+
+        tenants, real_context = self._gen_context(
+            tenants_count, users_per_tenant, metrics_per_tenant)
+
+        monasca_ctx = metrics.MonascaMetricGenerator(real_context)
+        monasca_ctx.setup()
+
+        self.assertEqual(tenants_count, mock_monasca_scenario.call_count,
+                         "Scenario should be constructed same times as "
+                         "number of tenants")
+        self.assertEqual(metrics_per_tenant * tenants_count,
+                         mock_monasca_scenario.return_value._create_metrics.
+                         call_count,
+                         "Total number of metrics created should be tenant"
+                         "counts times metrics per tenant")
+        first_call = mock.call(0.001)
+        second_call = mock.call(monasca_utils.CONF.benchmark.
+                                monasca_metric_create_prepoll_delay,
+                                atomic_delay=1)
+        self.assertEqual([first_call] * metrics_per_tenant * tenants_count +
+                         [second_call],
+                         mock_interruptable_sleep.call_args_list,
+                         "Method interruptable_sleep should be called "
+                         "tenant counts times metrics plus one")

tests/unit/plugins/openstack/scenarios/monasca/test_metrics.py

@@ -0,0 +1,35 @@
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import ddt
+import mock
+
+from rally.plugins.openstack.scenarios.monasca import metrics
+from tests.unit import test
+
+
+@ddt.ddt
+class MonascaMetricsTestCase(test.ScenarioTestCase):
+
+    @ddt.data(
+        {"region": None},
+        {"region": "fake_region"},
+    )
+    @ddt.unpack
+    def test_list_metrics(self, region=None):
+        scenario = metrics.MonascaMetrics()
+        self.region = region
+        scenario._list_metrics = mock.MagicMock()
+        scenario.list_metrics(region=self.region)
+        scenario._list_metrics.assert_called_once_with(region=self.region)

tests/unit/plugins/openstack/scenarios/monasca/test_utils.py

@@ -0,0 +1,51 @@
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import ddt
+
+from rally.plugins.openstack.scenarios.monasca import utils
+from tests.unit import test
+
+
+@ddt.ddt
+class MonascaScenarioTestCase(test.ScenarioTestCase):
+
+    def setUp(self):
+        super(MonascaScenarioTestCase, self).setUp()
+        self.scenario = utils.MonascaScenario(self.context)
+        self.kwargs = {
+            "dimensions": {
+                "region": "fake_region",
+                "hostname": "fake_host_name",
+                "service": "fake_service",
+                "url": "fake_url"
+            }
+        }
+
+    def test_list_metrics(self):
+        return_metric_value = self.scenario._list_metrics()
+        self.assertEqual(return_metric_value,
+                         self.clients("monasca").metrics.list.return_value)
+        self._test_atomic_action_timer(self.scenario.atomic_actions(),
+                                       "monasca.list_metrics")
+
+    @ddt.data(
+        {"name": ""},
+        {"name": "fake_metric"},
+    )
+    @ddt.unpack
+    def test_create_metrics(self, name=None):
+        self.name = name
+        self.scenario._create_metrics(name=self.name, kwargs=self.kwargs)
+        self.assertEqual(1, self.clients("monasca").metrics.create.call_count)